TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-04-15 10:21:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Code cleanup

Browse Source
This commit is contained in:
Isaac Abadi
2021-08-24 22:05:02 -06:00
parent d899f88164
commit cca76dd248
5 changed files with 16 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/consts.js
View File

@@ -134,14 +134,6 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_subscriptions_check_interval', 'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval' 'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
}, },
'ytdl_subscriptions_check_interval': {
'key': 'ytdl_subscriptions_check_interval',
'path': 'YoutubeDLMaterial.Subscriptions.subscriptions_check_interval'
},
'ytdl_subscriptions_download_delay': {
'key': 'ytdl_subscriptions_download_delay',
'path': 'YoutubeDLMaterial.Subscriptions.download_delay'
},
'ytdl_subscriptions_redownload_fresh_uploads': { 'ytdl_subscriptions_redownload_fresh_uploads': {
'key': 'ytdl_subscriptions_redownload_fresh_uploads', 'key': 'ytdl_subscriptions_redownload_fresh_uploads',
'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads' 'path': 'YoutubeDLMaterial.Subscriptions.redownload_fresh_uploads'

95
backend/db.js
View File

@@ -155,51 +155,17 @@ exports._connectToDB = async (custom_connection_string = null) => {
} }
} }
exports.registerFileDB = async (file_path, type, multiUserMode = null, sub = null, customPath = null, category = null, cropFileSettings = null, file_object = null) => { exports.registerFileDB = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
let db_path = null; if (!file_object) file_object = generateFileObject(file_path, type);
const file_id = utils.removeFileExtension(file_path);
if (!file_object) file_object = generateFileObject(file_id, type, customPath || multiUserMode && multiUserMode.file_path, sub);
if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_id}`);
return false;
}
utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path);
// add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, customPath || multiUserMode && multiUserMode.file_path);
// if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
// modify duration
if (cropFileSettings) {
file_object['duration'] = (cropFileSettings.cropFileEnd || file_object.duration) - cropFileSettings.cropFileStart;
}
if (multiUserMode) file_object['user_uid'] = multiUserMode.user;
const file_obj = await registerFileDBManual(file_object);
// remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
}
return file_obj;
}
exports.registerFileDB2 = async (file_path, type, user_uid = null, category = null, sub_id = null, cropFileSettings = null, file_object = null) => {
if (!file_object) file_object = generateFileObject2(file_path, type);
if (!file_object) { if (!file_object) {
logger.error(`Could not find associated JSON file for ${type} file ${file_path}`); logger.error(`Could not find associated JSON file for ${type} file ${file_path}`);
return false; return false;
} }
utils.fixVideoMetadataPerms2(file_path, type); utils.fixVideoMetadataPerms(file_path, type);
// add thumbnail path // add thumbnail path
file_object['thumbnailPath'] = utils.getDownloadedThumbnail2(file_path, type); file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_path);
// if category exists, only include essential info // if category exists, only include essential info
if (category) file_object['category'] = {name: category['name'], uid: category['uid']}; if (category) file_object['category'] = {name: category['name'], uid: category['uid']};
@@ -216,7 +182,7 @@ exports.registerFileDB2 = async (file_path, type, user_uid = null, category = nu
// remove metadata JSON if needed // remove metadata JSON if needed
if (!config_api.getConfigItem('ytdl_include_metadata')) { if (!config_api.getConfigItem('ytdl_include_metadata')) {
utils.deleteJSONFile2(file_path, type) utils.deleteJSONFile(file_path, type)
} }
return file_obj; return file_obj;
@@ -234,36 +200,7 @@ async function registerFileDBManual(file_object) {
return file_object; return file_object;
} }
function generateFileObject(id, type, customPath = null, sub = null) { function generateFileObject(file_path, type) {
if (!customPath && sub) {
customPath = getAppendedBasePathSub(sub, config_api.getConfigItem('ytdl_subscriptions_base_path'));
}
var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true);
if (!jsonobj) {
return null;
}
const ext = (type === 'audio') ? '.mp3' : '.mp4'
const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext);
// console.
var stats = fs.statSync(path.join(__dirname, file_path));
var title = jsonobj.title;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A';
var size = stats.size;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var isaudio = type === 'audio';
var description = jsonobj.description;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date, description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
return file_obj;
}
function generateFileObject2(file_path, type) {
var jsonobj = utils.getJSON(file_path, type); var jsonobj = utils.getJSON(file_path, type);
if (!jsonobj) { if (!jsonobj) {
return null; return null;
@@ -380,7 +317,7 @@ exports.importUnregisteredFiles = async () => {
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) { if (!file_is_registered) {
// add additional info // add additional info
await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
logger.verbose(`Added discovered file to the database: ${file.id}`); logger.verbose(`Added discovered file to the database: ${file.id}`);
} }
} }
@@ -388,24 +325,6 @@ exports.importUnregisteredFiles = async () => {
} }
exports.preimportUnregisteredSubscriptionFile = async (sub, appendedBasePath) => {
const preimported_file_paths = [];
const files = await utils.getDownloadedFilesByType(appendedBasePath, sub.type);
for (let i = 0; i < files.length; i++) {
const file = files[i];
// check if file exists in db, if not add it
const file_is_registered = await exports.getRecord('files', {id: file.id, sub_id: sub.id});
if (!file_is_registered) {
// add additional info
await exports.registerFileDB2(file['path'], sub.type, sub.user_uid, null, sub.id, null, file);
preimported_file_paths.push(file['path']);
logger.verbose(`Preemptively added subscription file to the database: ${file.id}`);
}
}
return preimported_file_paths;
}
exports.addMetadataPropertyToDB = async (property_key) => { exports.addMetadataPropertyToDB = async (property_key) => {
try { try {
const dirs_to_check = await exports.getFileDirectoriesAndDBs(); const dirs_to_check = await exports.getFileDirectoriesAndDBs();

14
backend/downloader.js
View File

@@ -194,9 +194,7 @@ async function collectInfo(download_uid) {
let info = await getVideoInfoByURL(url, args, download_uid); let info = await getVideoInfoByURL(url, args, download_uid);
if (!info) { if (!info) {
// info failed, record error and pause download // info failed, error presumably already recorded
const error = 'Failed to get info, see server logs for specific error.';
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false});
return; return;
} }
@@ -315,7 +313,7 @@ async function downloadQueuedFile(download_uid) {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']); fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name); logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) { } catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
} }
} }
@@ -333,7 +331,7 @@ async function downloadQueuedFile(download_uid) {
} }
// registers file in DB // registers file in DB
const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
file_objs.push(file_obj); file_objs.push(file_obj);
} }
@@ -544,9 +542,9 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
} }
resolve(outputs.length === 1 ? outputs[0] : outputs); resolve(outputs.length === 1 ? outputs[0] : outputs);
} catch(e) { } catch(e) {
logger.error(`Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`); const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) { if (download_uid) {
const error = 'Failed to get info, see server logs for specific error.';
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false});
} }
resolve(null); resolve(null);
@@ -558,7 +556,7 @@ async function getVideoInfoByURL(url, args = [], download_uid = null) {
} }
if (download_uid) { if (download_uid) {
const error = 'Failed to get info, see server logs for specific error.'; const error = 'Failed to get info, see server logs for specific error.';
await db_api.updateRecord('download_queue', {uid: download_uid}, {error: error, paused: true, running: false}); await db_api.updateRecord('download_queue', {uid: download_uid}, {error: err.stderr ? err.stderr : error, paused: true, running: false});
} }
resolve(null); resolve(null);
} }

1
backend/subscriptions.js
View File

@@ -3,7 +3,6 @@ const path = require('path');
const youtubedl = require('youtube-dl'); const youtubedl = require('youtube-dl');
const config_api = require('./config'); const config_api = require('./config');
const twitch_api = require('./twitch');
const utils = require('./utils'); const utils = require('./utils');
const logger = require('./logger'); const logger = require('./logger');

61
backend/utils.js
View File

@@ -144,24 +144,7 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
} }
function getDownloadedThumbnail(name, type, customPath = null) { function getDownloadedThumbnail(file_path) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
let jpgPath = path.join(customPath, name + '.jpg');
let webpPath = path.join(customPath, name + '.webp');
let pngPath = path.join(customPath, name + '.png');
if (fs.existsSync(jpgPath))
return jpgPath;
else if (fs.existsSync(webpPath))
return webpPath;
else if (fs.existsSync(pngPath))
return pngPath;
else
return null;
}
function getDownloadedThumbnail2(file_path, type) {
const file_path_no_extension = removeFileExtension(file_path); const file_path_no_extension = removeFileExtension(file_path);
let jpgPath = file_path_no_extension + '.jpg'; let jpgPath = file_path_no_extension + '.jpg';
@@ -199,29 +182,7 @@ function getExpectedFileSize(input_info_jsons) {
return expected_filesize; return expected_filesize;
} }
function fixVideoMetadataPerms(name, type, customPath = null) { function fixVideoMetadataPerms(file_path, type) {
if (is_windows) return;
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
path.join(customPath, name + ext + '.info.json'),
// Thumbnails
path.join(customPath, name + '.webp'),
path.join(customPath, name + '.jpg')
];
for (const file of files_to_fix) {
if (!fs.existsSync(file)) continue;
fs.chmodSync(file, 0o644);
}
}
function fixVideoMetadataPerms2(file_path, type) {
if (is_windows) return; if (is_windows) return;
const ext = type === 'audio' ? '.mp3' : '.mp4'; const ext = type === 'audio' ? '.mp3' : '.mp4';
@@ -243,19 +204,7 @@ function fixVideoMetadataPerms2(file_path, type) {
} }
} }
function deleteJSONFile(name, type, customPath = null) { function deleteJSONFile(file_path, type) {
if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
let json_path = path.join(customPath, name + '.info.json');
let alternate_json_path = path.join(customPath, name + ext + '.info.json');
if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
}
function deleteJSONFile2(file_path, type) {
const ext = type === 'audio' ? '.mp3' : '.mp4'; const ext = type === 'audio' ? '.mp3' : '.mp4';
const file_path_no_extension = removeFileExtension(file_path); const file_path_no_extension = removeFileExtension(file_path);
@@ -291,7 +240,6 @@ async function removeIDFromArchive(archive_path, id) {
const updatedData = dataArray.join('\n'); const updatedData = dataArray.join('\n');
await fs.writeFile(archive_path, updatedData); await fs.writeFile(archive_path, updatedData);
if (line) return line; if (line) return line;
if (err) throw err;
} }
function durationStringToNumber(dur_str) { function durationStringToNumber(dur_str) {
@@ -429,12 +377,9 @@ module.exports = {
getJSON: getJSON, getJSON: getJSON,
getTrueFileName: getTrueFileName, getTrueFileName: getTrueFileName,
getDownloadedThumbnail: getDownloadedThumbnail, getDownloadedThumbnail: getDownloadedThumbnail,
getDownloadedThumbnail2: getDownloadedThumbnail2,
getExpectedFileSize: getExpectedFileSize, getExpectedFileSize: getExpectedFileSize,
fixVideoMetadataPerms: fixVideoMetadataPerms, fixVideoMetadataPerms: fixVideoMetadataPerms,
fixVideoMetadataPerms2: fixVideoMetadataPerms2,
deleteJSONFile: deleteJSONFile, deleteJSONFile: deleteJSONFile,
deleteJSONFile2: deleteJSONFile2,
removeIDFromArchive: removeIDFromArchive, removeIDFromArchive: removeIDFromArchive,
getDownloadedFilesByType: getDownloadedFilesByType, getDownloadedFilesByType: getDownloadedFilesByType,
createContainerZipFile: createContainerZipFile, createContainerZipFile: createContainerZipFile,