From 7ff906fd35e57310b1907d34b159aadcaa0b3eb3 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Thu, 22 Jul 2021 23:04:29 -0600 Subject: [PATCH 01/11] Added issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 31 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 17 +++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..7e1a855 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** +- YoutubeDL-Material version +- Docker tag: (optional) + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d233fde --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]" +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. From 7174ef5f5777fe6dcf74b8363da7cd4a6a10a71b Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 26 Jul 2021 18:25:41 -0700 Subject: [PATCH 02/11] Fixed issue where config initialization did not occur early enough in lifecycle, causing db.js to throw an error if the config did not exist --- backend/db.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/db.js b/backend/db.js index c56db40..8e510b0 100644 --- a/backend/db.js +++ b/backend/db.js @@ -54,7 +54,7 @@ const local_db_defaults = {} tables_list.forEach(table => {local_db_defaults[table] = []}); local_db.defaults(local_db_defaults).write(); -let using_local_db = config_api.getConfigItem('ytdl_use_local_db'); +let using_local_db = null; function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db; @@ -69,6 +69,9 @@ function setLogger(input_logger) { exports.initialize = (input_db, input_users_db, input_logger) => { setDB(input_db, input_users_db); setLogger(input_logger); + + // must be done here to prevent getConfigItem from being called before init + using_local_db = config_api.getConfigItem('ytdl_use_local_db'); } exports.connectToDB = async (retries = 5, no_fallback = false) => { From 92413bd360397048402735a4509edd72aa2a7717 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 26 Jul 2021 20:10:22 -0700 Subject: [PATCH 03/11] Added ability to add file to playlist using the context menu --- backend/app.js | 47 +++++++++++-------- backend/db.js | 23 +++++---- .../custom-playlists.component.ts | 7 +++ .../recent-videos.component.html | 2 +- .../recent-videos/recent-videos.component.ts | 38 +++++++++++++++ .../unified-file-card.component.html | 4 ++ .../unified-file-card.component.ts | 9 ++++ .../modify-playlist.component.ts | 1 + src/app/main/main.component.ts | 4 +- src/app/posts.services.ts | 21 +++++++-- 10 files changed, 119 insertions(+), 37 deletions(-) diff --git a/backend/app.js b/backend/app.js index 4b89a0e..3ef0f3f 100644 --- a/backend/app.js +++ b/backend/app.js @@ -754,20 +754,6 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -function getThumbnailMp3(name) -{ - var obj = utils.getJSONMp3(name, audioFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - -function getThumbnailMp4(name) -{ - var obj = utils.getJSONMp4(name, videoFolderPath); - var thumbnailLink = obj.thumbnail; - return thumbnailLink; -} - function getFileSizeMp3(name) { var jsonPath = audioFolderPath+name+".mp3.info.json"; @@ -1061,7 +1047,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { // create playlist const playlist_name = file_objs.map(file_obj => file_obj.title).join(', '); const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0); - container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, file_objs[0]['thumbnailURL'], options.user); + container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user); } else if (file_objs.length === 1) { container = file_objs[0]; } else { @@ -2181,9 +2167,8 @@ app.post('/api/createPlaylist', optionalJwt, async (req, res) => { let playlistName = req.body.playlistName; let uids = req.body.uids; let type = req.body.type; - let thumbnailURL = req.body.thumbnailURL; - const new_playlist = await db_api.createPlaylist(playlistName, uids, type, thumbnailURL, req.isAuthenticated() ? req.user.uid : null); + const new_playlist = await db_api.createPlaylist(playlistName, uids, type, req.isAuthenticated() ? req.user.uid : null); res.send({ new_playlist: new_playlist, @@ -2216,8 +2201,18 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { }); }); +app.post('/api/getPlaylists', optionalJwt, async (req, res) => { + const uuid = req.isAuthenticated() ? req.user.uid : null; + + const playlists = await db_api.getRecords('playlists', {user_uid: uuid}); + + res.send({ + playlists: playlists + }); +}); + app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { - let playlistID = req.body.playlistID; + let playlistID = req.body.playlist_id; let uids = req.body.uids; let success = false; @@ -2238,6 +2233,20 @@ app.post('/api/updatePlaylistFiles', optionalJwt, async (req, res) => { }) }); +app.post('/api/addFileToPlaylist', optionalJwt, async (req, res) => { + let playlist_id = req.body.playlist_id; + let file_uid = req.body.file_uid; + + const playlist = await db_api.getRecord('playlists', {id: playlist_id}); + + playlist.uids.push(file_uid); + + let success = await db_api.updatePlaylist(playlist); + res.send({ + success: success + }); +}); + app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { let playlist = req.body.playlist; let success = await db_api.updatePlaylist(playlist, req.user && req.user.uid); @@ -2247,7 +2256,7 @@ app.post('/api/updatePlaylist', optionalJwt, async (req, res) => { }); app.post('/api/deletePlaylist', optionalJwt, async (req, res) => { - let playlistID = req.body.playlistID; + let playlistID = req.body.playlist_id; let success = null; try { diff --git a/backend/db.js b/backend/db.js index 8e510b0..47c4f00 100644 --- a/backend/db.js +++ b/backend/db.js @@ -411,23 +411,26 @@ exports.addMetadataPropertyToDB = async (property_key) => { } } -exports.createPlaylist = async (playlist_name, uids, type, thumbnail_url, user_uid = null) => { +exports.createPlaylist = async (playlist_name, uids, type, user_uid = null) => { + const first_video = await exports.getVideo(uids[0]); + const thumbnailToUse = first_video['thumbnailURL']; + let new_playlist = { name: playlist_name, uids: uids, id: uuid(), - thumbnailURL: thumbnail_url, + thumbnailURL: thumbnailToUse, type: type, registered: Date.now(), randomize_order: false }; - const duration = await exports.calculatePlaylistDuration(new_playlist, user_uid); - new_playlist.duration = duration; - new_playlist.user_uid = user_uid ? user_uid : undefined; await exports.insertRecordIntoTable('playlists', new_playlist); + + const duration = await exports.calculatePlaylistDuration(new_playlist); + await exports.updateRecord('playlists', {id: new_playlist.id}, {duration: duration}); return new_playlist; } @@ -463,10 +466,10 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal return playlist; } -exports.updatePlaylist = async (playlist, user_uid = null) => { +exports.updatePlaylist = async (playlist) => { let playlistID = playlist.id; - const duration = await exports.calculatePlaylistDuration(playlist, user_uid); + const duration = await exports.calculatePlaylistDuration(playlist); playlist.duration = duration; return await exports.updateRecord('playlists', {id: playlistID}, playlist); @@ -486,12 +489,12 @@ exports.setPlaylistProperty = async (playlist_id, assignment_obj, user_uid = nul return success; } -exports.calculatePlaylistDuration = async (playlist, uuid, playlist_file_objs = null) => { +exports.calculatePlaylistDuration = async (playlist, playlist_file_objs = null) => { if (!playlist_file_objs) { playlist_file_objs = []; for (let i = 0; i < playlist['uids'].length; i++) { const uid = playlist['uids'][i]; - const file_obj = await exports.getVideo(uid, uuid); + const file_obj = await exports.getVideo(uid); if (file_obj) playlist_file_objs.push(file_obj); } } @@ -588,7 +591,7 @@ exports.getVideoUIDByID = async (file_id, uuid = null) => { return file_obj ? file_obj['uid'] : null; } -exports.getVideo = async (file_uid, uuid = null, sub_id = null) => { +exports.getVideo = async (file_uid) => { return await exports.getRecord('files', {uid: file_uid}); } diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index 6071d3d..98819e1 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -24,10 +24,17 @@ export class CustomPlaylistsComponent implements OnInit { this.getAllPlaylists(); } }); + + this.postsService.playlists_changed.subscribe(changed => { + if (changed) { + this.getAllPlaylists(); + } + }); } getAllPlaylists() { this.playlists_received = false; + // must call getAllFiles as we need to get category playlists as well this.postsService.getAllFiles().subscribe(res => { this.playlists = res['playlists']; this.playlists_received = true; diff --git a/src/app/components/recent-videos/recent-videos.component.html b/src/app/components/recent-videos/recent-videos.component.html index ec4616c..9b00952 100644 --- a/src/app/components/recent-videos/recent-videos.component.html +++ b/src/app/components/recent-videos/recent-videos.component.html @@ -32,7 +32,7 @@
- +
No videos found. diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index 144f3bd..d795e29 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -50,6 +50,8 @@ export class RecentVideosComponent implements OnInit { } }; filterProperty = this.filterProperties['upload_date']; + + playlists = null; pageSize = 10; paged_data = null; @@ -68,14 +70,27 @@ export class RecentVideosComponent implements OnInit { ngOnInit(): void { if (this.postsService.initialized) { this.getAllFiles(); + this.getAllPlaylists(); } this.postsService.service_initialized.subscribe(init => { if (init) { this.getAllFiles(); + this.getAllPlaylists(); } }); + this.postsService.files_changed.subscribe(changed => { + if (changed) { + this.getAllFiles(); + } + }); + + this.postsService.playlists_changed.subscribe(changed => { + if (changed) { + this.getAllPlaylists(); + } + }); // set filter property to cached const cached_filter_property = localStorage.getItem('filter_property'); @@ -84,6 +99,12 @@ export class RecentVideosComponent implements OnInit { } } + getAllPlaylists() { + this.postsService.getPlaylists().subscribe(res => { + this.playlists = res['playlists']; + }); + } + // search onSearchInputChanged(newvalue) { @@ -288,6 +309,23 @@ export class RecentVideosComponent implements OnInit { this.filterByProperty(this.filterProperty['property']); } + addFileToPlaylist(info_obj) { + const file = info_obj['file']; + const playlist_id = info_obj['playlist_id']; + const playlist = this.playlists.find(potential_playlist => potential_playlist['id'] === playlist_id); + this.postsService.addFileToPlaylist(playlist_id, file['uid']).subscribe(res => { + if (res['success']) { + this.postsService.openSnackBar(`Successfully added ${file.title} to ${playlist.title}!`); + this.postsService.playlists_changed.next(true); + } else { + this.postsService.openSnackBar(`Failed to add ${file.title} to ${playlist.title}! Unknown error.`); + } + }, err => { + console.error(err); + this.postsService.openSnackBar(`Failed to add ${file.title} to ${playlist.title}! See browser console for error.`); + }); + } + // sorting and filtering sortFiles(a, b) { diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index 066d78e..287d7d4 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -23,6 +23,10 @@ + + + + - + + + +
diff --git a/src/app/settings/settings.component.ts b/src/app/settings/settings.component.ts index 9f8bc3d..d2d43e9 100644 --- a/src/app/settings/settings.component.ts +++ b/src/app/settings/settings.component.ts @@ -307,9 +307,9 @@ export class SettingsComponent implements OnInit { }); } - testConnectionString() { + testConnectionString(connection_string) { this.testing_connection_string = true; - this.postsService.testConnectionString().subscribe(res => { + this.postsService.testConnectionString(connection_string).subscribe(res => { this.testing_connection_string = false; if (res['success']) { this.postsService.openSnackBar('Connection successful!'); From c0a385ce78f0e5c4edca6acc2ed6566c5bc3701d Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 27 Jul 2021 22:36:32 -0600 Subject: [PATCH 06/11] Default file output now applies to subscriptions --- backend/subscriptions.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/subscriptions.js b/backend/subscriptions.js index cd15f40..30786bf 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -347,7 +347,9 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de let appendedBasePath = getAppendedBasePath(sub, basePath); - let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`; + const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; + + let fullOutput = `${appendedBasePath}/${file_output}.%(ext)s`; if (desired_path) { fullOutput = `${desired_path}.%(ext)s`; } else if (sub.custom_output) { From d09b244bc21e396ca7bf29b76e98d1de84e18118 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Wed, 28 Jul 2021 19:44:05 -0600 Subject: [PATCH 07/11] Fixed bug where unsubscribing from a channel would clear the entire files table Fixed issue where yt-dlp did not work with subscriptions --- backend/app.js | 24 +++++++++--------------- backend/consts.js | 5 ++++- backend/db.js | 19 ++++++++++++------- backend/subscriptions.js | 5 +++++ backend/utils.js | 7 +++++++ 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/backend/app.js b/backend/app.js index 86c0f89..5ad256f 100644 --- a/backend/app.js +++ b/backend/app.js @@ -29,8 +29,7 @@ const read_last_lines = require('read-last-lines'); var ps = require('ps-node'); // needed if bin/details somehow gets deleted -const DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details' -if (!fs.existsSync(DETAILS_BIN_PATH)) fs.writeJSONSync(DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) +if (!fs.existsSync(CONSTS.DETAILS_BIN_PATH)) fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2000.06.06","path":"node_modules\\youtube-dl\\bin\\youtube-dl.exe","exec":"youtube-dl.exe","downloader":"youtube-dl"}) var youtubedl = require('youtube-dl'); @@ -1180,7 +1179,7 @@ async function generateArgs(url, type, options) { downloadConfig = downloadConfig.concat(globalArgs.split(',,')); } - const default_downloader = getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); if (default_downloader === 'yt-dlp') { downloadConfig.push('--no-clean-infojson'); } @@ -1361,17 +1360,17 @@ async function autoUpdateYoutubeDL() { const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); const tags_url = download_sources[default_downloader]['tags_url']; // get current version - let current_app_details_exists = fs.existsSync(DETAILS_BIN_PATH); + let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); if (!current_app_details_exists) { - logger.warn(`Failed to get youtube-dl binary details at location '${DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); + logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); } - let current_app_details = JSON.parse(fs.readFileSync(DETAILS_BIN_PATH)); + let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); let current_version = current_app_details['version']; let current_downloader = current_app_details['downloader']; let stored_binary_path = current_app_details['path']; if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${DETAILS_BIN_PATH}, attempting to guess actual path...`); + // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); const guessed_base_path = 'node_modules/youtube-dl/bin/'; const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); if (fs.existsSync(guessed_file_path)) { @@ -1454,15 +1453,10 @@ async function downloadLatestYoutubeDLPBinary(new_version) { } function updateDetailsJSON(new_version, downloader) { - const details_json = fs.readJSONSync(DETAILS_BIN_PATH); + const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); if (new_version) details_json['version'] = new_version; details_json['downloader'] = downloader; - fs.writeJSONSync(DETAILS_BIN_PATH, details_json); -} - -function getCurrentDownloader() { - const details_json = fs.readJSONSync(DETAILS_BIN_PATH); - return details_json['downloader']; + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); } async function checkExistsWithTimeout(filePath, timeout) { diff --git a/backend/consts.js b/backend/consts.js index 91cfc75..36ffd7d 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -207,8 +207,11 @@ AVAILABLE_PERMISSIONS = [ 'downloads_manager' ]; +const DETAILS_BIN_PATH = 'node_modules/youtube-dl/bin/details' + module.exports = { CONFIG_ITEMS: CONFIG_ITEMS, AVAILABLE_PERMISSIONS: AVAILABLE_PERMISSIONS, - CURRENT_VERSION: 'v4.2' + CURRENT_VERSION: 'v4.2', + DETAILS_BIN_PATH: DETAILS_BIN_PATH } diff --git a/backend/db.js b/backend/db.js index c2a80f3..51393a6 100644 --- a/backend/db.js +++ b/backend/db.js @@ -256,6 +256,9 @@ function generateFileObject2(file_path, type) { var jsonobj = utils.getJSON(file_path, type); if (!jsonobj) { return null; + } else if (!jsonobj['_filename']) { + logger.error(`Failed to get filename from info JSON! File ${jsonobj['title']} could not be added.`); + return null; } const ext = (type === 'audio') ? '.mp3' : '.mp4' const true_file_path = utils.getTrueFileName(jsonobj['_filename'], type); @@ -781,26 +784,26 @@ exports.removeRecord = async (table, filter_obj) => { return !!(output['result']['ok']); } -exports.removeAllRecords = async (table = null) => { +exports.removeAllRecords = async (table = null, filter_obj = null) => { // local db override const tables_to_remove = table ? [table] : tables_list; + logger.debug(`Removing all records from: ${tables_to_remove} with filter: ${JSON.stringify(filter_obj)}`) if (using_local_db) { - logger.debug(`Removing all records from: ${tables_to_remove}`) for (let i = 0; i < tables_to_remove.length; i++) { const table_to_remove = tables_to_remove[i]; - local_db.assign({[table_to_remove]: []}).write(); - logger.debug(`Removed all records from ${table_to_remove}`); + if (filter_obj) applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write(); + else local_db.assign({[table_to_remove]: []}).write(); + logger.debug(`Successfully removed records from ${table_to_remove}`); } return true; } let success = true; - logger.debug(`Removing all records from: ${tables_to_remove}`) for (let i = 0; i < tables_to_remove.length; i++) { const table_to_remove = tables_to_remove[i]; - const output = await database.collection(table_to_remove).deleteMany({}); - logger.debug(`Removed all records from ${table_to_remove}`); + const output = await database.collection(table_to_remove).deleteMany(filter_obj ? filter_obj : {}); + logger.debug(`Successfully removed records from ${table_to_remove}`); success &= !!(output['result']['ok']); } return success; @@ -988,6 +991,8 @@ exports.transferDB = async (local_to_remote) => { config_api.setConfigItem('ytdl_use_local_db', using_local_db); + logger.debug('Transfer finished!'); + return success; } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 30786bf..537c516 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -413,6 +413,11 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de downloadConfig.push('--write-thumbnail'); } + const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader'); + if (default_downloader === 'yt-dlp') { + downloadConfig.push('--no-clean-infojson'); + } + return downloadConfig; } diff --git a/backend/utils.js b/backend/utils.js index dc425e8..9370cf4 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -1,6 +1,7 @@ const fs = require('fs-extra') const path = require('path') const config_api = require('./config'); +const CONSTS = require('./consts') const archiver = require('archiver'); const is_windows = process.platform === 'win32'; @@ -315,6 +316,11 @@ function addUIDsToCategory(category, files) { return files_that_match; } +function getCurrentDownloader() { + const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); + return details_json['downloader']; +} + async function recFindByExt(base,ext,files,result) { files = files || (await fs.readdir(base)) @@ -390,6 +396,7 @@ module.exports = { durationStringToNumber: durationStringToNumber, getMatchingCategoryFiles: getMatchingCategoryFiles, addUIDsToCategory: addUIDsToCategory, + getCurrentDownloader: getCurrentDownloader, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, wait: wait, From 5124e3b3330100e78e859310d7aec5ea037c2b98 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Thu, 29 Jul 2021 21:35:33 -0600 Subject: [PATCH 08/11] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7e1a855..79e05a2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -28,4 +28,4 @@ If applicable, add screenshots to help explain your problem. - Docker tag: (optional) **Additional context** -Add any other context about the problem here. +Add any other context about the problem here. For example, a YouTube link. From ec3ab17507c97ced0fda8377eb0a3ed726faacfc Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Sun, 1 Aug 2021 06:23:27 -0600 Subject: [PATCH 09/11] MongoDB is no longer the default in config, this will just be set through the docker-compose.yml --- backend/appdata/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/appdata/default.json b/backend/appdata/default.json index 5c2ba9c..b5d6b52 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -55,7 +55,7 @@ } }, "Database": { - "use_local_db": false, + "use_local_db": true, "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" }, "Advanced": { From c8f219d5b0420e01863f941ee19d3961b9a2aebf Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Sun, 1 Aug 2021 06:24:47 -0600 Subject: [PATCH 10/11] See previous commit, MongoDB is no longer on by default for all installs --- backend/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config.js b/backend/config.js index 4a61999..d27e8cd 100644 --- a/backend/config.js +++ b/backend/config.js @@ -232,7 +232,7 @@ DEFAULT_CONFIG = { } }, "Database": { - "use_local_db": false, + "use_local_db": true, "mongodb_connection_string": "mongodb://127.0.0.1:27017/?compressors=zlib" }, "Advanced": { From b03b4d173b590abb10108129b85c70f4f2066763 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 1 Aug 2021 14:38:34 -0600 Subject: [PATCH 11/11] Fixed issue where testing the connecting string would fail if local DB was being used Fixed issue where blacklisting video in with archiving would not work Cleaned up unused functions in app.js --- backend/app.js | 75 -------------------------------------------------- backend/db.js | 16 +++++++++-- 2 files changed, 14 insertions(+), 77 deletions(-) diff --git a/backend/app.js b/backend/app.js index 5ad256f..8074ff8 100644 --- a/backend/app.js +++ b/backend/app.js @@ -18,13 +18,10 @@ var mergeFiles = require('merge-files'); const low = require('lowdb') var ProgressBar = require('progress'); const NodeID3 = require('node-id3') -const downloader = require('youtube-dl/lib/downloader') const fetch = require('node-fetch'); var URL = require('url').URL; -const shortid = require('shortid') const url_api = require('url'); const CONSTS = require('./consts') -const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); var ps = require('ps-node'); @@ -753,67 +750,6 @@ function generateEnvVarConfigItem(key) { return {key: key, value: process['env'][key]}; } -function getFileSizeMp3(name) -{ - var jsonPath = audioFolderPath+name+".mp3.info.json"; - - if (fs.existsSync(jsonPath)) - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - else - var obj = 0; - - return obj.filesize; -} - -function getFileSizeMp4(name) -{ - var jsonPath = videoFolderPath+name+".info.json"; - var filesize = 0; - if (fs.existsSync(jsonPath)) - { - var obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - var format = obj.format.substring(0,3); - for (i = 0; i < obj.formats.length; i++) - { - if (obj.formats[i].format_id == format) - { - filesize = obj.formats[i].filesize; - } - } - } - - return filesize; -} - -function getAmountDownloadedMp3(name) -{ - var partPath = audioFolderPath+name+".mp3.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - - - -function getAmountDownloadedMp4(name) -{ - var format = getVideoFormatID(name); - var partPath = videoFolderPath+name+".f"+format+".mp4.part"; - if (fs.existsSync(partPath)) - { - const stats = fs.statSync(partPath); - const fileSizeInBytes = stats.size; - return fileSizeInBytes; - } - else - return 0; -} - function getVideoFormatID(name) { var jsonPath = videoFolderPath+name+".info.json"; @@ -1283,17 +1219,6 @@ async function cropFile(file_path, start, end, ext) { }); } -// archive helper functions - -async function writeToBlacklist(type, line) { - let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); - // adds newline to the beginning of the line - line.replace('\n', ''); - line.replace('\r', ''); - line = '\n' + line; - await fs.appendFile(blacklistPath, line); -} - // download management functions async function updateDownloads() { diff --git a/backend/db.js b/backend/db.js index 51393a6..802694d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -75,7 +75,7 @@ exports.initialize = (input_db, input_users_db, input_logger) => { } exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => { - if (using_local_db) return; + if (using_local_db && !custom_connection_string) return; const success = await exports._connectToDB(custom_connection_string); if (success) return true; @@ -1018,4 +1018,16 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => { return filtered; }); return return_val; -} \ No newline at end of file +} + +// archive helper functions + +async function writeToBlacklist(type, line) { + const archivePath = path.join(__dirname, 'appdata', 'archives'); + let blacklistPath = path.join(archivePath, (type === 'audio') ? 'blacklist_audio.txt' : 'blacklist_video.txt'); + // adds newline to the beginning of the line + line.replace('\n', ''); + line.replace('\r', ''); + line = '\n' + line; + await fs.appendFile(blacklistPath, line); +}