From 9f5b5845931c4b0b5807158d8a56a037a862a729 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:09:57 +0000 Subject: [PATCH 1/7] Bump actions/setup-node from 2 to 3 in /.github/workflows Bumps [actions/setup-node](https://github.com/actions/setup-node) from 2 to 3. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50f4eb5..9b3b2ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: - name: checkout code uses: actions/checkout@v2 - name: setup node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: '12' cache: 'npm' From 6579f2b59ec90719c882cde1115f7bfc4359ca31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:10:00 +0000 Subject: [PATCH 2/7] Bump docker/login-action from 1 to 2 in /.github/workflows Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-release.yml | 2 +- .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 6ace410..1a0e3b1 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -63,7 +63,7 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cd0373f..1df2c5d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -63,7 +63,7 @@ jobs: type=sha,prefix=sha-,format=short - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From c64140b60546e0f13706422a55a16800581243e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:10:04 +0000 Subject: [PATCH 3/7] Bump actions/upload-artifact from 1 to 3 in /.github/workflows Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 1 to 3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v1...v3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50f4eb5..3787e89 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: Copy-Item -Path ./backend/*.js -Destination ./build/youtubedl-material Copy-Item -Path ./backend/*.json -Destination ./build/youtubedl-material - name: upload build artifact - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: youtubedl-material path: build From c5d1b3ffcf5fcf4f912476d385269e46dc2c2567 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:10:06 +0000 Subject: [PATCH 4/7] Bump docker/setup-buildx-action from 1 to 2 in /.github/workflows Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker-release.yml | 2 +- .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 6ace410..1fd020e 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -60,7 +60,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: setup multi-arch docker build - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cd0373f..b01ecce 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -44,7 +44,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: setup multi-arch docker build - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Generate Docker image metadata id: docker-meta From 2a1af69f1f48aa4450b1fb556d443f9da8feaee6 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 27 May 2023 00:57:27 -0400 Subject: [PATCH 5/7] Removed additional download button for paused download --- src/app/components/downloads/downloads.component.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index f99c4ca..d5e3092 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -78,12 +78,6 @@ export class DownloadsComponent implements OnInit, OnDestroy { show: (download: Download) => !download.finished && download.paused && download.finished_step, icon: 'play_arrow' }, - { - tooltip: $localize`Resume`, - action: (download: Download) => this.resumeDownload(download), - show: (download: Download) => !download.finished && download.paused && download.finished_step, - icon: 'play_arrow' - }, { tooltip: $localize`Cancel`, action: (download: Download) => this.cancelDownload(download), From 8212acbac68889e0e3b18f9f0d6146709c3ea413 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 27 May 2023 01:39:40 -0400 Subject: [PATCH 6/7] Combined output json processing from youtube-dl between subscriptions and one-off downloads Added workaround for missing video in playlist to one-off downloads --- backend/downloader.js | 49 +++++++--------------------------------- backend/subscriptions.js | 47 +++++++++----------------------------- backend/utils.js | 31 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 77 deletions(-) diff --git a/backend/downloader.js b/backend/downloader.js index 8658502..1c57534 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -314,13 +314,14 @@ async function downloadQueuedFile(download_uid) { let difference = (end_time - start_time)/1000; logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); clearInterval(download_checker); - if (err) { + const parsed_output = utils.parseOutputJSON(output, err); + if (!parsed_output) { logger.error(err.stderr); await handleDownloadError(download, err.stderr, 'unknown_error'); resolve(false); return; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { + } else if (parsed_output) { + if (parsed_output.length === 0 || parsed_output[0].length === 0) { // ERROR! const error_message = `No output received for video download, check if it exists in your archive.`; await handleDownloadError(download, error_message, 'no_output'); @@ -329,17 +330,7 @@ async function downloadQueuedFile(download_uid) { return; } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - // we have to do this because sometimes there will be leading characters before the actual json - const start_idx = output[i].indexOf('{"'); - const clean_output = output[i].slice(start_idx, output[i].length); - output_json = JSON.parse(clean_output); - } catch(e) { - output_json = null; - } - + for (const output_json of parsed_output) { if (!output_json) { continue; } @@ -564,33 +555,9 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { new_args.push('--dump-json'); youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => { - if (output) { - let outputs = []; - try { - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - - if (!output_json) { - continue; - } - - outputs.push(output_json); - } - resolve(outputs.length === 1 ? outputs[0] : outputs); - } catch(e) { - const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`; - logger.error(error); - if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error, 'parse_failed'); - } - resolve(null); - } + const parsed_output = utils.parseOutputJSON(output, err); + if (parsed_output) { + resolve(parsed_output); } else { let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; if (err.stderr) error_message += `\n\n${err.stderr}`; diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 03c07b4..c1bbefe 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -254,26 +254,15 @@ exports.getVideosForSub = async (sub, user_uid = null) => { } logger.verbose('Subscription: finished check for ' + sub.name); - if (err && !output) { - logger.error(err.stderr ? err.stderr : err.message); - if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) { - logger.info('An error was encountered with at least one video, backup method will be used.') - try { - const outputs = err.stdout.split(/\r\n|\r|\n/); - const files_to_download = await handleOutputJSON(outputs, sub, user_uid); - resolve(files_to_download); - } catch(e) { - logger.error('Backup method failed. See error below:'); - logger.error(e); - } - } else { - logger.error('Subscription check failed!'); - } - resolve(false); - } else if (output) { - const files_to_download = await handleOutputJSON(output, sub, user_uid); - resolve(files_to_download); - } + const processed_output = utils.parseOutputJSON(output, err); + if (!processed_output) { + logger.error('Subscription check failed!'); + resolve(null); + return; + } + const files_to_download = await handleOutputJSON(processed_output, sub, user_uid); + resolve(files_to_download); + return; }); }, err => { logger.error(err); @@ -281,31 +270,17 @@ exports.getVideosForSub = async (sub, user_uid = null) => { }); } -async function handleOutputJSON(output, sub, user_uid) { +async function handleOutputJSON(output_jsons, sub, user_uid) { if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { await setFreshUploads(sub, user_uid); checkVideosForFreshUploads(sub, user_uid); } - if (output.length === 0 || (output.length === 1 && output[0] === '')) { + if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) { logger.verbose('No additional videos to download for ' + sub.name); return []; } - const output_jsons = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - output_jsons.push(output_json); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - } - const files_to_download = await getFilesToDownload(sub, output_jsons); const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid); diff --git a/backend/utils.js b/backend/utils.js index 157b6f0..8c6a1ae 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -530,6 +530,37 @@ exports.getDirectoriesInDirectory = async (basePath) => { } } +exports.parseOutputJSON = (output, err) => { + let split_output = []; + // const output_jsons = []; + if (err && !output) { + if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) { + return null; + } + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + split_output = err.stdout.split(/\r\n|\r|\n/); + } catch (e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + return null; + } + } else { + for (const output_item of output) { + // we have to do this because sometimes there will be leading characters before the actual json + const start_idx = output_item.indexOf('{"'); + const clean_output = output_item.slice(start_idx, output_item.length); + split_output.push(clean_output); + } + } + + try { + return split_output.map(split_output_str => JSON.parse(split_output_str)); + } catch(e) { + return null; + } +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { From f7d38351110481084086aa53b0f3e29e3334b418 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 27 May 2023 01:49:32 -0400 Subject: [PATCH 7/7] Fixed issue of case where output is empty --- backend/subscriptions.js | 6 +++--- backend/utils.js | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/subscriptions.js b/backend/subscriptions.js index c1bbefe..2d212ed 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -254,13 +254,13 @@ exports.getVideosForSub = async (sub, user_uid = null) => { } logger.verbose('Subscription: finished check for ' + sub.name); - const processed_output = utils.parseOutputJSON(output, err); - if (!processed_output) { + const parsed_output = utils.parseOutputJSON(output, err); + if (!parsed_output) { logger.error('Subscription check failed!'); resolve(null); return; } - const files_to_download = await handleOutputJSON(processed_output, sub, user_uid); + const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid); resolve(files_to_download); return; }); diff --git a/backend/utils.js b/backend/utils.js index 8c6a1ae..837a063 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -545,7 +545,10 @@ exports.parseOutputJSON = (output, err) => { logger.error(e); return null; } - } else { + } else if (output.length === 0 || (output.length === 1 && output[0].length === 0)) { + // output is '' or [''] + return []; + } else { for (const output_item of output) { // we have to do this because sometimes there will be leading characters before the actual json const start_idx = output_item.indexOf('{"');