From 1f9f07ac562cf4c146ea91a2bb6417479e860029 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Tue, 17 Mar 2020 23:22:26 -0400 Subject: [PATCH 1/2] youtube-dl now auto updates on server restart --- backend/app.js | 50 ++++++++++++++++++++++++++++++++++++++++++++ backend/package.json | 1 + 2 files changed, 51 insertions(+) diff --git a/backend/app.js b/backend/app.js index a02a6cd..dfc0a90 100644 --- a/backend/app.js +++ b/backend/app.js @@ -10,6 +10,8 @@ var bodyParser = require("body-parser"); var archiver = require('archiver'); var mergeFiles = require('merge-files'); const low = require('lowdb') +const downloader = require('youtube-dl/lib/downloader') +const fetch = require('node-fetch'); var URL = require('url').URL; const shortid = require('shortid') const url_api = require('url'); @@ -56,6 +58,9 @@ let debugMode = process.env.YTDL_MODE === 'debug'; if (debugMode) console.log('YTDL-Material in debug mode!'); +// updates & starts youtubedl +startYoutubeDL(); + var validDownloadingAgents = [ 'aria2c', 'avconv', @@ -613,6 +618,51 @@ function writeToBlacklist(type, line) { fs.appendFileSync(blacklistBasePath + 'blacklist.txt', line); } +async function startYoutubeDL() { + // auto update youtube-dl + await autoUpdateYoutubeDL(); +} + +// auto updates the underlying youtube-dl binary, not YoutubeDL-Material +async function autoUpdateYoutubeDL() { + return new Promise(resolve => { + // get current version + let current_app_details_path = 'node_modules/youtube-dl/bin/details'; + let current_app_details_exists = fs.existsSync(current_app_details_path); + if (!current_app_details_exists) { + console.log(`Failed to get youtube-dl binary details at location: ${current_app_details_path}. Cancelling update check.`); + resolve(false); + return; + } + let current_app_details = JSON.parse(fs.readFileSync(current_app_details_path)); + let current_version = current_app_details['version']; + + // got version, now let's check the latest version from the youtube-dl API + let youtubedl_api_path = 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'; + fetch(youtubedl_api_path, {method: 'Get'}) + .then(res => res.json()) + .then((json) => { + // check if the versions are different + const latest_update_version = json[0]['name']; + if (current_version !== latest_update_version) { + let binary_path = 'node_modules/youtube-dl/bin'; + // versions different, download new update + console.log('INFO: Found new update for youtube-dl. Updating binary...'); + downloader(binary_path, function error(err, done) { + 'use strict' + if (err) { + resolve(false); + throw err; + } + console.log(`INFO: Binary successfully updated: ${current_version} -> ${latest_update_version}`); + resolve(true); + }); + } + + }); + }); +} + app.use(function(req, res, next) { res.header("Access-Control-Allow-Origin", getOrigin()); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); diff --git a/backend/package.json b/backend/package.json index b67c591..8992bc6 100644 --- a/backend/package.json +++ b/backend/package.json @@ -26,6 +26,7 @@ "express": "^4.17.1", "lowdb": "^1.0.0", "merge-files": "^0.1.2", + "node-fetch": "^2.6.0", "shortid": "^2.2.15", "uuidv4": "^6.0.6", "youtube-dl": "^3.0.2" From 2e71a0bef112f3ccbd89872d891c0785acddbba9 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Wed, 18 Mar 2020 00:04:18 -0400 Subject: [PATCH 2/2] fixed bug that caused youtube downloader update to fail when the binary was being used to check for new subscription videos. now it waits for file access with a 10 second timeout --- backend/app.js | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/backend/app.js b/backend/app.js index dfc0a90..76cf30a 100644 --- a/backend/app.js +++ b/backend/app.js @@ -636,18 +636,20 @@ async function autoUpdateYoutubeDL() { } let current_app_details = JSON.parse(fs.readFileSync(current_app_details_path)); let current_version = current_app_details['version']; + let stored_binary_path = current_app_details['path']; // got version, now let's check the latest version from the youtube-dl API let youtubedl_api_path = 'https://api.github.com/repos/ytdl-org/youtube-dl/tags'; fetch(youtubedl_api_path, {method: 'Get'}) - .then(res => res.json()) - .then((json) => { + .then(async res => res.json()) + .then(async (json) => { // check if the versions are different const latest_update_version = json[0]['name']; if (current_version !== latest_update_version) { let binary_path = 'node_modules/youtube-dl/bin'; // versions different, download new update console.log('INFO: Found new update for youtube-dl. Updating binary...'); + await checkExistsWithTimeout(stored_binary_path, 10000); downloader(binary_path, function error(err, done) { 'use strict' if (err) { @@ -663,6 +665,34 @@ async function autoUpdateYoutubeDL() { }); } +async function checkExistsWithTimeout(filePath, timeout) { + return new Promise(function (resolve, reject) { + + var timer = setTimeout(function () { + watcher.close(); + reject(new Error('File did not exists and was not created during the timeout.')); + }, timeout); + + fs.access(filePath, fs.constants.R_OK, function (err) { + if (!err) { + clearTimeout(timer); + watcher.close(); + resolve(); + } + }); + + var dir = path.dirname(filePath); + var basename = path.basename(filePath); + var watcher = fs.watch(dir, function (eventType, filename) { + if (eventType === 'rename' && filename === basename) { + clearTimeout(timer); + watcher.close(); + resolve(); + } + }); + }); +} + app.use(function(req, res, next) { res.header("Access-Control-Allow-Origin", getOrigin()); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");