Fixed issue where video info could not be retrieved

If youtube-dl update fails, error will show and server won't crash

Dockerfile

@@ -19,6 +19,7 @@ ENV USER=youtube
 ENV NO_UPDATE_NOTIFIER=true
 ENV PM2_HOME=/app/pm2
 ENV ALLOW_CONFIG_MUTATIONS=true
+ENV npm_config_cache=/app/.npm
 
 # Use NVM to get specific node version
 ENV NODE_VERSION=16.14.2

backend/app.js

@@ -1,4 +1,4 @@
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 const fs = require('fs-extra');
 const { promisify } = require('util');
 const auth_api = require('./authentication/auth');
@@ -534,13 +534,7 @@ async function loadConfig() {
         subscriptions.forEach(async sub => subscriptions_api.writeSubscriptionMetadata(sub));
         subscriptions_api.updateSubscriptionPropertyMultiple(subscriptions, {downloading: false, child_process: null});
         // runs initially, then runs every ${subscriptionCheckInterval} seconds
-        const watchSubscriptionsInterval = function() {
-            watchSubscriptions();
-            const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
-            setTimeout(watchSubscriptionsInterval, subscriptionsCheckInterval*1000);
-        }
-
-        watchSubscriptionsInterval();
+        subscriptions_api.watchSubscriptionsInterval();
     }
 
     // start the server here
@@ -570,62 +564,6 @@ function loadConfigValues() {
     utils.updateLoggerLevel(logger_level);
 }
 
-function calculateSubcriptionRetrievalDelay(subscriptions_amount) {
-    // frequency is once every 5 mins by default
-    const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
-    let interval_in_ms = subscriptionsCheckInterval * 1000;
-    const subinterval_in_ms = interval_in_ms/subscriptions_amount;
-    return subinterval_in_ms;
-}
-
-async function watchSubscriptions() {
-    let subscriptions = await subscriptions_api.getAllSubscriptions();
-
-    if (!subscriptions) return;
-
-    // auto pause deprecated streamingOnly mode
-    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
-    subscriptions_api.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
-
-    const valid_subscriptions = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly);
-
-    let subscriptions_amount = valid_subscriptions.length;
-    let delay_interval = calculateSubcriptionRetrievalDelay(subscriptions_amount);
-
-    let current_delay = 0;
-
-    const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
-    for (let i = 0; i < valid_subscriptions.length; i++) {
-        let sub = valid_subscriptions[i];
-
-        // don't check the sub if the last check for the same subscription has not completed
-        if (subscription_timeouts[sub.id]) {
-            logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`);
-            continue;
-        }
-
-        if (!sub.name) {
-            logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
-            continue;
-        }
-
-        logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval);
-        setTimeout(async () => {
-            const multiUserModeChanged = config_api.getConfigItem('ytdl_multi_user_mode') !== multiUserMode;
-            if (multiUserModeChanged) {
-                logger.verbose(`Skipping subscription ${sub.name} due to multi-user mode change.`);
-                return;
-            }
-            await subscriptions_api.getVideosForSub(sub.id);
-            subscription_timeouts[sub.id] = false;
-        }, current_delay);
-        subscription_timeouts[sub.id] = true;
-        current_delay += delay_interval;
-        const subscriptionsCheckInterval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
-        if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0;
-    }
-}
-
 function getOrigin() {
     return url_domain.origin;
 }
@@ -651,17 +589,6 @@ function generateEnvVarConfigItem(key) {
     return {key: key, value: process['env'][key]};
 }
 
-// currently only works for single urls
-async function getUrlInfos(url) {
-    const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, ['--dump-json']);
-    if (!parsed_output || parsed_output.length !== 1) {
-        logger.error(`Failed to retrieve available formats for url: ${url}`);
-        if (err) logger.error(err);
-        return null;
-    }
-    return parsed_output[0];
-}
-
 // youtube-dl functions
 
 async function startYoutubeDL() {
@@ -1651,6 +1578,7 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
     }
     if (!fs.existsSync(file_path)) {
         logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
+        return;
     }
     const stat = fs.statSync(file_path);
     const fileSize = stat.size;
@@ -1931,11 +1859,11 @@ app.post('/api/clearAllLogs', optionalJwt, async function(req, res) {
 });
 
   app.post('/api/getFileFormats', optionalJwt, async (req, res) => {
-    let url = req.body.url;
-    let result = await getUrlInfos(url);
+    const url = req.body.url;
+    const result = await downloader_api.getVideoInfoByURL(url);
     res.send({
-        result: result,
-        success: !!result
+        result: result && result.length === 1 ? result[0] : null,
+        success: result && result.length === 0
     })
 });
 

backend/archive.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const fs = require('fs-extra');
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 
 const db_api = require('./db');
 

backend/authentication/auth.js

@@ -4,7 +4,7 @@ const logger = require('../logger');
 const db_api = require('../db');
 
 const jwt = require('jsonwebtoken');
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 const bcrypt = require('bcryptjs');
 const fs = require('fs-extra');
 const path = require('path');

backend/db.js

@@ -1,7 +1,6 @@
 const fs = require('fs-extra')
 const path = require('path')
 const { MongoClient } = require("mongodb");
-const { uuid } = require('uuidv4');
 const _ = require('lodash');
 
 const config_api = require('./config');

backend/downloader.js

@@ -1,5 +1,5 @@
 const fs = require('fs-extra');
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 const path = require('path');
 const NodeID3 = require('node-id3')
 const Mutex = require('async-mutex').Mutex;

backend/files.js

@@ -1,6 +1,6 @@
 const fs = require('fs-extra')
 const path = require('path')
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 
 const config_api = require('./config');
 const db_api = require('./db');

backend/notifications.js

@@ -4,7 +4,7 @@ const logger = require('./logger');
 const utils = require('./utils');
 const consts = require('./consts');
 
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 
 const fetch = require('node-fetch');
 const { gotify } = require("gotify");

backend/package.json

@@ -64,7 +64,7 @@
     "shortid": "^2.2.15",
     "tree-kill": "^1.2.2",
     "unzipper": "^0.10.10",
-    "uuidv4": "^6.2.13",
+    "uuid": "^9.0.1",
     "winston": "^3.7.2",
     "xmlbuilder2": "^3.0.2"
   }

backend/subscriptions.js

@@ -205,6 +205,64 @@ exports.deleteSubscriptionFile = async (sub, file, deleteForever, file_uid = nul
     }
 }
 
+let current_sub_index = 0; // To keep track of the current subscription
+exports.watchSubscriptionsInterval = async () => {
+    const subscriptions_check_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+    let parent_interval = setInterval(() => watchSubscriptions(), subscriptions_check_interval*1000);
+    watchSubscriptions();
+    config_api.config_updated.subscribe(change => {
+        if (!change) return;
+        if (change['key'] === 'ytdl_subscriptions_check_interval' || change['key'] === 'ytdl_multi_user_mode') {
+            current_sub_index = 0; // TODO: start after the last sub check
+            logger.verbose('Resetting sub check schedule due to config change');
+            clearInterval(parent_interval);
+            const new_interval = config_api.getConfigItem('ytdl_subscriptions_check_interval');
+            parent_interval = setInterval(() => watchSubscriptions(), new_interval*1000);
+            watchSubscriptions();
+        }
+    });
+}
+
+async function watchSubscriptions() {
+    const subscription_ids = await getValidSubscriptionsToCheck();
+    if (subscription_ids.length === 0) {
+        logger.info('Skipping subscription check as no valid subscriptions exist.');
+        return;
+    }
+    checkSubscription(subscription_ids[current_sub_index]);
+    current_sub_index = (current_sub_index + 1) % subscription_ids.length;
+}
+
+async function checkSubscription(sub_id) {
+    let sub = await exports.getSubscription(sub_id);
+
+    // don't check the sub if the last check for the same subscription has not completed
+    if (sub.downloading) {
+        logger.verbose(`Subscription: skipped checking ${sub.name} as it's downloading videos.`);
+        return;
+    }
+
+    if (!sub.name) {
+        logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`);
+        return;
+    }
+
+    await exports.getVideosForSub(sub.id);
+}
+
+async function getValidSubscriptionsToCheck() {
+    const subscriptions = await exports.getAllSubscriptions();
+
+    if (!subscriptions) return;
+
+    // auto pause deprecated streamingOnly mode
+    const streaming_only_subs = subscriptions.filter(sub => sub.streamingOnly);
+    exports.updateSubscriptionPropertyMultiple(streaming_only_subs, {paused: true});
+
+    const valid_subscription_ids = subscriptions.filter(sub => !sub.paused && !sub.streamingOnly).map(sub => sub.id);
+    return valid_subscription_ids;
+}
+
 exports.getVideosForSub = async (sub_id) => {
     const sub = await exports.getSubscription(sub_id);
     if (!sub || sub['downloading']) {

backend/tasks.js

@@ -13,7 +13,6 @@ const CONSTS = require('./consts');
 const fs = require('fs-extra');
 const path = require('path');
 const scheduler = require('node-schedule');
-const { uuid } = require('uuidv4');
 
 const TASKS = {
     backup_local_db: {

backend/test/tests.js

@@ -5,7 +5,7 @@ const winston = require('winston');
 const path = require('path');
 const util = require('util');
 const fs = require('fs-extra');
-const { uuid } = require('uuidv4');
+const { v4: uuid } = require('uuid');
 const NodeID3 = require('node-id3');
 const exec = util.promisify(require('child_process').exec);
 

backend/youtube-dl.js

@@ -88,13 +88,13 @@ exports.checkForYoutubeDLUpdate = async () => {
     const output_file_path = getYoutubeDLPath();
     // get current version
     let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
-    if (!current_app_details_exists) {
+    if (!current_app_details_exists[selected_fork]) {
         logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
         updateDetailsJSON(CONSTS.OUTDATED_YOUTUBEDL_VERSION, selected_fork, output_file_path);
     }
     const current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
-    const current_version = current_app_details['version'];
-    const current_fork = current_app_details['downloader'];
+    const current_version = current_app_details[selected_fork]['version'];
+    const current_fork = current_app_details[selected_fork]['downloader'];
 
     const latest_version = await exports.getLatestUpdateVersion(selected_fork);
     // if the binary does not exist, or default_downloader doesn't match existing fork, or if the fork has been updated, redownload
@@ -118,10 +118,16 @@ async function downloadLatestYoutubeDLBinaryGeneric(youtubedl_fork, new_version,
     const download_url = `${exports.youtubedl_forks[youtubedl_fork]['download_url']}${file_ext}`;
     const output_path = custom_output_path || getYoutubeDLPath(youtubedl_fork);
 
-    await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`);
-    fs.chmod(output_path, 0o777);
+    try {
+        await utils.fetchFile(download_url, output_path, `${youtubedl_fork} ${new_version}`);
+        fs.chmod(output_path, 0o777);
 
-    updateDetailsJSON(new_version, youtubedl_fork, output_path);
+        updateDetailsJSON(new_version, youtubedl_fork, output_path);
+    } catch (e) {
+        logger.error(`Failed to download new ${youtubedl_fork} version: ${new_version}`);
+        logger.error(e);
+        return;
+    }
 } 
 
 exports.getLatestUpdateVersion = async (youtubedl_fork) => {

chart/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: youtubedl-material
-description: A Helm chart for Kubernetes
+description: A Helm chart for https://github.com/Tzahi12345/YoutubeDL-Material
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

chart/templates/ingress.yaml

@@ -1,7 +1,14 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "youtubedl-material.fullname" . -}}
 {{- $svcPort := .Values.service.port -}}
-{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1beta1
 {{- else -}}
 apiVersion: extensions/v1beta1
@@ -16,6 +23,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
   {{- if .Values.ingress.tls }}
   tls:
     {{- range .Values.ingress.tls }}
@@ -33,9 +43,19 @@ spec:
         paths:
           {{- range .paths }}
           - path: {{ .path }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            pathType: {{ .pathType }}
+            {{- end }}
             backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ $fullName }}
+                port:
+                  number: {{ $svcPort }}
+              {{- else }}
               serviceName: {{ $fullName }}
               servicePort: {{ $svcPort }}
+              {{- end }}
           {{- end }}
     {{- end }}
-  {{- end }}
+{{- end }}

package.json

@@ -78,5 +78,11 @@
     "protractor": "~7.0.0",
     "ts-node": "~3.0.4",
     "tslint": "~6.1.0"
-  }
+  },
+  "overrides": {
+    "ngx-avatars": {
+      "@angular/common": "15.0.1",
+      "@angular/core": "15.0.1"
+    }
+}
 }

src/app/main/main.component.ts

@@ -548,7 +548,7 @@ export class MainComponent implements OnInit {
     }
     if (!(this.cachedAvailableFormats[url] && this.cachedAvailableFormats[url]['formats'])) {
       this.cachedAvailableFormats[url]['formats_loading'] = true;
-      this.postsService.getFileFormats([url]).subscribe(res => {
+      this.postsService.getFileFormats(url).subscribe(res => {
         this.cachedAvailableFormats[url]['formats_loading'] = false;
         const infos = res['result'];
         if (!infos || !infos.formats) {

src/app/posts.services.ts

@@ -459,7 +459,7 @@ export class PostsService implements CanActivate {
         return this.http.post<SuccessObject>(this.path + 'deleteArchiveItems', body, this.httpOptions);
     }
 
-    getFileFormats(url) {
+    getFileFormats(url: string) {
         const body: GetFileFormatsRequest = {url: url};
         return this.http.post<GetFileFormatsResponse>(this.path + 'getFileFormats', body, this.httpOptions);
     }