Merge pull request #1031 from Tzahi12345/bot-requests

Add ability to request downloads using telegram (new)

backend/app.js

@@ -30,6 +30,7 @@ const twitch_api = require('./twitch');
 const youtubedl_api = require('./youtube-dl');
 const archive_api = require('./archive');
 const files_api = require('./files');
+const notifications_api = require('./notifications');
 
 var app = express();
 
@@ -685,7 +686,7 @@ app.use(function(req, res, next) {
         next();
     } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
         next();
-    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
+    } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss') || req.path.includes('/api/telegramRequest')) {
         next();
     } else {
         logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -1784,6 +1785,10 @@ app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
 app.post('/api/getTasks', optionalJwt, async (req, res) => {
     const tasks = await db_api.getRecords('tasks');
     for (let task of tasks) {
+        if (!tasks_api.TASKS[task['key']]) {
+            logger.verbose(`Task ${task['key']} does not exist!`);
+            continue;
+        }
         if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime();
     }
     res.send({tasks: tasks});
@@ -2092,6 +2097,25 @@ app.post('/api/deleteAllNotifications', optionalJwt, async (req, res) => {
     res.send({success: success});
 });
 
+app.post('/api/telegramRequest', async (req, res) => {
+    if (!req.body.message  && !req.body.message.text) {
+        logger.error('Invalid Telegram request received!');
+        res.sendStatus(400);
+        return;
+    }
+    const text = req.body.message.text;
+    const regex_exp = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)?/gi;
+    const url_regex = new RegExp(regex_exp);
+    if (text.match(url_regex)) {
+        downloader_api.createDownload(text, 'video', {}, req.query.user_uid ? req.query.user_uid : null);
+        res.sendStatus(200);
+    } else {
+        logger.error('Invalid Telegram request received! Make sure you only send a valid URL.');
+        notifications_api.sendTelegramNotification({title: 'Invalid Telegram Request', body: 'Make sure you only send a valid URL.', url: text});
+        res.sendStatus(400);
+    }
+});
+
 // rss feed
 
 app.get('/api/rss', async function (req, res) {

backend/appdata/default.json

@@ -49,6 +49,7 @@
       "use_telegram_API": false,
       "telegram_bot_token": "",
       "telegram_chat_id": "",
+      "telegram_webhook_proxy": "",
       "webhook_URL": "",
       "discord_webhook_URL": "",
       "slack_webhook_URL": ""

backend/config.js

@@ -1,22 +1,26 @@
 const logger = require('./logger');
 
 const fs = require('fs');
+const { BehaviorSubject } = require('rxjs');
+
+exports.CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
+exports.descriptors = {}; // to get rid of file locks when needed, TODO: move to youtube-dl.js
 
-let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS'];
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 let configPath = debugMode ? '../src/assets/default.json' : 'appdata/default.json';
+exports.config_updated = new BehaviorSubject();
 
-function initialize() {
+exports.initialize = () => {
     ensureConfigFileExists();
     ensureConfigItemsExist();
 }
 
 function ensureConfigItemsExist() {
-    const config_keys = Object.keys(CONFIG_ITEMS);
+    const config_keys = Object.keys(exports.CONFIG_ITEMS);
     for (let i = 0; i < config_keys.length; i++) {
         const config_key = config_keys[i];
-        getConfigItem(config_key);
+        exports.getConfigItem(config_key);
     }
 }
 
@@ -57,17 +61,17 @@ function getElementNameInConfig(path) {
 /**
  * Check if config exists. If not, write default config to config path
  */
-function configExistsCheck() {
+exports.configExistsCheck = () => {
     let exists = fs.existsSync(configPath);
     if (!exists) {
-        setConfigFile(DEFAULT_CONFIG);
+        exports.setConfigFile(DEFAULT_CONFIG);
     }
 }
 
 /*
 * Gets config file and returns as a json
 */
-function getConfigFile() {
+exports.getConfigFile = () => {
     try {
         let raw_data = fs.readFileSync(configPath);
         let parsed_data = JSON.parse(raw_data);
@@ -78,35 +82,40 @@ function getConfigFile() {
     }
 }
 
-function setConfigFile(config) {
+exports.setConfigFile = (config) => {
     try {
+        const old_config = exports.getConfigFile();
         fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+        const changes = exports.findChangedConfigItems(old_config, config);
+        if (changes.length > 0) {
+            for (const change of changes) exports.config_updated.next(change);
+        }
         return true;
     } catch(e) {
         return false;
     }
 }
 
-function getConfigItem(key) {
+exports.getConfigItem = (key) => {
-    let config_json = getConfigFile();
+    let config_json = exports.getConfigFile();
-    if (!CONFIG_ITEMS[key]) {
+    if (!exports.CONFIG_ITEMS[key]) {
         logger.error(`Config item with key '${key}' is not recognized.`);
         return null;
     }
-    let path = CONFIG_ITEMS[key]['path'];
+    let path = exports.CONFIG_ITEMS[key]['path'];
     const val = Object.byString(config_json, path);
     if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
         logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
-        setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
+        exports.setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
         return Object.byString(DEFAULT_CONFIG, path);
     }
     return Object.byString(config_json, path);
 }
 
-function setConfigItem(key, value) {
+exports.setConfigItem = (key, value) => {
     let success = false;
-    let config_json = getConfigFile();
+    let config_json = exports.getConfigFile();
-    let path = CONFIG_ITEMS[key]['path'];
+    let path = exports.CONFIG_ITEMS[key]['path'];
     let element_name = getElementNameInConfig(path);
     let parent_path = getParentPath(path);
     let parent_object = Object.byString(config_json, parent_path);
@@ -118,20 +127,18 @@ function setConfigItem(key, value) {
         parent_parent_object[parent_parent_single_key] = {};
         parent_object = Object.byString(config_json, parent_path);
     }
+    if (value === 'false') value = false;
+    if (value === 'true') value = true;
+    parent_object[element_name] = value;
 
-    if (value === 'false' || value === 'true') {
+    success = exports.setConfigFile(config_json);
-        parent_object[element_name] = (value === 'true');
-    } else {
-        parent_object[element_name] = value;
-    }
-    success = setConfigFile(config_json);
 
     return success;
 }
 
-function setConfigItems(items) {
+exports.setConfigItems = (items) => {
     let success = false;
-    let config_json = getConfigFile();
+    let config_json = exports.getConfigFile();
     for (let i = 0; i < items.length; i++) {
         let key = items[i].key;
         let value = items[i].value;
@@ -141,7 +148,7 @@ function setConfigItems(items) {
             value = (value === 'true');
         }
 
-        let item_path = CONFIG_ITEMS[key]['path'];
+        let item_path = exports.CONFIG_ITEMS[key]['path'];
         let item_parent_path = getParentPath(item_path);
         let item_element_name = getElementNameInConfig(item_path);
 
@@ -149,28 +156,41 @@ function setConfigItems(items) {
         item_parent_object[item_element_name] = value;
     }
 
-    success = setConfigFile(config_json);
+    success = exports.setConfigFile(config_json);
     return success;
 }
 
-function globalArgsRequiresSafeDownload() {
+exports.globalArgsRequiresSafeDownload = () => {
-    const globalArgs = getConfigItem('ytdl_custom_args').split(',,');
+    const globalArgs = exports.getConfigItem('ytdl_custom_args').split(',,');
     const argsThatRequireSafeDownload = ['--write-sub', '--write-srt', '--proxy'];
     const failedArgs = globalArgs.filter(arg => argsThatRequireSafeDownload.includes(arg));
     return failedArgs && failedArgs.length > 0;
 }
 
-module.exports = {
+exports.findChangedConfigItems = (old_config, new_config, path = '', changedConfigItems = [], depth = 0) => {
-    getConfigItem: getConfigItem,
+    if (typeof old_config === 'object' && typeof new_config === 'object' && depth < 3) {
-    setConfigItem: setConfigItem,
+        for (const key in old_config) {
-    setConfigItems: setConfigItems,
+            if (Object.prototype.hasOwnProperty.call(new_config, key)) {
-    getConfigFile: getConfigFile,
+                exports.findChangedConfigItems(old_config[key], new_config[key], `${path}${path ? '.' : ''}${key}`, changedConfigItems, depth + 1);
-    setConfigFile: setConfigFile,
+            }
-    configExistsCheck: configExistsCheck,
+        }
-    CONFIG_ITEMS: CONFIG_ITEMS,
+    } else {
-    initialize: initialize,
+        if (JSON.stringify(old_config) !== JSON.stringify(new_config)) {
-    descriptors: {},
+            const key = getConfigItemKeyByPath(path);
-    globalArgsRequiresSafeDownload: globalArgsRequiresSafeDownload
+            changedConfigItems.push({
+                key: key ? key : path.split('.')[path.split('.').length - 1], // return key in CONFIG_ITEMS or the object key
+                old_value: JSON.parse(JSON.stringify(old_config)),
+                new_value: JSON.parse(JSON.stringify(new_config))
+            });
+        }
+    }
+    return changedConfigItems;
+}
+
+function getConfigItemKeyByPath(path) {
+    const found_item = Object.values(exports.CONFIG_ITEMS).find(item => item.path === path);
+    if (found_item) return found_item['key'];
+    else return null;
 }
 
 const DEFAULT_CONFIG = {
@@ -219,6 +239,7 @@ const DEFAULT_CONFIG = {
         "use_telegram_API": false,
         "telegram_bot_token": "",
         "telegram_chat_id": "",
+        "telegram_webhook_proxy": "",
         "webhook_URL": "",
         "discord_webhook_URL": "",
         "slack_webhook_URL": "",

backend/consts.js

@@ -154,6 +154,10 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_telegram_chat_id',
         'path': 'YoutubeDLMaterial.API.telegram_chat_id'
     },
+    'ytdl_telegram_webhook_proxy': {
+        'key': 'ytdl_telegram_webhook_proxy',
+        'path': 'YoutubeDLMaterial.API.telegram_webhook_proxy'
+    },
     'ytdl_webhook_url': {
         'key': 'ytdl_webhook_url',
         'path': 'YoutubeDLMaterial.API.webhook_URL'

backend/notifications.js

@@ -8,7 +8,8 @@ const { uuid } = require('uuidv4');
 
 const fetch = require('node-fetch');
 const { gotify } = require("gotify");
-const TelegramBot = require('node-telegram-bot-api');
+const TelegramBotAPI = require('node-telegram-bot-api');
+let telegram_bot = null;
 const REST = require('@discordjs/rest').REST;
 const API = require('@discordjs/core').API;
 const EmbedBuilder = require('@discordjs/builders').EmbedBuilder;
@@ -56,7 +57,7 @@ exports.sendNotification = async (notification) => {
         sendGotifyNotification(data);
     }
     if (config_api.getConfigItem('ytdl_use_telegram_API') && config_api.getConfigItem('ytdl_telegram_bot_token') && config_api.getConfigItem('ytdl_telegram_chat_id')) {
-        sendTelegramNotification(data);
+        exports.sendTelegramNotification(data);
     }
     if (config_api.getConfigItem('ytdl_webhook_url')) {
         sendGenericNotification(data);
@@ -113,6 +114,8 @@ function notificationEnabled(type) {
     return config_api.getConfigItem('ytdl_enable_notifications') && (config_api.getConfigItem('ytdl_enable_all_notifications') || config_api.getConfigItem('ytdl_allowed_notification_types').includes(type));
 }
 
+// ntfy
+
 function sendNtfyNotification({body, title, type, url, thumbnail}) {
     logger.verbose('Sending notification to ntfy');
     fetch(config_api.getConfigItem('ytdl_ntfy_topic_url'), {
@@ -127,6 +130,8 @@ function sendNtfyNotification({body, title, type, url, thumbnail}) {
     });
 }
 
+// Gotify
+
 async function sendGotifyNotification({body, title, type, url, thumbnail}) {
     logger.verbose('Sending notification to gotify');
     await gotify({
@@ -145,15 +150,50 @@ async function sendGotifyNotification({body, title, type, url, thumbnail}) {
       });
 }
 
-async function sendTelegramNotification({body, title, type, url, thumbnail}) {
+// Telegram
-    logger.verbose('Sending notification to Telegram');
+
+setupTelegramBot();
+config_api.config_updated.subscribe(change => {
+    const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
     const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
-    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    if (!use_telegram_api || !bot_token) return;
-    const bot = new TelegramBot(bot_token);
+    if (!change) return;
-    if (thumbnail) await bot.sendPhoto(chat_id, thumbnail);
+    if (change['key'] === 'ytdl_use_telegram_API' || change['key'] === 'ytdl_telegram_bot_token' || change['key'] === 'ytdl_telegram_webhook_proxy') {
-    bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+        logger.debug('Telegram bot setting up');
+        setupTelegramBot();
+    }
+});
+
+async function setupTelegramBot() {
+    const use_telegram_api = config_api.getConfigItem('ytdl_use_telegram_API');
+    const bot_token = config_api.getConfigItem('ytdl_telegram_bot_token');
+    if (!use_telegram_api || !bot_token) return;
+    
+    telegram_bot = new TelegramBotAPI(bot_token);
+    const webhook_proxy = config_api.getConfigItem('ytdl_telegram_webhook_proxy');
+    const webhook_url = webhook_proxy ? webhook_proxy : `${utils.getBaseURL()}/api/telegramRequest`;
+    telegram_bot.setWebHook(webhook_url);
 }
 
+exports.sendTelegramNotification = async ({body, title, type, url, thumbnail}) => {
+    if (!telegram_bot){
+        logger.error('Telegram bot not found!');
+        return;
+    }
+
+    const chat_id = config_api.getConfigItem('ytdl_telegram_chat_id');
+    if (!chat_id){
+        logger.error('Telegram chat ID required!');
+        return;
+    }
+    
+    logger.verbose('Sending notification to Telegram');
+    if (thumbnail) await telegram_bot.sendPhoto(chat_id, thumbnail);
+    telegram_bot.sendMessage(chat_id, `<b>${title}</b>\n\n${body}\n<a href="${url}">${url}</a>`, {parse_mode: 'HTML'});
+}
+
+// Discord
+
 async function sendDiscordNotification({body, title, type, url, thumbnail}) {
     const discord_webhook_url = config_api.getConfigItem('ytdl_discord_webhook_url');
     const url_split = discord_webhook_url.split('webhooks/');
@@ -177,6 +217,8 @@ async function sendDiscordNotification({body, title, type, url, thumbnail}) {
     return result;
 }
 
+// Slack
+
 function sendSlackNotification({body, title, type, url, thumbnail}) {
     const slack_webhook_url = config_api.getConfigItem('ytdl_slack_webhook_url');
     logger.verbose(`Sending slack notification to ${slack_webhook_url}`);
@@ -236,6 +278,8 @@ function sendSlackNotification({body, title, type, url, thumbnail}) {
     });
 }
 
+// Generic
+
 function sendGenericNotification(data) {
     const webhook_url = config_api.getConfigItem('ytdl_webhook_url');
     logger.verbose(`Sending generic notification to ${webhook_url}`);

backend/test/tests.js

@@ -1037,6 +1037,66 @@ describe('Categories', async function() {
     });
 });
 
+describe('Config', async function() {
+    it('findChangedConfigItems', async function() {
+        const old_config = {
+            "YoutubeDLMaterial": {
+                "test_object1": {
+                    "test_prop1": true,
+                    "test_prop2": false
+                },
+                "test_object2": {
+                    "test_prop3": {
+                        "test_prop3_1": true,
+                        "test_prop3_2": false
+                    },
+                    "test_prop4": false
+                },
+                "test_object3": {
+                    "test_prop5": {
+                        "test_prop5_1": true,
+                        "test_prop5_2": false
+                    },
+                    "test_prop6": false
+                }
+            }
+        };
+
+        const new_config = {
+            "YoutubeDLMaterial": {
+                "test_object1": {
+                    "test_prop1": false,
+                    "test_prop2": false
+                },
+                "test_object2": {
+                    "test_prop3": {
+                        "test_prop3_1": false,
+                        "test_prop3_2": false
+                    },
+                    "test_prop4": true
+                },
+                "test_object3": {
+                    "test_prop5": {
+                        "test_prop5_1": true,
+                        "test_prop5_2": false
+                    },
+                    "test_prop6": true
+                }
+            }
+        };
+
+        const changes = config_api.findChangedConfigItems(old_config, new_config);
+        assert(changes[0]['key'] === 'test_prop1' && changes[0]['old_value'] === true && changes[0]['new_value'] === false);
+        assert(changes[1]['key'] === 'test_prop3' &&
+                changes[1]['old_value']['test_prop3_1'] === true &&
+                changes[1]['new_value']['test_prop3_1'] === false &&
+                changes[1]['old_value']['test_prop3_2'] === false &&
+                changes[1]['new_value']['test_prop3_2'] === false);
+        assert(changes[2]['key'] === 'test_prop4' && changes[2]['old_value'] === false && changes[2]['new_value'] === true);
+        assert(changes[3]['key'] === 'test_prop6' && changes[3]['old_value'] === false && changes[3]['new_value'] === true);
+    });
+});
+
 const generateEmptyVideoFile = async (file_path) => {
     if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
     return await exec(`ffmpeg -t 1 -f lavfi -i color=c=black:s=640x480 -c:v libx264 -tune stillimage -pix_fmt yuv420p "${file_path}"`);

src/app/settings/settings.component.html

@@ -426,6 +426,13 @@
                             <mat-hint><a target="_blank" href="https://stackoverflow.com/a/37396871/8088021"><ng-container i18n="Telegram chat ID help">How do I get the chat ID?</ng-container></a></mat-hint>
                         </mat-form-field>
                     </div>
+                    <div class="col-12 mb-2">
+                        <mat-form-field class="text-field" color="accent">
+                            <mat-label i18n="Telegram webhook proxy">Telegram webhook proxy</mat-label>
+                            <input placeholder="https://smee.io/XXXXX" [disabled]="!new_config['Extra']['enable_notifications'] || !new_config['API']['use_telegram_API']" [(ngModel)]="new_config['API']['telegram_webhook_proxy']" matInput>
+                            <mat-hint><a target="_blank" href="https://smee.io/"><ng-container i18n="Telegram webhook proxy help">Example service</ng-container></a></mat-hint>
+                        </mat-form-field>
+                    </div>
                 </div>
             </div>
         </ng-template>