Added v1 of chat sidebar for Twitch VODs

backend/app.js

@@ -27,6 +27,7 @@ const url_api = require('url');
 var config_api = require('./config.js');
 var subscriptions_api = require('./subscriptions')
 var categories_api = require('./categories');
+var twitch_api = require('./twitch');
 const CONSTS = require('./consts')
 const { spawn } = require('child_process')
 const read_last_lines = require('read-last-lines');
@@ -38,6 +39,7 @@ var app = express();
 
 // database setup
 const FileSync = require('lowdb/adapters/FileSync');
+const config = require('./config.js');
 
 const adapter = new FileSync('./appdata/db.json');
 const db = low(adapter)
@@ -1186,6 +1188,13 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
                     var full_file_path = filepath_no_extension + ext;
                     var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
 
+                    if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
+                        && config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) {
+                            let vodId = url.split('twitch.tv/videos/')[1];
+                            vodId = vodId.split('?')[0];
+                            downloadTwitchChatByVODID(vodId, file_name, type, options.user);
+                    }
+
                     // renames file if necessary due to bug
                     if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
                         try {
@@ -1759,6 +1768,42 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
+async function getTwitchChatByFileID(id, type, user_uid, uuid) {
+    let file_path = null;
+
+    if (user_uid) {
+        file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+    } else {
+        file_path = path.join(type, id + '.twitch_chat.json');
+    }
+
+    var chat_file = null;
+    if (fs.existsSync(file_path)) {
+        chat_file = fs.readJSONSync(file_path);
+    }
+
+    return chat_file;
+}
+
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid) {
+    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
+    const chat = await twitch_api.getCommentsForVOD(twitch_api_key, vodId);
+
+    // save file if needec params are included
+    if (id && type) {
+        let file_path = null;
+        if (user_uid) {
+            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+        } else {
+            file_path = path.join(type, id + '.twitch_chat.json');
+        }
+
+        if (chat) fs.writeJSONSync(file_path, chat);
+    }
+
+    return chat;
+}
+
 app.use(function(req, res, next) {
     res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization");
     res.header("Access-Control-Allow-Origin", getOrigin());
@@ -2058,6 +2103,54 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     });
 });
 
+app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
+    var id = req.body.id;
+    var type = req.body.type;
+    var uuid = req.body.uuid;
+    var user_uid = null;
+
+    if (req.isAuthenticated()) user_uid = req.user.uid;
+
+    const chat_file = await getTwitchChatByFileID(id, type, user_uid, uuid);
+
+    res.send({
+        chat: chat_file
+    });
+});
+
+app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
+    var id = req.body.id;
+    var type = req.body.type;
+    var vodId = req.body.vodId;
+    var uuid = req.body.uuid;
+    var user_uid = null;
+
+    if (req.isAuthenticated()) user_uid = req.user.uid;
+
+    // check if file already exists. if so, send that instead
+    const file_exists_check = await getTwitchChatByFileID(id, type, user_uid, uuid);
+    if (file_exists_check) {
+        res.send({chat: file_exists_check});
+        return;
+    }
+
+    const full_chat = await downloadTwitchChatByVODID(vodId);
+
+    let file_path = null;
+
+    if (user_uid) {
+        file_path = path.join('users', req.user.uid, type, id + '.twitch_chat.json');
+    } else {
+        file_path = path.join(type, id + '.twitch_chat.json');
+    }
+
+    if (full_chat) fs.writeJSONSync(file_path, full_chat);
+
+    res.send({
+        chat: full_chat
+    });
+});
+
 // video sharing
 app.post('/api/enableSharing', optionalJwt, function(req, res) {
     var type = req.body.type;

backend/appdata/default.json

@@ -26,7 +26,10 @@
       "use_API_key": false,
       "API_key": "",
       "use_youtube_API": false,
-      "youtube_API_key": ""
+      "youtube_API_key": "",
+      "use_twitch_API": false,
+      "twitch_API_key": "",
+      "twitch_auto_download_chat": false
     },
     "Themes": {
       "default_theme": "default",

backend/config.js

@@ -203,7 +203,10 @@ DEFAULT_CONFIG = {
         "use_API_key": false,
         "API_key": "",
         "use_youtube_API": false,
-        "youtube_API_key": ""
+        "youtube_API_key": "",
+        "use_twitch_API": false,
+        "twitch_API_key": "",
+        "twitch_auto_download_chat": false
       },
       "Themes": {
         "default_theme": "default",

backend/consts.js

@@ -86,6 +86,18 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_youtube_api_key',
         'path': 'YoutubeDLMaterial.API.youtube_API_key'
     },
+    'ytdl_use_twitch_api': {
+        'key': 'ytdl_use_twitch_api',
+        'path': 'YoutubeDLMaterial.API.use_twitch_API'
+    },
+    'ytdl_twitch_api_key': {
+        'key': 'ytdl_twitch_api_key',
+        'path': 'YoutubeDLMaterial.API.twitch_API_key'
+    },
+    'ytdl_twitch_auto_download_chat': {
+        'key': 'ytdl_twitch_auto_download_chat',
+        'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
+    },
 
     // Themes
     'ytdl_default_theme': {

backend/twitch.js

@@ -0,0 +1,71 @@
+var moment = require('moment');
+var Axios = require('axios');
+
+async function getCommentsForVOD(clientID, vodId) {
+    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
+        batch,
+        cursor;
+
+    let comments = null;
+
+    try {
+        do {
+            batch = (await Axios.get(url, {
+                headers: {
+                    'Client-ID': clientID,
+                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
+                    'Content-Type': 'application/json; charset=UTF-8',
+                }
+            })).data;
+
+            const str = batch.comments.map(c => {
+                let {
+                        created_at: msgCreated,
+                        content_offset_seconds: timestamp,
+                        commenter: {
+                            name,
+                            _id,
+                            created_at: acctCreated
+                        },
+                        message: {
+                            body: msg
+                        }
+                    } = c;
+
+                const timestamp_str = moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+                        });
+
+                acctCreated = moment(acctCreated).utc();
+                msgCreated = moment(msgCreated).utc();
+
+                if (!comments) comments = [];
+
+                comments.push({
+                    timestamp: timestamp,
+                    timestamp_str: timestamp_str,
+                    name: name,
+                    message: msg
+                });
+                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
+                // return line;
+            }).join('\n');
+
+            cursor = batch._next;
+            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
+            await new Promise(res => setTimeout(res, 300));
+        } while (cursor);
+    } catch (err) {
+        console.error(err);
+    }
+
+    return comments;
+}
+
+module.exports = {
+    getCommentsForVOD: getCommentsForVOD
+}