Fixed twitch chat downloads, tcd is now required and client secret must be supplied

backend/appdata/default.json

@@ -31,7 +31,8 @@
       "use_youtube_API": false,
       "youtube_API_key": "",
       "use_twitch_API": false,
-      "twitch_API_key": "",
+      "twitch_client_ID": "",
+      "twitch_client_secret": "",
       "twitch_auto_download_chat": false,
       "use_sponsorblock_API": false,
       "generate_NFO_files": false

backend/config.js

@@ -206,7 +206,8 @@ const DEFAULT_CONFIG = {
         "use_youtube_API": false,
         "youtube_API_key": "",
         "use_twitch_API": false,
-        "twitch_API_key": "",
+        "twitch_client_ID": "",
+        "twitch_client_secret": "",
         "twitch_auto_download_chat": false,
         "use_sponsorblock_API": false,
         "generate_NFO_files": false

backend/consts.js

@@ -102,9 +102,13 @@ exports.CONFIG_ITEMS = {
         'key': 'ytdl_use_twitch_api',
         'path': 'YoutubeDLMaterial.API.use_twitch_API'
     },
-    'ytdl_twitch_api_key': {
+    'ytdl_twitch_client_id': {
-        'key': 'ytdl_twitch_api_key',
+        'key': 'ytdl_twitch_client_id',
-        'path': 'YoutubeDLMaterial.API.twitch_API_key'
+        'path': 'YoutubeDLMaterial.API.twitch_client_ID'
+    },
+    'ytdl_twitch_client_secret': {
+        'key': 'ytdl_twitch_client_secret',
+        'path': 'YoutubeDLMaterial.API.twitch_client_secret'
     },
     'ytdl_twitch_auto_download_chat': {
         'key': 'ytdl_twitch_auto_download_chat',

backend/test/tests.js

@@ -1,6 +1,7 @@
-var assert = require('assert');
+const assert = require('assert');
 const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
 
 process.chdir('./backend')
 
@@ -465,6 +466,20 @@ describe('Downloader', function() {
         console.log(updated_args2);
         assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
     });
+    describe('Twitch', async function () {
+        const twitch_api = require('../twitch');
+        const example_vod = '1493770675';
+        it('Download VOD', async function() {
+            const sample_path = path.join('test', 'sample.twitch_chat.json');
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+            this.timeout(300000);
+            await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+            assert(fs.existsSync(sample_path));
+
+            // cleanup
+            if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+        });
+    });
 });
 
 describe('Tasks', function() {

backend/twitch.js

@@ -1,90 +1,53 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
 const config_api = require('./config');
+const logger = require('./logger');
 
-async function getCommentsForVOD(clientID, vodId) {
+const moment = require('moment');
-    let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
+const fs = require('fs-extra')
-        batch,
+const path = require('path');
-        cursor;
 
-    let comments = null;
+async function getCommentsForVOD(clientID, clientSecret, vodId) {
+    const { promisify } = require('util');
+    const child_process = require('child_process');
+    const exec = promisify(child_process.exec);
+    const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
 
-    try {
+    if (result['stderr']) {
-        do {
+        logger.error(`Failed to download twitch comments for ${vodId}`);
-            batch = (await Axios.get(url, {
+        logger.error(result['stderr']);
-                headers: {
+        return null;
-                    'Client-ID': clientID,
-                    Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
-                    'Content-Type': 'application/json; charset=UTF-8',
     }
-            })).data;
 
-            const str = batch.comments.map(c => {
+    const raw_json = fs.readJSONSync(path.join('appdata', `${vodId}.json`));
-                let {
+    const new_json = raw_json.comments.map(comment_obj => {
-                        created_at: msgCreated,
+        return {
-                        content_offset_seconds: timestamp,
+            timestamp: comment_obj.content_offset_seconds,
-                        commenter: {
+            timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
-                            name,
+            name: comment_obj.commenter.name,
-                            _id,
+            message: comment_obj.message.body,
-                            created_at: acctCreated
+            user_color: comment_obj.message.user_color
-                        },
-                        message: {
-                            body: msg,
-                            user_color: user_color
         }
-                    } = c;
-
-                const timestamp_str = moment.duration(timestamp, 'seconds')
-                    .toISOString()
-                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
-                        (_, ...ms) => {
-                            const seg = v => v ? v.padStart(2, '0') : '00';
-                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
     });
 
-                acctCreated = moment(acctCreated).utc();
+    return new_json;
-                msgCreated = moment(msgCreated).utc();
-
-                if (!comments) comments = [];
-
-                comments.push({
-                    timestamp: timestamp,
-                    timestamp_str: timestamp_str,
-                    name: name,
-                    message: msg,
-                    user_color: user_color
-                });
-                // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
-                // return line;
-            }).join('\n');
-
-            cursor = batch._next;
-            url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
-            await new Promise(res => setTimeout(res, 300));
-        } while (cursor);
-    } catch (err) {
-        console.error(err);
-    }
-
-    return comments;
 }
 
 async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+    const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+    const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
     let file_path = null;
 
     if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+            file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
         }
     }
 
@@ -96,23 +59,28 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
     return chat_file;
 }
 
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
-    const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
+    const usersFileFolder           = config_api.getConfigItem('ytdl_users_base_path');
-    const chat = await getCommentsForVOD(twitch_api_key, vodId);
+    const subscriptionsFileFolder   = config_api.getConfigItem('ytdl_subscriptions_base_path');
+    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+    const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
 
     // save file if needed params are included
     let file_path = null;
-    if (user_uid) {
+    if (customFileFolderPath) {
+        file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+    } else if (user_uid) {
         if (sub) {
-            file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+            file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
         }
     } else {
         if (sub) {
-            file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+            file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
         } else {
-            file_path = path.join(type, id + '.twitch_chat.json');
+            file_path = path.join(type, `${id}.twitch_chat.json`);
         }
     }
 
@@ -121,6 +89,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
     return chat;
 }
 
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+                    .toISOString()
+                    .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+                        (_, ...ms) => {
+                            const seg = v => v ? v.padStart(2, '0') : '00';
+                            return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
 module.exports = {
     getCommentsForVOD: getCommentsForVOD,
     getTwitchChatByFileID: getTwitchChatByFileID,

src/app/settings/settings.component.html

@@ -263,11 +263,16 @@
                     </div>
                     <div class="col-12">
                         <mat-form-field class="text-field" color="accent">
-                            <input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_API_key']" matInput placeholder="Twitch API Key" i18n-placeholder="Twitch API Key setting placeholder" required>
+                            <input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_ID']" matInput placeholder="Twitch Client ID" i18n-placeholder="Twitch Client ID setting placeholder" required>
-                            <mat-hint><ng-container i18n="Twitch API Key setting hint AKA preamble">Also known as a Client ID.</ng-container>&nbsp;<a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch API Key setting hint">Generating a key is easy!</ng-container></a></mat-hint>
+                            <mat-hint><a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch Client ID setting hint">Generating an ID/secret is easy!</ng-container></a></mat-hint>
                         </mat-form-field>
                     </div>
-                    <div class="col-12 mt-4">
+                    <div class="col-12 mt-2">
+                        <mat-form-field class="text-field" color="accent">
+                            <input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_secret']" matInput placeholder="Twitch Client Secret" i18n-placeholder="Twitch Client Secret setting placeholder" required>
+                        </mat-form-field>
+                    </div>
+                    <div class="col-12 mt-2">
                         <mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_sponsorblock_API']" matTooltip="Enables a button to skip ads when viewing supported videos." i18n-matTooltip="SponsorBlock API tooltip"><ng-container i18n="Use SponsorBlock API setting">Use SponsorBlock API</ng-container></mat-checkbox>
                     </div>
                     <div class="col-12 mt-2 mb-3">