Added ability to download twitch emotes in the backend

backend/package.json

@@ -25,7 +25,7 @@
   "dependencies": {
     "@discordjs/builders": "^1.6.1",
     "@discordjs/core": "^0.5.2",
-    "@tzahi12345/twitch-emoticons": "^1.0.3",
+    "@tzahi12345/twitch-emoticons": "^1.0.4",
     "archiver": "^5.3.1",
     "async": "^3.2.3",
     "async-mutex": "^0.4.0",

backend/test/tests.js

@@ -550,9 +550,40 @@ describe('Downloader', function() {
         const expected_args3 =  ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
         assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
     });
+});
+
 describe('Twitch', async function () {
     const twitch_api = require('../twitch');
     const example_vod = '1710641401';
+    const example_channel = 'keffals';
+
+    it('Get OAuth Token', async function() {
+        this.timeout(300000);
+        const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+        const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+        if (!twitch_client_id || !twitch_client_secret) {
+            logger.info(`Skipping test 'Get OAuth Token' as Twitch client ID or Twitch client secret is missing.`);
+            assert(true);
+            return;
+        }
+        const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
+        assert(token);
+    });
+
+    it('Get channel ID', async function() {
+        this.timeout(300000);
+        const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+        const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+        if (!twitch_client_id || !twitch_client_secret) {
+            logger.info(`Skipping test 'Get channel ID' as Twitch client ID or Twitch client secret is missing.`);
+            assert(true);
+            return;
+        }
+        const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
+        const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
+        assert(channel_id === '494493142');
+    });
+
     it('Download VOD chat', async function() {
         this.timeout(300000);
         if (!fs.existsSync('TwitchDownloaderCLI')) {
@@ -571,6 +602,20 @@ describe('Downloader', function() {
         // cleanup
         if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
     });
+
+    it('Download Twitch emotes', async function() {
+        this.timeout(300000);
+        const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+        const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+        if (!twitch_client_id || !twitch_client_secret) {
+            logger.info(`Skipping test 'Download Twitch emotes' as Twitch client ID or Twitch client secret is missing.`);
+            assert(true);
+            return;
+        }
+        const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
+        const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
+        const emotesJSON = await twitch_api.downloadTwitchEmotes(channel_id, example_channel);
+        assert(emotesJSON && emotesJSON.length > 0);
     });
 });
 

backend/twitch.js

@@ -1,8 +1,11 @@
 const config_api = require('./config');
 const logger = require('./logger');
+const utils = require('./utils');
 
 const moment = require('moment');
 const fs = require('fs-extra')
+const axios = require('axios');
+const { EmoteFetcher } = require('@tzahi12345/twitch-emoticons');
 const path = require('path');
 const { promisify } = require('util');
 const child_process = require('child_process');
@@ -108,6 +111,96 @@ exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, custo
     return chat;
 }
 
+exports.downloadTwitchEmotes = async (channel_id, channel_name) => {
+    const twitch_client_id          = config_api.getConfigItem('ytdl_twitch_client_id');
+    const twitch_client_secret      = config_api.getConfigItem('ytdl_twitch_client_secret');
+
+    const fetcher = new EmoteFetcher(twitch_client_id, twitch_client_secret);
+
+    try {
+        await Promise.all([
+            fetcher.fetchTwitchEmotes(),
+            fetcher.fetchTwitchEmotes(channel_id),
+            fetcher.fetchBTTVEmotes(),
+            fetcher.fetchBTTVEmotes(channel_id),
+            // fetcher.fetchSevenTVEmotes(),
+            // fetcher.fetchSevenTVEmotes(channel_id),
+            fetcher.fetchFFZEmotes(),
+            fetcher.fetchFFZEmotes(channel_id)
+        ]);
+
+        const channel_dir = path.join('appdata', 'emotes', channel_id);
+        fs.ensureDirSync(channel_dir);
+        
+        const emotesJSON = [];
+        let failed_emote_count = 0;
+        for (const [, emote] of fetcher.emotes) {
+            const emoteJSON = emote.toJSON();
+
+            const ext = emote.imageType;
+            const emote_path = path.join(channel_dir, `${emote.id}.${ext}`);
+
+            if (fs.existsSync(emote_path)) continue;
+            
+            try {
+                const link = emote.toLink();
+                await utils.fetchFile(link, emote_path);
+                emotesJSON.push(emoteJSON);
+            } catch (err) {
+                failed_emote_count++;
+            }
+        }
+        if (failed_emote_count) logger.warn(`${failed_emote_count} emotes failed to download for channel ${channel_name}`);
+        return emotesJSON;
+    } catch (err) {
+        logger.error(err);
+        return null;
+    }
+}
+
+exports.getTwitchOAuthToken = async (client_id, client_secret) => {
+    const url = `https://id.twitch.tv/oauth2/token`;
+  
+    try {
+        const response = await axios.post(url, {client_id: client_id, client_secret: client_secret, grant_type: 'client_credentials'});
+        const token = response['data']['access_token'];
+        if (token) return token;
+    
+        logger.error(`Failed to get token.`);
+        return null;
+    } catch (err) {
+        logger.error(`Failed to get token.`);
+        logger.error(err);
+        return null;
+    }
+}
+
+exports.getChannelID = async (username, client_id, oauth_token) => {
+    const url = `https://api.twitch.tv/helix/users?login=${username}`;
+    const headers = {
+        'Client-ID': client_id,
+        'Authorization': 'Bearer ' + oauth_token,
+        Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8'
+    };
+  
+    try {
+        const response = await axios.get(url, {headers: headers});
+        const data = response.data.data;
+    
+        if (data && data.length > 0) {
+            const channelID = data[0].id;
+            return channelID;
+        }
+    
+        logger.error(`Failed to get channel ID for user ${username}`);
+        if (data.error) logger.error(data.error);
+        return null; // User not found
+    } catch (err) {
+        logger.error(`Failed to get channel ID for user ${username}`);
+        logger.error(err);
+    }
+}
+
 const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
                     .toISOString()
                     .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,

backend/utils.js

@@ -364,26 +364,29 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
 }
 
 // helper function to download file using fetch
-exports.fetchFile = async (url, path, file_label) => {
+exports.fetchFile = async (url, output_path, file_label = null) => {
     var len = null;
     const res = await fetch(url);
-
+    let bar = null;
+    if (file_label) {
         len = parseInt(res.headers.get("Content-Length"), 10);
 
-    var bar = new ProgressBar(`  Downloading ${file_label} [:bar] :percent :etas`, {
+        bar = new ProgressBar(`  Downloading ${file_label} [:bar] :percent :etas`, {
             complete: '=',
             incomplete: ' ',
             width: 20,
             total: len
         });
-    const fileStream = fs.createWriteStream(path);
+    }
+
+    const fileStream = fs.createWriteStream(output_path);
     await new Promise((resolve, reject) => {
         res.body.pipe(fileStream);
         res.body.on("error", (err) => {
           reject(err);
         });
         res.body.on('data', function (chunk) {
-            bar.tick(chunk.length);
+            if (file_label) bar.tick(chunk.length);
         });
         fileStream.on("finish", function() {
           resolve();