diff --git a/backend/app.js b/backend/app.js index 10776658..761d5ce9 100644 --- a/backend/app.js +++ b/backend/app.js @@ -706,7 +706,7 @@ app.use(function(req, res, next) { next(); } else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) { next(); - } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) { + } else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/emote/') || req.path.includes('/api/rss')) { next(); } else { logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`); @@ -993,11 +993,11 @@ app.post('/api/updateConcurrentStream', optionalJwt, async (req, res) => { }); app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => { - var id = req.body.id; - var type = req.body.type; - var uuid = req.body.uuid; - var sub = req.body.sub; - var user_uid = null; + const id = req.body.id; + const type = req.body.type; + const uuid = req.body.uuid; + const sub = req.body.sub; + let user_uid = null; if (req.isAuthenticated()) user_uid = req.user.uid; @@ -1009,12 +1009,12 @@ app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => { }); app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { - var id = req.body.id; - var type = req.body.type; - var vodId = req.body.vodId; - var uuid = req.body.uuid; - var sub = req.body.sub; - var user_uid = null; + const id = req.body.id; + const type = req.body.type; + const vodId = req.body.vodId; + const uuid = req.body.uuid; + const sub = req.body.sub; + let user_uid = null; if (req.isAuthenticated()) user_uid = req.user.uid; @@ -1025,17 +1025,47 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { return; } + logger.info(`Downloading Twitch chat for ${id}`); const full_chat = await twitch_api.downloadTwitchChatByVODID(vodId, id, type, user_uid, sub); + logger.info(`Finished downloading Twitch chat for ${id}`); res.send({ chat: full_chat }); }); +app.post('/api/getTwitchEmotes', async (req, res) => { + const uid = req.body.uid; + + const file = await files_api.getVideo(uid); + const channel_name = file['uploader']; + + const emotes_path = path.join('appdata', 'emotes', uid, 'emotes.json') + if (!fs.existsSync(emotes_path)) { + logger.info(`Downloading Twitch emotes for ${channel_name}`); + await twitch_api.downloadTwitchEmotes(channel_name, file.uid); + logger.info(`Finished downloading Twitch emotes for ${channel_name}`); + } + + const emotes = await twitch_api.getTwitchEmotes(file.uid); + + res.send({ + emotes: emotes + }); +}); + +app.get('/api/emote/:uid/:id', async (req, res) => { + const file_uid = decodeURIComponent(req.params.uid); + const emote_id = decodeURIComponent(req.params.id); + const emote_path = path.join('appdata', 'emotes', file_uid, emote_id); + if (fs.existsSync(emote_path)) path.isAbsolute(emote_path) ? res.sendFile(emote_path) : res.sendFile(path.join(__dirname, emote_path)); + else res.sendStatus(404); +}); + // video sharing app.post('/api/enableSharing', optionalJwt, async (req, res) => { - var uid = req.body.uid; - var is_playlist = req.body.is_playlist; + const uid = req.body.uid; + const is_playlist = req.body.is_playlist; let success = false; // multi-user mode if (req.isAuthenticated()) { @@ -1643,6 +1673,8 @@ app.get('/api/stream', optionalJwt, async (req, res) => { } if (!fs.existsSync(file_path)) { logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`); + res.sendStatus(404); + return; } const stat = fs.statSync(file_path); const fileSize = stat.size; diff --git a/backend/package-lock.json b/backend/package-lock.json index e66d0e8b..8f8b898e 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@discordjs/builders": "^1.6.1", "@discordjs/core": "^0.5.2", - "@tzahi12345/twitch-emoticons": "^1.0.4", + "@tzahi12345/twitch-emoticons": "^1.0.9", "archiver": "^5.3.1", "async": "^3.2.3", "async-mutex": "^0.4.0", @@ -548,9 +548,9 @@ } }, "node_modules/@tzahi12345/twitch-emoticons": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.4.tgz", - "integrity": "sha512-SeefOMIZsZJurUo0Qe3sS8TOqo6EtEoE5Cm1/REBv6MpD6jXmTjYdGKrfMkeKOyjyVv92A/CukL1PnkIUn3Qbg==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz", + "integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==", "dependencies": { "@twurple/api": "^6.2.1", "@twurple/auth": "^6.2.1", @@ -6087,9 +6087,9 @@ } }, "@tzahi12345/twitch-emoticons": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.4.tgz", - "integrity": "sha512-SeefOMIZsZJurUo0Qe3sS8TOqo6EtEoE5Cm1/REBv6MpD6jXmTjYdGKrfMkeKOyjyVv92A/CukL1PnkIUn3Qbg==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz", + "integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==", "requires": { "@twurple/api": "^6.2.1", "@twurple/auth": "^6.2.1", diff --git a/backend/package.json b/backend/package.json index 98652a1e..0bc08d5a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -25,7 +25,7 @@ "dependencies": { "@discordjs/builders": "^1.6.1", "@discordjs/core": "^0.5.2", - "@tzahi12345/twitch-emoticons": "^1.0.4", + "@tzahi12345/twitch-emoticons": "^1.0.9", "archiver": "^5.3.1", "async": "^3.2.3", "async-mutex": "^0.4.0", diff --git a/backend/test/tests.js b/backend/test/tests.js index c476f70c..0927e858 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -579,8 +579,7 @@ describe('Twitch', async function () { assert(true); return; } - const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret); - const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token); + const channel_id = await twitch_api.getChannelID(example_channel); assert(channel_id === '494493142'); }); @@ -612,9 +611,7 @@ describe('Twitch', async function () { assert(true); return; } - const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret); - const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token); - const emotesJSON = await twitch_api.downloadTwitchEmotes(channel_id, example_channel); + const emotesJSON = await twitch_api.downloadTwitchEmotes(example_channel, 'test_uid'); assert(emotesJSON && emotesJSON.length > 0); }); }); diff --git a/backend/twitch.js b/backend/twitch.js index 4758fbbb..51552c50 100644 --- a/backend/twitch.js +++ b/backend/twitch.js @@ -11,6 +11,21 @@ const { promisify } = require('util'); const child_process = require('child_process'); const commandExistsSync = require('command-exists').sync; +let auth_timeout = null; +let cached_oauth = null; + +exports.ensureTwitchAuth = async () => { + const TIMEOUT_MARGIN_MS = 60*1000; + const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id'); + const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret'); + if (cached_oauth && auth_timeout && (Date.now() - TIMEOUT_MARGIN_MS) < auth_timeout) return cached_oauth; + + const {token, expires_in} = await exports.getTwitchOAuthToken(twitch_client_id, twitch_client_secret); + cached_oauth = token; + auth_timeout = Date.now() + expires_in; + return token; +} + exports.getCommentsForVOD = async (vodId) => { const exec = promisify(child_process.exec); @@ -111,47 +126,61 @@ exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, custo return chat; } -exports.downloadTwitchEmotes = async (channel_id, channel_name) => { +exports.getTwitchEmotes = async (file_uid) => { + const emotes_path = path.join('appdata', 'emotes', file_uid, 'emotes.json') + if (!fs.existsSync(emotes_path)) return null; + const emote_objs = fs.readJSONSync(emotes_path); + // inject custom url + for (const emote_obj of emote_objs) { + emote_obj.custom_url = `${utils.getBaseURL()}/api/emote/${file_uid}/${emote_obj.id}.${emote_obj.ext}` + } + return emote_objs; +} + +exports.downloadTwitchEmotes = async (channel_name, file_uid) => { const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id'); const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret'); + const channel_id = await exports.getChannelID(channel_name); + const fetcher = new EmoteFetcher(twitch_client_id, twitch_client_secret); try { - await Promise.all([ + await Promise.allSettled([ fetcher.fetchTwitchEmotes(), fetcher.fetchTwitchEmotes(channel_id), fetcher.fetchBTTVEmotes(), fetcher.fetchBTTVEmotes(channel_id), - // fetcher.fetchSevenTVEmotes(), - // fetcher.fetchSevenTVEmotes(channel_id), + fetcher.fetchSevenTVEmotes(), + fetcher.fetchSevenTVEmotes(channel_id), fetcher.fetchFFZEmotes(), fetcher.fetchFFZEmotes(channel_id) ]); - const channel_dir = path.join('appdata', 'emotes', channel_id); - fs.ensureDirSync(channel_dir); - - const emotesJSON = []; + const emotes_dir = path.join('appdata', 'emotes', file_uid); + const emote_json_path = path.join(emotes_dir, `emotes.json`); + fs.ensureDirSync(emotes_dir); + + const emote_objs = []; let failed_emote_count = 0; for (const [, emote] of fetcher.emotes) { - const emoteJSON = emote.toJSON(); + const emote_obj = emote.toObject(); const ext = emote.imageType; - const emote_path = path.join(channel_dir, `${emote.id}.${ext}`); - - if (fs.existsSync(emote_path)) continue; + const emote_image_path = path.join(emotes_dir, `${emote.id}.${ext}`); try { const link = emote.toLink(); - await utils.fetchFile(link, emote_path); - emotesJSON.push(emoteJSON); + if (!fs.existsSync(emote_image_path)) await utils.fetchFile(link, emote_image_path); + emote_obj['ext'] = ext; + emote_objs.push(emote_obj); } catch (err) { failed_emote_count++; } } if (failed_emote_count) logger.warn(`${failed_emote_count} emotes failed to download for channel ${channel_name}`); - return emotesJSON; + await fs.writeJSON(emote_json_path, emote_objs); + return emote_objs; } catch (err) { logger.error(err); return null; @@ -159,12 +188,14 @@ exports.downloadTwitchEmotes = async (channel_id, channel_name) => { } exports.getTwitchOAuthToken = async (client_id, client_secret) => { + logger.verbose('Generating new Twitch auth token'); const url = `https://id.twitch.tv/oauth2/token`; try { const response = await axios.post(url, {client_id: client_id, client_secret: client_secret, grant_type: 'client_credentials'}); const token = response['data']['access_token']; - if (token) return token; + const expires_in = response['data']['expires_in']; + if (token) return {token, expires_in}; logger.error(`Failed to get token.`); return null; @@ -175,12 +206,14 @@ exports.getTwitchOAuthToken = async (client_id, client_secret) => { } } -exports.getChannelID = async (username, client_id, oauth_token) => { - const url = `https://api.twitch.tv/helix/users?login=${username}`; +exports.getChannelID = async (channel_name) => { + const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id'); + const token = await exports.ensureTwitchAuth(); + const url = `https://api.twitch.tv/helix/users?login=${channel_name}`; const headers = { - 'Client-ID': client_id, - 'Authorization': 'Bearer ' + oauth_token, - Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8' + 'Client-ID': twitch_client_id, + 'Authorization': 'Bearer ' + token, + // Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8' }; try { @@ -192,11 +225,11 @@ exports.getChannelID = async (username, client_id, oauth_token) => { return channelID; } - logger.error(`Failed to get channel ID for user ${username}`); + logger.error(`Failed to get channel ID for user ${channel_name}`); if (data.error) logger.error(data.error); return null; // User not found } catch (err) { - logger.error(`Failed to get channel ID for user ${username}`); + logger.error(`Failed to get channel ID for user ${channel_name}`); logger.error(err); } } diff --git a/package-lock.json b/package-lock.json index 3a6c5593..2058c2af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@angular/router": "^16.0.3", "@fontsource/material-icons": "^4.5.4", "@ngneat/content-loader": "^7.0.0", - "@tzahi12345/twitch-emoticons": "^1.0.3", + "@tzahi12345/twitch-emoticons": "^1.0.9", "@videogular/ngx-videogular": "^6.0.0", "core-js": "^2.4.1", "crypto-js": "^4.1.1", @@ -5441,9 +5441,9 @@ } }, "node_modules/@tzahi12345/twitch-emoticons": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.3.tgz", - "integrity": "sha512-V8FC+beYhAYn2Nmk66y2SzG+Lr9cNtklVp4cHSr+rUCIKJSFVjmO8cQd9shFRoQm0kpZX5X5+2TGhYFsUSr8/w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz", + "integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==", "dependencies": { "@twurple/api": "^6.2.1", "@twurple/auth": "^6.2.1", @@ -21502,9 +21502,9 @@ } }, "@tzahi12345/twitch-emoticons": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.3.tgz", - "integrity": "sha512-V8FC+beYhAYn2Nmk66y2SzG+Lr9cNtklVp4cHSr+rUCIKJSFVjmO8cQd9shFRoQm0kpZX5X5+2TGhYFsUSr8/w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz", + "integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==", "requires": { "@twurple/api": "^6.2.1", "@twurple/auth": "^6.2.1", diff --git a/package.json b/package.json index 32adf808..b56075c4 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@angular/router": "^16.0.3", "@fontsource/material-icons": "^4.5.4", "@ngneat/content-loader": "^7.0.0", - "@tzahi12345/twitch-emoticons": "^1.0.3", + "@tzahi12345/twitch-emoticons": "^1.0.9", "@videogular/ngx-videogular": "^6.0.0", "core-js": "^2.4.1", "crypto-js": "^4.1.1", @@ -80,4 +80,4 @@ "ts-node": "~3.0.4", "tslint": "~6.1.0" } -} \ No newline at end of file +} diff --git a/src/app/components/twitch-chat/twitch-chat.component.ts b/src/app/components/twitch-chat/twitch-chat.component.ts index 04a44b38..e643721b 100644 --- a/src/app/components/twitch-chat/twitch-chat.component.ts +++ b/src/app/components/twitch-chat/twitch-chat.component.ts @@ -1,7 +1,7 @@ import { Component, ElementRef, Input, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core'; import { DatabaseFile } from 'api-types'; import { PostsService } from 'app/posts.services'; -import { EmoteFetcher, EmoteJSON, EmoteParser } from '@tzahi12345/twitch-emoticons'; +import { EmoteFetcher, EmoteObject, EmoteParser } from '@tzahi12345/twitch-emoticons'; @Component({ selector: 'app-twitch-chat', @@ -14,6 +14,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy { visible_chat = null; chat_response_received = false; downloading_chat = false; + got_emotes = false; current_chat_index = null; @@ -36,6 +37,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy { ngOnInit(): void { this.getFullChat(); + this.getEmotes(); } ngOnDestroy(): void { @@ -75,7 +77,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy { for (let i = this.current_chat_index + 1; i < this.full_chat.length; i++) { const new_chat = this.full_chat[i]; if (new_chat['timestamp'] >= latest_chat_timestamp && new_chat['timestamp'] <= this.current_timestamp) { - new_chat['message'] = this.parseChat(new_chat['message']); + new_chat['message'] = this.got_emotes ? this.parseChat(new_chat['message']) : new_chat['message']; this.visible_chat.push(new_chat); this.current_chat_index = i; } else if (new_chat['timestamp'] > this.current_timestamp) { @@ -125,22 +127,22 @@ export class TwitchChatComponent implements OnInit, OnDestroy { } getEmotes() { - this.postsService.getTwitchEmotes().subscribe(res => { + this.postsService.getTwitchEmotes(this.db_file['uid']).subscribe(res => { const emotes = res['emotes']; this.processEmotes(emotes); }); } - processEmotes(emotes: EmoteJSON[]) { + processEmotes(emotes: EmoteObject[]) { this.fetcher = new EmoteFetcher(); this.parser = new EmoteParser(this.fetcher, { // Custom HTML format - template: '{name}', + template: `{name}`, // Match without :colons: match: /(\w+)+?/g }); - - this.fetcher.fromJSON(emotes); + this.fetcher.fromObject(emotes); + this.got_emotes = true; } parseChat(chat_message: string) { diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 793faf30..2b1055ed 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -118,7 +118,7 @@ import { import { isoLangs } from './dialogs/user-profile-dialog/locales_list'; import { Title } from '@angular/platform-browser'; import { MatDrawerMode } from '@angular/material/sidenav'; -import type { EmoteJSON } from '@tzahi12345/twitch-emoticons'; +import type { EmoteObject } from '@tzahi12345/twitch-emoticons'; @Injectable() export class PostsService { @@ -408,8 +408,8 @@ export class PostsService { return this.http.post(this.path + 'downloadTwitchChatByVODID', body, this.httpOptions); } - getTwitchEmotes() { - return this.http.post<{emotes: EmoteJSON[]}>(this.path + 'getTwitchEmotes', {}, this.httpOptions); + getTwitchEmotes(uid: string) { + return this.http.post<{emotes: EmoteObject[]}>(this.path + 'getTwitchEmotes', {uid: uid}, this.httpOptions); } downloadPlaylistFromServer(playlist_id, uuid = null) {