TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-28 07:30:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Updated and complete Twitch emoticon logic in the frontend and backend

Browse Source
This commit is contained in:
Tzahi12345
2023-06-03 16:08:27 -04:00
parent 3a20e03490
commit 8ac0ac2976
9 changed files with 133 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
backend/app.js
View File

@@ -706,7 +706,7 @@ app.use(function(req, res, next) {
next();
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
next();
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/emote/') || req.path.includes('/api/rss')) {
next();
} else {
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
@@ -993,11 +993,11 @@ app.post('/api/updateConcurrentStream', optionalJwt, async (req, res) => {
});
app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
var id = req.body.id;
var type = req.body.type;
var uuid = req.body.uuid;
var sub = req.body.sub;
var user_uid = null;
const id = req.body.id;
const type = req.body.type;
const uuid = req.body.uuid;
const sub = req.body.sub;
let user_uid = null;
if (req.isAuthenticated()) user_uid = req.user.uid;
@@ -1009,12 +1009,12 @@ app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
});
app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
var id = req.body.id;
var type = req.body.type;
var vodId = req.body.vodId;
var uuid = req.body.uuid;
var sub = req.body.sub;
var user_uid = null;
const id = req.body.id;
const type = req.body.type;
const vodId = req.body.vodId;
const uuid = req.body.uuid;
const sub = req.body.sub;
let user_uid = null;
if (req.isAuthenticated()) user_uid = req.user.uid;
@@ -1025,17 +1025,47 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
return;
}
logger.info(`Downloading Twitch chat for ${id}`);
const full_chat = await twitch_api.downloadTwitchChatByVODID(vodId, id, type, user_uid, sub);
logger.info(`Finished downloading Twitch chat for ${id}`);
res.send({
chat: full_chat
});
});
app.post('/api/getTwitchEmotes', async (req, res) => {
const uid = req.body.uid;
const file = await files_api.getVideo(uid);
const channel_name = file['uploader'];
const emotes_path = path.join('appdata', 'emotes', uid, 'emotes.json')
if (!fs.existsSync(emotes_path)) {
logger.info(`Downloading Twitch emotes for ${channel_name}`);
await twitch_api.downloadTwitchEmotes(channel_name, file.uid);
logger.info(`Finished downloading Twitch emotes for ${channel_name}`);
}
const emotes = await twitch_api.getTwitchEmotes(file.uid);
res.send({
emotes: emotes
});
});
app.get('/api/emote/:uid/:id', async (req, res) => {
const file_uid = decodeURIComponent(req.params.uid);
const emote_id = decodeURIComponent(req.params.id);
const emote_path = path.join('appdata', 'emotes', file_uid, emote_id);
if (fs.existsSync(emote_path)) path.isAbsolute(emote_path) ? res.sendFile(emote_path) : res.sendFile(path.join(__dirname, emote_path));
else res.sendStatus(404);
});
// video sharing
app.post('/api/enableSharing', optionalJwt, async (req, res) => {
var uid = req.body.uid;
var is_playlist = req.body.is_playlist;
const uid = req.body.uid;
const is_playlist = req.body.is_playlist;
let success = false;
// multi-user mode
if (req.isAuthenticated()) {
@@ -1643,6 +1673,8 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
}
if (!fs.existsSync(file_path)) {
logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
res.sendStatus(404);
return;
}
const stat = fs.statSync(file_path);
const fileSize = stat.size;

14
backend/package-lock.json generated
View File

@@ -11,7 +11,7 @@
"dependencies": {
"@discordjs/builders": "^1.6.1",
"@discordjs/core": "^0.5.2",
"@tzahi12345/twitch-emoticons": "^1.0.4",
"@tzahi12345/twitch-emoticons": "^1.0.9",
"archiver": "^5.3.1",
"async": "^3.2.3",
"async-mutex": "^0.4.0",
@@ -548,9 +548,9 @@
}
},
"node_modules/@tzahi12345/twitch-emoticons": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.4.tgz",
"integrity": "sha512-SeefOMIZsZJurUo0Qe3sS8TOqo6EtEoE5Cm1/REBv6MpD6jXmTjYdGKrfMkeKOyjyVv92A/CukL1PnkIUn3Qbg==",
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz",
"integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==",
"dependencies": {
"@twurple/api": "^6.2.1",
"@twurple/auth": "^6.2.1",
@@ -6087,9 +6087,9 @@
}
},
"@tzahi12345/twitch-emoticons": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.4.tgz",
"integrity": "sha512-SeefOMIZsZJurUo0Qe3sS8TOqo6EtEoE5Cm1/REBv6MpD6jXmTjYdGKrfMkeKOyjyVv92A/CukL1PnkIUn3Qbg==",
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz",
"integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==",
"requires": {
"@twurple/api": "^6.2.1",
"@twurple/auth": "^6.2.1",

2
backend/package.json
View File

@@ -25,7 +25,7 @@
"dependencies": {
"@discordjs/builders": "^1.6.1",
"@discordjs/core": "^0.5.2",
"@tzahi12345/twitch-emoticons": "^1.0.4",
"@tzahi12345/twitch-emoticons": "^1.0.9",
"archiver": "^5.3.1",
"async": "^3.2.3",
"async-mutex": "^0.4.0",

7
backend/test/tests.js
View File

@@ -579,8 +579,7 @@ describe('Twitch', async function () {
assert(true);
return;
}
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
const channel_id = await twitch_api.getChannelID(example_channel);
assert(channel_id === '494493142');
});
@@ -612,9 +611,7 @@ describe('Twitch', async function () {
assert(true);
return;
}
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
const emotesJSON = await twitch_api.downloadTwitchEmotes(channel_id, example_channel);
const emotesJSON = await twitch_api.downloadTwitchEmotes(example_channel, 'test_uid');
assert(emotesJSON && emotesJSON.length > 0);
});
});

79
backend/twitch.js
View File

@@ -11,6 +11,21 @@ const { promisify } = require('util');
const child_process = require('child_process');
const commandExistsSync = require('command-exists').sync;
let auth_timeout = null;
let cached_oauth = null;
exports.ensureTwitchAuth = async () => {
const TIMEOUT_MARGIN_MS = 60*1000;
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
if (cached_oauth && auth_timeout && (Date.now() - TIMEOUT_MARGIN_MS) < auth_timeout) return cached_oauth;
const {token, expires_in} = await exports.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
cached_oauth = token;
auth_timeout = Date.now() + expires_in;
return token;
}
exports.getCommentsForVOD = async (vodId) => {
const exec = promisify(child_process.exec);
@@ -111,47 +126,61 @@ exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, custo
return chat;
}
exports.downloadTwitchEmotes = async (channel_id, channel_name) => {
exports.getTwitchEmotes = async (file_uid) => {
const emotes_path = path.join('appdata', 'emotes', file_uid, 'emotes.json')
if (!fs.existsSync(emotes_path)) return null;
const emote_objs = fs.readJSONSync(emotes_path);
// inject custom url
for (const emote_obj of emote_objs) {
emote_obj.custom_url = `${utils.getBaseURL()}/api/emote/${file_uid}/${emote_obj.id}.${emote_obj.ext}`
}
return emote_objs;
}
exports.downloadTwitchEmotes = async (channel_name, file_uid) => {
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const channel_id = await exports.getChannelID(channel_name);
const fetcher = new EmoteFetcher(twitch_client_id, twitch_client_secret);
try {
await Promise.all([
await Promise.allSettled([
fetcher.fetchTwitchEmotes(),
fetcher.fetchTwitchEmotes(channel_id),
fetcher.fetchBTTVEmotes(),
fetcher.fetchBTTVEmotes(channel_id),
// fetcher.fetchSevenTVEmotes(),
// fetcher.fetchSevenTVEmotes(channel_id),
fetcher.fetchSevenTVEmotes(),
fetcher.fetchSevenTVEmotes(channel_id),
fetcher.fetchFFZEmotes(),
fetcher.fetchFFZEmotes(channel_id)
]);
const channel_dir = path.join('appdata', 'emotes', channel_id);
fs.ensureDirSync(channel_dir);
const emotesJSON = [];
const emotes_dir = path.join('appdata', 'emotes', file_uid);
const emote_json_path = path.join(emotes_dir, `emotes.json`);
fs.ensureDirSync(emotes_dir);
const emote_objs = [];
let failed_emote_count = 0;
for (const [, emote] of fetcher.emotes) {
const emoteJSON = emote.toJSON();
const emote_obj = emote.toObject();
const ext = emote.imageType;
const emote_path = path.join(channel_dir, `${emote.id}.${ext}`);
if (fs.existsSync(emote_path)) continue;
const emote_image_path = path.join(emotes_dir, `${emote.id}.${ext}`);
try {
const link = emote.toLink();
await utils.fetchFile(link, emote_path);
emotesJSON.push(emoteJSON);
if (!fs.existsSync(emote_image_path)) await utils.fetchFile(link, emote_image_path);
emote_obj['ext'] = ext;
emote_objs.push(emote_obj);
} catch (err) {
failed_emote_count++;
}
}
if (failed_emote_count) logger.warn(`${failed_emote_count} emotes failed to download for channel ${channel_name}`);
return emotesJSON;
await fs.writeJSON(emote_json_path, emote_objs);
return emote_objs;
} catch (err) {
logger.error(err);
return null;
@@ -159,12 +188,14 @@ exports.downloadTwitchEmotes = async (channel_id, channel_name) => {
}
exports.getTwitchOAuthToken = async (client_id, client_secret) => {
logger.verbose('Generating new Twitch auth token');
const url = `https://id.twitch.tv/oauth2/token`;
try {
const response = await axios.post(url, {client_id: client_id, client_secret: client_secret, grant_type: 'client_credentials'});
const token = response['data']['access_token'];
if (token) return token;
const expires_in = response['data']['expires_in'];
if (token) return {token, expires_in};
logger.error(`Failed to get token.`);
return null;
@@ -175,12 +206,14 @@ exports.getTwitchOAuthToken = async (client_id, client_secret) => {
}
}
exports.getChannelID = async (username, client_id, oauth_token) => {
const url = `https://api.twitch.tv/helix/users?login=${username}`;
exports.getChannelID = async (channel_name) => {
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const token = await exports.ensureTwitchAuth();
const url = `https://api.twitch.tv/helix/users?login=${channel_name}`;
const headers = {
'Client-ID': client_id,
'Authorization': 'Bearer ' + oauth_token,
Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8'
'Client-ID': twitch_client_id,
'Authorization': 'Bearer ' + token,
// Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8'
};
try {
@@ -192,11 +225,11 @@ exports.getChannelID = async (username, client_id, oauth_token) => {
return channelID;
}
logger.error(`Failed to get channel ID for user ${username}`);
logger.error(`Failed to get channel ID for user ${channel_name}`);
if (data.error) logger.error(data.error);
return null; // User not found
} catch (err) {
logger.error(`Failed to get channel ID for user ${username}`);
logger.error(`Failed to get channel ID for user ${channel_name}`);
logger.error(err);
}
}

14
package-lock.json generated
View File

@@ -23,7 +23,7 @@
"@angular/router": "^16.0.3",
"@fontsource/material-icons": "^4.5.4",
"@ngneat/content-loader": "^7.0.0",
"@tzahi12345/twitch-emoticons": "^1.0.3",
"@tzahi12345/twitch-emoticons": "^1.0.9",
"@videogular/ngx-videogular": "^6.0.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
@@ -5441,9 +5441,9 @@
}
},
"node_modules/@tzahi12345/twitch-emoticons": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.3.tgz",
"integrity": "sha512-V8FC+beYhAYn2Nmk66y2SzG+Lr9cNtklVp4cHSr+rUCIKJSFVjmO8cQd9shFRoQm0kpZX5X5+2TGhYFsUSr8/w==",
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz",
"integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==",
"dependencies": {
"@twurple/api": "^6.2.1",
"@twurple/auth": "^6.2.1",
@@ -21502,9 +21502,9 @@
}
},
"@tzahi12345/twitch-emoticons": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.3.tgz",
"integrity": "sha512-V8FC+beYhAYn2Nmk66y2SzG+Lr9cNtklVp4cHSr+rUCIKJSFVjmO8cQd9shFRoQm0kpZX5X5+2TGhYFsUSr8/w==",
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/@tzahi12345/twitch-emoticons/-/twitch-emoticons-1.0.9.tgz",
"integrity": "sha512-VVlBbha90WfFN8VUkQQ42XxBs1wbxfvq0jogPlLT7KkB6mxZUiRjutFLA59uue1PbYZj07tZs0vh9Tz5derHbA==",
"requires": {
"@twurple/api": "^6.2.1",
"@twurple/auth": "^6.2.1",

4
package.json
View File

@@ -35,7 +35,7 @@
"@angular/router": "^16.0.3",
"@fontsource/material-icons": "^4.5.4",
"@ngneat/content-loader": "^7.0.0",
"@tzahi12345/twitch-emoticons": "^1.0.3",
"@tzahi12345/twitch-emoticons": "^1.0.9",
"@videogular/ngx-videogular": "^6.0.0",
"core-js": "^2.4.1",
"crypto-js": "^4.1.1",
@@ -80,4 +80,4 @@
"ts-node": "~3.0.4",
"tslint": "~6.1.0"
}
}
}

16
src/app/components/twitch-chat/twitch-chat.component.ts
View File

@@ -1,7 +1,7 @@
import { Component, ElementRef, Input, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
import { DatabaseFile } from 'api-types';
import { PostsService } from 'app/posts.services';
import { EmoteFetcher, EmoteJSON, EmoteParser } from '@tzahi12345/twitch-emoticons';
import { EmoteFetcher, EmoteObject, EmoteParser } from '@tzahi12345/twitch-emoticons';
@Component({
selector: 'app-twitch-chat',
@@ -14,6 +14,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
visible_chat = null;
chat_response_received = false;
downloading_chat = false;
got_emotes = false;
current_chat_index = null;
@@ -36,6 +37,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
ngOnInit(): void {
this.getFullChat();
this.getEmotes();
}
ngOnDestroy(): void {
@@ -75,7 +77,7 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
for (let i = this.current_chat_index + 1; i < this.full_chat.length; i++) {
const new_chat = this.full_chat[i];
if (new_chat['timestamp'] >= latest_chat_timestamp && new_chat['timestamp'] <= this.current_timestamp) {
new_chat['message'] = this.parseChat(new_chat['message']);
new_chat['message'] = this.got_emotes ? this.parseChat(new_chat['message']) : new_chat['message'];
this.visible_chat.push(new_chat);
this.current_chat_index = i;
} else if (new_chat['timestamp'] > this.current_timestamp) {
@@ -125,22 +127,22 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
}
getEmotes() {
this.postsService.getTwitchEmotes().subscribe(res => {
this.postsService.getTwitchEmotes(this.db_file['uid']).subscribe(res => {
const emotes = res['emotes'];
this.processEmotes(emotes);
});
}
processEmotes(emotes: EmoteJSON[]) {
processEmotes(emotes: EmoteObject[]) {
this.fetcher = new EmoteFetcher();
this.parser = new EmoteParser(this.fetcher, {
// Custom HTML format
template: '<img class="emote" alt="{name}" src="{link}">',
template: `<img class="emote" alt="{name}" src="{link}">`,
// Match without :colons:
match: /(\w+)+?/g
});
this.fetcher.fromJSON(emotes);
this.fetcher.fromObject(emotes);
this.got_emotes = true;
}
parseChat(chat_message: string) {

6
src/app/posts.services.ts
View File

@@ -118,7 +118,7 @@ import {
import { isoLangs } from './dialogs/user-profile-dialog/locales_list';
import { Title } from '@angular/platform-browser';
import { MatDrawerMode } from '@angular/material/sidenav';
import type { EmoteJSON } from '@tzahi12345/twitch-emoticons';
import type { EmoteObject } from '@tzahi12345/twitch-emoticons';
@Injectable()
export class PostsService {
@@ -408,8 +408,8 @@ export class PostsService {
return this.http.post<DownloadTwitchChatByVODIDResponse>(this.path + 'downloadTwitchChatByVODID', body, this.httpOptions);
}
getTwitchEmotes() {
return this.http.post<{emotes: EmoteJSON[]}>(this.path + 'getTwitchEmotes', {}, this.httpOptions);
getTwitchEmotes(uid: string) {
return this.http.post<{emotes: EmoteObject[]}>(this.path + 'getTwitchEmotes', {uid: uid}, this.httpOptions);
}
downloadPlaylistFromServer(playlist_id, uuid = null) {