TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 20:10:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added ability to download twitch emotes in the backend

Browse Source
This commit is contained in:
Tzahi12345
2023-05-29 23:41:30 -04:00
parent 7124792721
commit 3a20e03490
5 changed files with 6157 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6074
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
backend/package.json
View File

@@ -25,7 +25,7 @@
"dependencies": {
"@discordjs/builders": "^1.6.1",
"@discordjs/core": "^0.5.2",
"@tzahi12345/twitch-emoticons": "^1.0.3",
"@tzahi12345/twitch-emoticons": "^1.0.4",
"archiver": "^5.3.1",
"async": "^3.2.3",
"async-mutex": "^0.4.0",

85
backend/test/tests.js
View File

@@ -550,27 +550,72 @@ describe('Downloader', function() {
const expected_args3 = ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1710641401';
it('Download VOD chat', async function() {
this.timeout(300000);
if (!fs.existsSync('TwitchDownloaderCLI')) {
try {
await exec('sh ../docker-utils/fetch-twitchdownloader.sh');
fs.copyFileSync('../docker-utils/TwitchDownloaderCLI', 'TwitchDownloaderCLI');
} catch (e) {
logger.info('TwitchDownloaderCLI fetch failed, file may exist regardless.');
}
}
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
assert(fs.existsSync(sample_path));
});
// cleanup
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1710641401';
const example_channel = 'keffals';
it('Get OAuth Token', async function() {
this.timeout(300000);
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
if (!twitch_client_id || !twitch_client_secret) {
logger.info(`Skipping test 'Get OAuth Token' as Twitch client ID or Twitch client secret is missing.`);
assert(true);
return;
}
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
assert(token);
});
it('Get channel ID', async function() {
this.timeout(300000);
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
if (!twitch_client_id || !twitch_client_secret) {
logger.info(`Skipping test 'Get channel ID' as Twitch client ID or Twitch client secret is missing.`);
assert(true);
return;
}
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
assert(channel_id === '494493142');
});
it('Download VOD chat', async function() {
this.timeout(300000);
if (!fs.existsSync('TwitchDownloaderCLI')) {
try {
await exec('sh ../docker-utils/fetch-twitchdownloader.sh');
fs.copyFileSync('../docker-utils/TwitchDownloaderCLI', 'TwitchDownloaderCLI');
} catch (e) {
logger.info('TwitchDownloaderCLI fetch failed, file may exist regardless.');
}
}
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
assert(fs.existsSync(sample_path));
// cleanup
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
});
it('Download Twitch emotes', async function() {
this.timeout(300000);
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
if (!twitch_client_id || !twitch_client_secret) {
logger.info(`Skipping test 'Download Twitch emotes' as Twitch client ID or Twitch client secret is missing.`);
assert(true);
return;
}
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
const channel_id = await twitch_api.getChannelID(example_channel, twitch_client_id, token);
const emotesJSON = await twitch_api.downloadTwitchEmotes(channel_id, example_channel);
assert(emotesJSON && emotesJSON.length > 0);
});
});

93
backend/twitch.js
View File

@@ -1,8 +1,11 @@
const config_api = require('./config');
const logger = require('./logger');
const utils = require('./utils');
const moment = require('moment');
const fs = require('fs-extra')
const axios = require('axios');
const { EmoteFetcher } = require('@tzahi12345/twitch-emoticons');
const path = require('path');
const { promisify } = require('util');
const child_process = require('child_process');
@@ -108,6 +111,96 @@ exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, custo
return chat;
}
exports.downloadTwitchEmotes = async (channel_id, channel_name) => {
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const fetcher = new EmoteFetcher(twitch_client_id, twitch_client_secret);
try {
await Promise.all([
fetcher.fetchTwitchEmotes(),
fetcher.fetchTwitchEmotes(channel_id),
fetcher.fetchBTTVEmotes(),
fetcher.fetchBTTVEmotes(channel_id),
// fetcher.fetchSevenTVEmotes(),
// fetcher.fetchSevenTVEmotes(channel_id),
fetcher.fetchFFZEmotes(),
fetcher.fetchFFZEmotes(channel_id)
]);
const channel_dir = path.join('appdata', 'emotes', channel_id);
fs.ensureDirSync(channel_dir);
const emotesJSON = [];
let failed_emote_count = 0;
for (const [, emote] of fetcher.emotes) {
const emoteJSON = emote.toJSON();
const ext = emote.imageType;
const emote_path = path.join(channel_dir, `${emote.id}.${ext}`);
if (fs.existsSync(emote_path)) continue;
try {
const link = emote.toLink();
await utils.fetchFile(link, emote_path);
emotesJSON.push(emoteJSON);
} catch (err) {
failed_emote_count++;
}
}
if (failed_emote_count) logger.warn(`${failed_emote_count} emotes failed to download for channel ${channel_name}`);
return emotesJSON;
} catch (err) {
logger.error(err);
return null;
}
}
exports.getTwitchOAuthToken = async (client_id, client_secret) => {
const url = `https://id.twitch.tv/oauth2/token`;
try {
const response = await axios.post(url, {client_id: client_id, client_secret: client_secret, grant_type: 'client_credentials'});
const token = response['data']['access_token'];
if (token) return token;
logger.error(`Failed to get token.`);
return null;
} catch (err) {
logger.error(`Failed to get token.`);
logger.error(err);
return null;
}
}
exports.getChannelID = async (username, client_id, oauth_token) => {
const url = `https://api.twitch.tv/helix/users?login=${username}`;
const headers = {
'Client-ID': client_id,
'Authorization': 'Bearer ' + oauth_token,
Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8'
};
try {
const response = await axios.get(url, {headers: headers});
const data = response.data.data;
if (data && data.length > 0) {
const channelID = data[0].id;
return channelID;
}
logger.error(`Failed to get channel ID for user ${username}`);
if (data.error) logger.error(data.error);
return null; // User not found
} catch (err) {
logger.error(`Failed to get channel ID for user ${username}`);
logger.error(err);
}
}
const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
.toISOString()
.replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,

23
backend/utils.js
View File

@@ -364,26 +364,29 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
}
// helper function to download file using fetch
exports.fetchFile = async (url, path, file_label) => {
exports.fetchFile = async (url, output_path, file_label = null) => {
var len = null;
const res = await fetch(url);
let bar = null;
if (file_label) {
len = parseInt(res.headers.get("Content-Length"), 10);
len = parseInt(res.headers.get("Content-Length"), 10);
bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
}
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
const fileStream = fs.createWriteStream(output_path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
if (file_label) bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();