mirror of
https://github.com/Tzahi12345/YoutubeDL-Material.git
synced 2026-03-07 20:10:03 +03:00
Compare commits
2 Commits
twitch-emo
...
subscripti
| Author | SHA1 | Date | |
|---|---|---|---|
|
cfb08a32fe | ||
|
|
b20586f892 |
40
.github/workflows/mocha.yml
vendored
40
.github/workflows/mocha.yml
vendored
@@ -1,40 +0,0 @@
|
||||
name: Tests
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: 'Backend - mocha'
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
node:
|
||||
- 16
|
||||
steps:
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: '${{ matrix.node }}'
|
||||
- uses: actions/checkout@v2
|
||||
- name: 'Cache node_modules'
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: ${{ runner.os }}-node-v${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-v${{ matrix.node }}-
|
||||
working-directory: ./backend
|
||||
- uses: FedericoCarboni/setup-ffmpeg@v2
|
||||
id: setup-ffmpeg
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
working-directory: ./backend
|
||||
- name: Run All Node.js Tests
|
||||
run: npm run test
|
||||
working-directory: ./backend
|
||||
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@@ -3,6 +3,6 @@
|
||||
"mochaExplorer.cwd": "backend",
|
||||
"mochaExplorer.globImplementation": "vscode",
|
||||
"mochaExplorer.env": {
|
||||
// "YTDL_MODE": "debug"
|
||||
"YTDL_MODE": "debug"
|
||||
}
|
||||
}
|
||||
@@ -706,7 +706,7 @@ app.use(function(req, res, next) {
|
||||
next();
|
||||
} else if (req.query.apiKey && config_api.getConfigItem('ytdl_use_api_key') && req.query.apiKey === config_api.getConfigItem('ytdl_api_key')) {
|
||||
next();
|
||||
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/emote/') || req.path.includes('/api/rss')) {
|
||||
} else if (req.path.includes('/api/stream/') || req.path.includes('/api/thumbnail/') || req.path.includes('/api/rss')) {
|
||||
next();
|
||||
} else {
|
||||
logger.verbose(`Rejecting request - invalid API use for endpoint: ${req.path}. API key received: ${req.query.apiKey}`);
|
||||
@@ -993,11 +993,11 @@ app.post('/api/updateConcurrentStream', optionalJwt, async (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
|
||||
const id = req.body.id;
|
||||
const type = req.body.type;
|
||||
const uuid = req.body.uuid;
|
||||
const sub = req.body.sub;
|
||||
let user_uid = null;
|
||||
var id = req.body.id;
|
||||
var type = req.body.type;
|
||||
var uuid = req.body.uuid;
|
||||
var sub = req.body.sub;
|
||||
var user_uid = null;
|
||||
|
||||
if (req.isAuthenticated()) user_uid = req.user.uid;
|
||||
|
||||
@@ -1009,12 +1009,12 @@ app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
|
||||
const id = req.body.id;
|
||||
const type = req.body.type;
|
||||
const vodId = req.body.vodId;
|
||||
const uuid = req.body.uuid;
|
||||
const sub = req.body.sub;
|
||||
let user_uid = null;
|
||||
var id = req.body.id;
|
||||
var type = req.body.type;
|
||||
var vodId = req.body.vodId;
|
||||
var uuid = req.body.uuid;
|
||||
var sub = req.body.sub;
|
||||
var user_uid = null;
|
||||
|
||||
if (req.isAuthenticated()) user_uid = req.user.uid;
|
||||
|
||||
@@ -1025,47 +1025,17 @@ app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Downloading Twitch chat for ${id}`);
|
||||
const full_chat = await twitch_api.downloadTwitchChatByVODID(vodId, id, type, user_uid, sub);
|
||||
logger.info(`Finished downloading Twitch chat for ${id}`);
|
||||
|
||||
res.send({
|
||||
chat: full_chat
|
||||
});
|
||||
});
|
||||
|
||||
app.post('/api/getTwitchEmotes', async (req, res) => {
|
||||
const uid = req.body.uid;
|
||||
|
||||
const file = await files_api.getVideo(uid);
|
||||
const channel_name = file['uploader'];
|
||||
|
||||
const emotes_path = path.join('appdata', 'emotes', uid, 'emotes.json')
|
||||
if (!fs.existsSync(emotes_path)) {
|
||||
logger.info(`Downloading Twitch emotes for ${channel_name}`);
|
||||
await twitch_api.downloadTwitchEmotes(channel_name, file.uid);
|
||||
logger.info(`Finished downloading Twitch emotes for ${channel_name}`);
|
||||
}
|
||||
|
||||
const emotes = await twitch_api.getTwitchEmotes(file.uid);
|
||||
|
||||
res.send({
|
||||
emotes: emotes
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/emote/:uid/:id', async (req, res) => {
|
||||
const file_uid = decodeURIComponent(req.params.uid);
|
||||
const emote_id = decodeURIComponent(req.params.id);
|
||||
const emote_path = path.join('appdata', 'emotes', file_uid, emote_id);
|
||||
if (fs.existsSync(emote_path)) path.isAbsolute(emote_path) ? res.sendFile(emote_path) : res.sendFile(path.join(__dirname, emote_path));
|
||||
else res.sendStatus(404);
|
||||
});
|
||||
|
||||
// video sharing
|
||||
app.post('/api/enableSharing', optionalJwt, async (req, res) => {
|
||||
const uid = req.body.uid;
|
||||
const is_playlist = req.body.is_playlist;
|
||||
var uid = req.body.uid;
|
||||
var is_playlist = req.body.is_playlist;
|
||||
let success = false;
|
||||
// multi-user mode
|
||||
if (req.isAuthenticated()) {
|
||||
@@ -1673,8 +1643,6 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
|
||||
}
|
||||
if (!fs.existsSync(file_path)) {
|
||||
logger.error(`File ${file_path} could not be found! UID: ${uid}, ID: ${file_obj && file_obj.id}`);
|
||||
res.sendStatus(404);
|
||||
return;
|
||||
}
|
||||
const stat = fs.statSync(file_path);
|
||||
const fileSize = stat.size;
|
||||
|
||||
@@ -208,9 +208,6 @@ const DEFAULT_CONFIG = {
|
||||
"API_key": "",
|
||||
"use_youtube_API": false,
|
||||
"youtube_API_key": "",
|
||||
"use_twitch_API": false,
|
||||
"twitch_client_ID": "",
|
||||
"twitch_client_secret": "",
|
||||
"twitch_auto_download_chat": false,
|
||||
"use_sponsorblock_API": false,
|
||||
"generate_NFO_files": false,
|
||||
|
||||
@@ -110,18 +110,6 @@ exports.CONFIG_ITEMS = {
|
||||
'key': 'ytdl_youtube_api_key',
|
||||
'path': 'YoutubeDLMaterial.API.youtube_API_key'
|
||||
},
|
||||
'ytdl_use_twitch_api': {
|
||||
'key': 'ytdl_use_twitch_api',
|
||||
'path': 'YoutubeDLMaterial.API.use_twitch_API'
|
||||
},
|
||||
'ytdl_twitch_client_id': {
|
||||
'key': 'ytdl_twitch_client_id',
|
||||
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
|
||||
},
|
||||
'ytdl_twitch_client_secret': {
|
||||
'key': 'ytdl_twitch_client_secret',
|
||||
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
|
||||
},
|
||||
'ytdl_twitch_auto_download_chat': {
|
||||
'key': 'ytdl_twitch_auto_download_chat',
|
||||
'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat'
|
||||
@@ -366,4 +354,4 @@ exports.YTDL_ARGS_WITH_VALUES = new Set(YTDL_ARGS_WITH_VALUES);
|
||||
|
||||
exports.ICON_URL = 'https://i.imgur.com/IKOlr0N.png';
|
||||
|
||||
exports.CURRENT_VERSION = 'v4.3.2';
|
||||
exports.CURRENT_VERSION = 'v4.3.1';
|
||||
|
||||
@@ -11,8 +11,9 @@ const logger = require('./logger');
|
||||
const low = require('lowdb')
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
const { BehaviorSubject } = require('rxjs');
|
||||
const local_adapter = new FileSync('./appdata/local_db.json');
|
||||
const local_db = low(local_adapter);
|
||||
|
||||
let local_db = null;
|
||||
let database = null;
|
||||
exports.database_initialized = false;
|
||||
exports.database_initialized_bs = new BehaviorSubject(false);
|
||||
@@ -72,6 +73,10 @@ const tables = {
|
||||
|
||||
const tables_list = Object.keys(tables);
|
||||
|
||||
const local_db_defaults = {}
|
||||
tables_list.forEach(table => {local_db_defaults[table] = []});
|
||||
local_db.defaults(local_db_defaults).write();
|
||||
|
||||
let using_local_db = null;
|
||||
|
||||
function setDB(input_db, input_users_db) {
|
||||
@@ -80,18 +85,11 @@ function setDB(input_db, input_users_db) {
|
||||
exports.users_db = input_users_db
|
||||
}
|
||||
|
||||
exports.initialize = (input_db, input_users_db, db_name = 'local_db.json') => {
|
||||
exports.initialize = (input_db, input_users_db) => {
|
||||
setDB(input_db, input_users_db);
|
||||
|
||||
// must be done here to prevent getConfigItem from being called before init
|
||||
using_local_db = config_api.getConfigItem('ytdl_use_local_db');
|
||||
|
||||
const local_adapter = new FileSync(`./appdata/${db_name}`);
|
||||
local_db = low(local_adapter);
|
||||
|
||||
const local_db_defaults = {}
|
||||
tables_list.forEach(table => {local_db_defaults[table] = []});
|
||||
local_db.defaults(local_db_defaults).write();
|
||||
}
|
||||
|
||||
exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {
|
||||
@@ -815,9 +813,6 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
|
||||
filtered &= record[filter_prop] === undefined || record[filter_prop] === null;
|
||||
} else {
|
||||
if (typeof filter_prop_value === 'object') {
|
||||
if (!record[filter_prop]) {
|
||||
continue;
|
||||
}
|
||||
if ('$regex' in filter_prop_value) {
|
||||
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
|
||||
} else if ('$ne' in filter_prop_value) {
|
||||
@@ -833,14 +828,10 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
|
||||
}
|
||||
} else {
|
||||
// handle case of nested property check
|
||||
if (filter_prop.includes('.')) {
|
||||
if (filter_prop.includes('.'))
|
||||
filtered &= utils.searchObjectByString(record, filter_prop) === filter_prop_value;
|
||||
} else {
|
||||
if (!record[filter_prop]) {
|
||||
continue;
|
||||
}
|
||||
else
|
||||
filtered &= record[filter_prop] === filter_prop_value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ const Mutex = require('async-mutex').Mutex;
|
||||
const youtubedl = require('youtube-dl');
|
||||
|
||||
const logger = require('./logger');
|
||||
const youtubedl_api = require('./youtube-dl');
|
||||
const config_api = require('./config');
|
||||
const twitch_api = require('./twitch');
|
||||
const { create } = require('xmlbuilder2');
|
||||
@@ -22,10 +21,10 @@ const mutex = new Mutex();
|
||||
let should_check_downloads = true;
|
||||
|
||||
if (db_api.database_initialized) {
|
||||
exports.setupDownloads();
|
||||
setupDownloads();
|
||||
} else {
|
||||
db_api.database_initialized_bs.subscribe(init => {
|
||||
if (init) exports.setupDownloads();
|
||||
if (init) setupDownloads();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -48,7 +47,7 @@ We use checkDownloads() to move downloads through the steps and call their respe
|
||||
|
||||
*/
|
||||
|
||||
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null, paused = false) => {
|
||||
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
|
||||
return await mutex.runExclusive(async () => {
|
||||
const download = {
|
||||
url: url,
|
||||
@@ -61,7 +60,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
|
||||
options: options,
|
||||
uid: uuid(),
|
||||
step_index: 0,
|
||||
paused: paused,
|
||||
paused: false,
|
||||
running: false,
|
||||
finished_step: true,
|
||||
error: null,
|
||||
@@ -134,7 +133,7 @@ async function handleDownloadError(download, error_message, error_type = null) {
|
||||
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
|
||||
}
|
||||
|
||||
exports.setupDownloads = async () => {
|
||||
async function setupDownloads() {
|
||||
await fixDownloadState();
|
||||
setInterval(checkDownloads, 1000);
|
||||
}
|
||||
@@ -189,7 +188,7 @@ async function checkDownloads() {
|
||||
if (waiting_download['step_index'] === 0) {
|
||||
collectInfo(waiting_download['uid']);
|
||||
} else if (waiting_download['step_index'] === 1) {
|
||||
exports.downloadQueuedFile(waiting_download['uid']);
|
||||
downloadQueuedFile(waiting_download['uid']);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,7 +240,7 @@ async function collectInfo(download_uid) {
|
||||
let category = null;
|
||||
|
||||
// check if it fits into a category. If so, then get info again using new args
|
||||
if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
||||
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
||||
|
||||
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
|
||||
if (category && category['custom_output']) {
|
||||
@@ -260,9 +259,13 @@ async function collectInfo(download_uid) {
|
||||
const files_to_check_for_progress = [];
|
||||
|
||||
// store info in download for future use
|
||||
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
|
||||
if (Array.isArray(info)) {
|
||||
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
|
||||
} else {
|
||||
files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
|
||||
}
|
||||
|
||||
const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null;
|
||||
const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
|
||||
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
|
||||
finished_step: true,
|
||||
running: false,
|
||||
@@ -275,7 +278,7 @@ async function collectInfo(download_uid) {
|
||||
});
|
||||
}
|
||||
|
||||
exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => {
|
||||
async function downloadQueuedFile(download_uid) {
|
||||
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
||||
if (download['paused']) {
|
||||
return;
|
||||
@@ -303,102 +306,114 @@ exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec
|
||||
const start_time = Date.now();
|
||||
|
||||
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
|
||||
const file_objs = [];
|
||||
|
||||
// download file
|
||||
const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
|
||||
clearInterval(download_checker);
|
||||
let end_time = Date.now();
|
||||
let difference = (end_time - start_time)/1000;
|
||||
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
|
||||
if (!parsed_output) {
|
||||
logger.error(err.stderr);
|
||||
await handleDownloadError(download, err.stderr, 'unknown_error');
|
||||
resolve(false);
|
||||
return;
|
||||
} else if (parsed_output) {
|
||||
if (parsed_output.length === 0 || parsed_output[0].length === 0) {
|
||||
// ERROR!
|
||||
const error_message = `No output received for video download, check if it exists in your archive.`;
|
||||
await handleDownloadError(download, error_message, 'no_output');
|
||||
logger.warn(error_message);
|
||||
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
|
||||
const file_objs = [];
|
||||
let end_time = Date.now();
|
||||
let difference = (end_time - start_time)/1000;
|
||||
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
|
||||
clearInterval(download_checker);
|
||||
if (err) {
|
||||
logger.error(err.stderr);
|
||||
await handleDownloadError(download, err.stderr, 'unknown_error');
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const output_json of parsed_output) {
|
||||
if (!output_json) {
|
||||
continue;
|
||||
} else if (output) {
|
||||
if (output.length === 0 || output[0].length === 0) {
|
||||
// ERROR!
|
||||
const error_message = `No output received for video download, check if it exists in your archive.`;
|
||||
await handleDownloadError(download, error_message, 'no_output');
|
||||
logger.warn(error_message);
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// get filepath with no extension
|
||||
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
|
||||
|
||||
const ext = type === 'audio' ? '.mp3' : '.mp4';
|
||||
var full_file_path = filepath_no_extension + ext;
|
||||
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
|
||||
|
||||
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
|
||||
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
|
||||
let vodId = url.split('twitch.tv/videos/')[1];
|
||||
vodId = vodId.split('?')[0];
|
||||
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
|
||||
}
|
||||
|
||||
// renames file if necessary due to bug
|
||||
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
let output_json = null;
|
||||
try {
|
||||
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
|
||||
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
|
||||
// we have to do this because sometimes there will be leading characters before the actual json
|
||||
const start_idx = output[i].indexOf('{"');
|
||||
const clean_output = output[i].slice(start_idx, output[i].length);
|
||||
output_json = JSON.parse(clean_output);
|
||||
} catch(e) {
|
||||
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
|
||||
output_json = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'audio') {
|
||||
let tags = {
|
||||
title: output_json['title'],
|
||||
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
|
||||
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
|
||||
|
||||
// get filepath with no extension
|
||||
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
|
||||
|
||||
const ext = type === 'audio' ? '.mp3' : '.mp4';
|
||||
var full_file_path = filepath_no_extension + ext;
|
||||
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
|
||||
|
||||
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
|
||||
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
|
||||
let vodId = url.split('twitch.tv/videos/')[1];
|
||||
vodId = vodId.split('?')[0];
|
||||
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
|
||||
}
|
||||
|
||||
// renames file if necessary due to bug
|
||||
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
|
||||
try {
|
||||
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
|
||||
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
|
||||
} catch(e) {
|
||||
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'audio') {
|
||||
let tags = {
|
||||
title: output_json['title'],
|
||||
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
|
||||
}
|
||||
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
|
||||
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
|
||||
}
|
||||
|
||||
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
|
||||
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
|
||||
}
|
||||
|
||||
if (options.cropFileSettings) {
|
||||
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
|
||||
}
|
||||
|
||||
// registers file in DB
|
||||
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
|
||||
|
||||
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
|
||||
|
||||
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
|
||||
|
||||
file_objs.push(file_obj);
|
||||
}
|
||||
|
||||
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
|
||||
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
|
||||
let container = null;
|
||||
|
||||
if (file_objs.length > 1) {
|
||||
// create playlist
|
||||
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
|
||||
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
|
||||
} else if (file_objs.length === 1) {
|
||||
container = file_objs[0];
|
||||
} else {
|
||||
const error_message = 'Downloaded file failed to result in metadata object.';
|
||||
logger.error(error_message);
|
||||
await handleDownloadError(download, error_message, 'no_metadata');
|
||||
}
|
||||
|
||||
if (options.cropFileSettings) {
|
||||
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
|
||||
}
|
||||
|
||||
// registers file in DB
|
||||
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
|
||||
|
||||
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
|
||||
|
||||
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
|
||||
|
||||
file_objs.push(file_obj);
|
||||
const file_uids = file_objs.map(file_obj => file_obj.uid);
|
||||
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
|
||||
resolve();
|
||||
}
|
||||
|
||||
let container = null;
|
||||
|
||||
if (file_objs.length > 1) {
|
||||
// create playlist
|
||||
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
|
||||
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
|
||||
} else if (file_objs.length === 1) {
|
||||
container = file_objs[0];
|
||||
} else {
|
||||
const error_message = 'Downloaded file failed to result in metadata object.';
|
||||
logger.error(error_message);
|
||||
await handleDownloadError(download, error_message, 'no_metadata');
|
||||
}
|
||||
|
||||
const file_uids = file_objs.map(file_obj => file_obj.uid);
|
||||
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
|
||||
resolve(file_uids);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -549,9 +564,33 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
|
||||
new_args.push('--dump-json');
|
||||
|
||||
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
|
||||
const parsed_output = utils.parseOutputJSON(output, err);
|
||||
if (parsed_output) {
|
||||
resolve(parsed_output);
|
||||
if (output) {
|
||||
let outputs = [];
|
||||
try {
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
let output_json = null;
|
||||
try {
|
||||
output_json = JSON.parse(output[i]);
|
||||
} catch(e) {
|
||||
output_json = null;
|
||||
}
|
||||
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
|
||||
outputs.push(output_json);
|
||||
}
|
||||
resolve(outputs.length === 1 ? outputs[0] : outputs);
|
||||
} catch(e) {
|
||||
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
|
||||
logger.error(error);
|
||||
if (download_uid) {
|
||||
const download = await db_api.getRecord('download_queue', {uid: download_uid});
|
||||
await handleDownloadError(download, error, 'parse_failed');
|
||||
}
|
||||
resolve(null);
|
||||
}
|
||||
} else {
|
||||
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
|
||||
if (err.stderr) error_message += `\n\n${err.stderr}`;
|
||||
|
||||
6074
backend/package-lock.json
generated
6074
backend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@
|
||||
"description": "backend for YoutubeDL-Material",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "mocha test --exit -s 1000",
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"start": "pm2-runtime --raw pm2.config.js",
|
||||
"debug": "set YTDL_MODE=debug && node app.js"
|
||||
},
|
||||
@@ -25,7 +25,6 @@
|
||||
"dependencies": {
|
||||
"@discordjs/builders": "^1.6.1",
|
||||
"@discordjs/core": "^0.5.2",
|
||||
"@tzahi12345/twitch-emoticons": "^1.0.9",
|
||||
"archiver": "^5.3.1",
|
||||
"async": "^3.2.3",
|
||||
"async-mutex": "^0.4.0",
|
||||
|
||||
@@ -254,15 +254,26 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
|
||||
}
|
||||
|
||||
logger.verbose('Subscription: finished check for ' + sub.name);
|
||||
const parsed_output = utils.parseOutputJSON(output, err);
|
||||
if (!parsed_output) {
|
||||
logger.error('Subscription check failed!');
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
const files_to_download = await handleOutputJSON(parsed_output, sub, user_uid);
|
||||
resolve(files_to_download);
|
||||
return;
|
||||
if (err && !output) {
|
||||
logger.error(err.stderr ? err.stderr : err.message);
|
||||
if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
|
||||
logger.info('An error was encountered with at least one video, backup method will be used.')
|
||||
try {
|
||||
const outputs = err.stdout.split(/\r\n|\r|\n/);
|
||||
const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
|
||||
resolve(files_to_download);
|
||||
} catch(e) {
|
||||
logger.error('Backup method failed. See error below:');
|
||||
logger.error(e);
|
||||
}
|
||||
} else {
|
||||
logger.error('Subscription check failed!');
|
||||
}
|
||||
resolve(false);
|
||||
} else if (output) {
|
||||
const files_to_download = await handleOutputJSON(output, sub, user_uid);
|
||||
resolve(files_to_download);
|
||||
}
|
||||
});
|
||||
}, err => {
|
||||
logger.error(err);
|
||||
@@ -270,24 +281,38 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
|
||||
});
|
||||
}
|
||||
|
||||
async function handleOutputJSON(output_jsons, sub, user_uid) {
|
||||
async function handleOutputJSON(output, sub, user_uid) {
|
||||
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
|
||||
await setFreshUploads(sub, user_uid);
|
||||
checkVideosForFreshUploads(sub, user_uid);
|
||||
}
|
||||
|
||||
if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) {
|
||||
if (output.length === 0 || (output.length === 1 && output[0] === '')) {
|
||||
logger.verbose('No additional videos to download for ' + sub.name);
|
||||
return [];
|
||||
}
|
||||
|
||||
const output_jsons = [];
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
let output_json = null;
|
||||
try {
|
||||
output_json = JSON.parse(output[i]);
|
||||
output_jsons.push(output_json);
|
||||
} catch(e) {
|
||||
output_json = null;
|
||||
}
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const files_to_download = await getFilesToDownload(sub, output_jsons);
|
||||
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
|
||||
|
||||
for (let j = 0; j < files_to_download.length; j++) {
|
||||
const file_to_download = files_to_download[j];
|
||||
file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']); // prevent download object from blowing up in size
|
||||
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, [file_to_download]);
|
||||
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
|
||||
}
|
||||
|
||||
return files_to_download;
|
||||
|
||||
1
backend/test/sample.info.json
Normal file
1
backend/test/sample.info.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -3,8 +3,7 @@ const assert = require('assert');
|
||||
const low = require('lowdb')
|
||||
const winston = require('winston');
|
||||
const path = require('path');
|
||||
const util = require('util');
|
||||
const exec = util.promisify(require('child_process').exec);
|
||||
|
||||
|
||||
const FileSync = require('lowdb/adapters/FileSync');
|
||||
|
||||
@@ -42,13 +41,11 @@ const subscriptions_api = require('../subscriptions');
|
||||
const archive_api = require('../archive');
|
||||
const categories_api = require('../categories');
|
||||
const files_api = require('../files');
|
||||
const youtubedl_api = require('../youtube-dl');
|
||||
const config_api = require('../config');
|
||||
const fs = require('fs-extra');
|
||||
const { uuid } = require('uuidv4');
|
||||
const NodeID3 = require('node-id3');
|
||||
|
||||
db_api.initialize(db, users_db, 'local_db_test.json');
|
||||
db_api.initialize(db, users_db);
|
||||
|
||||
const sample_video_json = {
|
||||
id: "Sample Video",
|
||||
@@ -71,9 +68,9 @@ const sample_video_json = {
|
||||
}
|
||||
|
||||
describe('Database', async function() {
|
||||
describe.skip('Import', async function() {
|
||||
describe('Import', async function() {
|
||||
// it('Migrate', async function() {
|
||||
// // await db_api.connectToDB();
|
||||
// await db_api.connectToDB();
|
||||
// await db_api.removeAllRecords();
|
||||
// const success = await db_api.importJSONToDB(db.value(), users_db.value());
|
||||
// assert(success);
|
||||
@@ -89,7 +86,7 @@ describe('Database', async function() {
|
||||
});
|
||||
|
||||
it('Transfer to local', async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('test');
|
||||
await db_api.insertRecordIntoTable('test', {test: 'test'});
|
||||
|
||||
@@ -117,8 +114,7 @@ describe('Database', async function() {
|
||||
|
||||
for (const local_db_mode of local_db_modes) {
|
||||
let use_local_db = local_db_mode;
|
||||
const describe_skippable = use_local_db ? describe : describe.skip;
|
||||
describe_skippable(`Use local DB - ${use_local_db}`, async function() {
|
||||
describe(`Use local DB - ${use_local_db}`, async function() {
|
||||
beforeEach(async function() {
|
||||
if (!use_local_db) {
|
||||
this.timeout(120000);
|
||||
@@ -171,7 +167,7 @@ describe('Database', async function() {
|
||||
];
|
||||
await db_api.insertRecordsIntoTable('test', test_duplicates);
|
||||
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
|
||||
assert(duplicates && duplicates.length === 2 && duplicates[0]['key'] === '2' && duplicates[1]['key'] === '4')
|
||||
console.log(duplicates);
|
||||
});
|
||||
|
||||
it('Update record', async function() {
|
||||
@@ -283,7 +279,7 @@ describe('Database', async function() {
|
||||
assert(stats);
|
||||
});
|
||||
|
||||
it.skip('Query speed', async function() {
|
||||
it('Query speed', async function() {
|
||||
this.timeout(120000);
|
||||
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
|
||||
const test_records = [];
|
||||
@@ -341,13 +337,12 @@ describe('Database', async function() {
|
||||
});
|
||||
|
||||
describe('Multi User', async function() {
|
||||
this.timeout(120000);
|
||||
const user_to_test = 'test_user';
|
||||
const user_password = 'test_pass';
|
||||
const sub_to_test = '';
|
||||
const playlist_to_test = '';
|
||||
beforeEach(async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.connectToDB();
|
||||
await auth_api.deleteUser(user_to_test);
|
||||
});
|
||||
describe('Basic', function() {
|
||||
@@ -374,17 +369,17 @@ describe('Multi User', async function() {
|
||||
|
||||
it('Video access - disallowed', async function() {
|
||||
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
|
||||
const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
|
||||
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
|
||||
assert(!video_obj);
|
||||
});
|
||||
|
||||
it('Video access - allowed', async function() {
|
||||
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
|
||||
const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
|
||||
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
|
||||
assert(video_obj);
|
||||
});
|
||||
});
|
||||
describe.skip('Zip generators', function() {
|
||||
describe('Zip generators', function() {
|
||||
it('Playlist zip generator', async function() {
|
||||
const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
|
||||
assert(playlist);
|
||||
@@ -440,50 +435,35 @@ describe('Multi User', async function() {
|
||||
|
||||
describe('Downloader', function() {
|
||||
const downloader_api = require('../downloader');
|
||||
const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
|
||||
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
|
||||
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
|
||||
const options = {
|
||||
ui_uid: uuid()
|
||||
ui_uid: uuid(),
|
||||
user: 'admin'
|
||||
}
|
||||
|
||||
before(async function() {
|
||||
const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
|
||||
if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
|
||||
config_api.setConfigItem('ytdl_max_concurrent_downloads', 0);
|
||||
});
|
||||
|
||||
beforeEach(async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('download_queue');
|
||||
});
|
||||
|
||||
it('Get file info', async function() {
|
||||
this.timeout(300000);
|
||||
const info = await downloader_api.getVideoInfoByURL(url);
|
||||
assert(!!info && info.length > 0);
|
||||
assert(!!info);
|
||||
});
|
||||
|
||||
it('Download file', async function() {
|
||||
this.timeout(300000);
|
||||
await downloader_api.setupDownloads();
|
||||
const args = await downloader_api.generateArgs(url, 'video', options, null, true);
|
||||
const [info] = await downloader_api.getVideoInfoByURL(url, args);
|
||||
if (fs.existsSync(info['_filename'])) fs.unlinkSync(info['_filename']);
|
||||
this.timeout(300000);
|
||||
const returned_download = await downloader_api.createDownload(url, 'video', options);
|
||||
assert(returned_download);
|
||||
const custom_download_method = async (url, args, options, callback) => {
|
||||
fs.writeJSONSync(utils.getTrueFileName(info['_filename'], 'video', '.info.json'), info);
|
||||
await generateEmptyVideoFile(info['_filename']);
|
||||
return await callback(null, [JSON.stringify(info)]);
|
||||
}
|
||||
const success = await downloader_api.downloadQueuedFile(returned_download['uid'], custom_download_method);
|
||||
assert(success);
|
||||
console.log(returned_download);
|
||||
await utils.wait(20000);
|
||||
|
||||
});
|
||||
|
||||
it('Tag file', async function() {
|
||||
const success = await generateEmptyAudioFile('test/sample_mp3.mp3');
|
||||
const audio_path = './test/sample_mp3.mp3';
|
||||
const sample_json = fs.readJSONSync('./test/sample_mp3.info.json');
|
||||
const audio_path = './test/sample.mp3';
|
||||
const sample_json = fs.readJSONSync('./test/sample.info.json');
|
||||
const tags = {
|
||||
title: sample_json['title'],
|
||||
artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
|
||||
@@ -491,13 +471,14 @@ describe('Downloader', function() {
|
||||
}
|
||||
NodeID3.write(tags, audio_path);
|
||||
const written_tags = NodeID3.read(audio_path);
|
||||
assert(success && written_tags['raw']['TRCK'] === '27');
|
||||
assert(written_tags['raw']['TRCK'] === '27');
|
||||
});
|
||||
|
||||
it('Queue file', async function() {
|
||||
this.timeout(300000);
|
||||
const returned_download = await downloader_api.createDownload(url, 'video', options, null, null, null, null, true);
|
||||
assert(returned_download);
|
||||
const returned_download = await downloader_api.createDownload(url, 'video', options);
|
||||
console.log(returned_download);
|
||||
await utils.wait(20000);
|
||||
});
|
||||
|
||||
it('Pause file', async function() {
|
||||
@@ -512,7 +493,7 @@ describe('Downloader', function() {
|
||||
assert(args.length > 0);
|
||||
});
|
||||
|
||||
it.skip('Generate args - subscription', async function() {
|
||||
it('Generate args - subscription', async function() {
|
||||
const sub = await subscriptions_api.getSubscription(sub_id);
|
||||
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
|
||||
const args_normal = await downloader_api.generateArgs(url, 'video', options);
|
||||
@@ -525,7 +506,7 @@ describe('Downloader', function() {
|
||||
if (fs.existsSync(nfo_file_path)) {
|
||||
fs.unlinkSync(nfo_file_path);
|
||||
}
|
||||
const sample_json = fs.readJSONSync('./test/sample_mp4.info.json');
|
||||
const sample_json = fs.readJSONSync('./test/sample.info.json');
|
||||
downloader_api.generateNFOFile(sample_json, nfo_file_path);
|
||||
assert(fs.existsSync(nfo_file_path), true);
|
||||
fs.unlinkSync(nfo_file_path);
|
||||
@@ -550,76 +531,26 @@ describe('Downloader', function() {
|
||||
const expected_args3 = ['-o', '%(title)s.%(ext)s', '--min-filesize', '1'];
|
||||
assert(JSON.stringify(updated_args3) === JSON.stringify(expected_args3));
|
||||
});
|
||||
});
|
||||
describe('Twitch', async function () {
|
||||
const twitch_api = require('../twitch');
|
||||
const example_vod = '1710641401';
|
||||
it('Download VOD', async function() {
|
||||
const sample_path = path.join('test', 'sample.twitch_chat.json');
|
||||
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
|
||||
this.timeout(300000);
|
||||
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
|
||||
assert(fs.existsSync(sample_path));
|
||||
|
||||
describe('Twitch', async function () {
|
||||
const twitch_api = require('../twitch');
|
||||
const example_vod = '1710641401';
|
||||
const example_channel = 'keffals';
|
||||
|
||||
it('Get OAuth Token', async function() {
|
||||
this.timeout(300000);
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
|
||||
if (!twitch_client_id || !twitch_client_secret) {
|
||||
logger.info(`Skipping test 'Get OAuth Token' as Twitch client ID or Twitch client secret is missing.`);
|
||||
assert(true);
|
||||
return;
|
||||
}
|
||||
const token = await twitch_api.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
|
||||
assert(token);
|
||||
});
|
||||
|
||||
it('Get channel ID', async function() {
|
||||
this.timeout(300000);
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
|
||||
if (!twitch_client_id || !twitch_client_secret) {
|
||||
logger.info(`Skipping test 'Get channel ID' as Twitch client ID or Twitch client secret is missing.`);
|
||||
assert(true);
|
||||
return;
|
||||
}
|
||||
const channel_id = await twitch_api.getChannelID(example_channel);
|
||||
assert(channel_id === '494493142');
|
||||
});
|
||||
|
||||
it('Download VOD chat', async function() {
|
||||
this.timeout(300000);
|
||||
if (!fs.existsSync('TwitchDownloaderCLI')) {
|
||||
try {
|
||||
await exec('sh ../docker-utils/fetch-twitchdownloader.sh');
|
||||
fs.copyFileSync('../docker-utils/TwitchDownloaderCLI', 'TwitchDownloaderCLI');
|
||||
} catch (e) {
|
||||
logger.info('TwitchDownloaderCLI fetch failed, file may exist regardless.');
|
||||
}
|
||||
}
|
||||
const sample_path = path.join('test', 'sample.twitch_chat.json');
|
||||
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
|
||||
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
|
||||
assert(fs.existsSync(sample_path));
|
||||
|
||||
// cleanup
|
||||
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
|
||||
});
|
||||
|
||||
it('Download Twitch emotes', async function() {
|
||||
this.timeout(300000);
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
|
||||
if (!twitch_client_id || !twitch_client_secret) {
|
||||
logger.info(`Skipping test 'Download Twitch emotes' as Twitch client ID or Twitch client secret is missing.`);
|
||||
assert(true);
|
||||
return;
|
||||
}
|
||||
const emotesJSON = await twitch_api.downloadTwitchEmotes(example_channel, 'test_uid');
|
||||
assert(emotesJSON && emotesJSON.length > 0);
|
||||
// cleanup
|
||||
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tasks', function() {
|
||||
const tasks_api = require('../tasks');
|
||||
beforeEach(async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('tasks');
|
||||
|
||||
const dummy_task = {
|
||||
@@ -638,7 +569,7 @@ describe('Tasks', function() {
|
||||
await tasks_api.executeTask('backup_local_db');
|
||||
const backups_new = await utils.recFindByExt('appdata', 'bak');
|
||||
const new_length = backups_new.length;
|
||||
assert(original_length === new_length-1);
|
||||
assert(original_length, new_length-1);
|
||||
});
|
||||
|
||||
it('Check for missing files', async function() {
|
||||
@@ -648,7 +579,7 @@ describe('Tasks', function() {
|
||||
await db_api.insertRecordIntoTable('files', test_missing_file);
|
||||
await tasks_api.executeTask('missing_files_check');
|
||||
const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
|
||||
assert(!missing_file_db_record);
|
||||
assert(!missing_file_db_record, true);
|
||||
});
|
||||
|
||||
it('Check for duplicate files', async function() {
|
||||
@@ -668,29 +599,27 @@ describe('Tasks', function() {
|
||||
|
||||
await tasks_api.executeTask('duplicate_files_check');
|
||||
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
|
||||
assert(duplicated_record_count === 1);
|
||||
assert(duplicated_record_count == 1, true);
|
||||
});
|
||||
|
||||
it('Import unregistered files', async function() {
|
||||
this.timeout(300000);
|
||||
|
||||
const success = await generateEmptyVideoFile('test/sample_mp4.mp4');
|
||||
|
||||
// pre-test cleanup
|
||||
await db_api.removeAllRecords('files', {title: 'Sample File'});
|
||||
if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
|
||||
if (fs.existsSync('video/sample_mp4.mp4')) fs.unlinkSync('video/sample_mp4.mp4');
|
||||
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
|
||||
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
|
||||
|
||||
// copies in files
|
||||
fs.copyFileSync('test/sample_mp4.info.json', 'video/sample_mp4.info.json');
|
||||
fs.copyFileSync('test/sample_mp4.mp4', 'video/sample_mp4.mp4');
|
||||
fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
|
||||
fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
|
||||
await tasks_api.executeTask('missing_db_records');
|
||||
const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
|
||||
assert(success && !!imported_file);
|
||||
assert(!!imported_file === true);
|
||||
|
||||
// post-test cleanup
|
||||
if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
|
||||
if (fs.existsSync('video/sample_mp4.mp4')) fs.unlinkSync('video/sample_mp4.mp4');
|
||||
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
|
||||
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
|
||||
});
|
||||
|
||||
it('Schedule and cancel task', async function() {
|
||||
@@ -730,12 +659,12 @@ describe('Tasks', function() {
|
||||
|
||||
describe('Archive', async function() {
|
||||
beforeEach(async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('archives');
|
||||
await db_api.connectToDB();
|
||||
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
|
||||
});
|
||||
|
||||
afterEach(async function() {
|
||||
await db_api.removeAllRecords('archives');
|
||||
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
|
||||
});
|
||||
|
||||
it('Import archive', async function() {
|
||||
@@ -749,6 +678,7 @@ describe('Archive', async function() {
|
||||
const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
|
||||
assert(count === 4)
|
||||
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
|
||||
console.log(archive_items);
|
||||
assert(archive_items.length === 4);
|
||||
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
|
||||
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
|
||||
@@ -779,9 +709,9 @@ describe('Archive', async function() {
|
||||
});
|
||||
|
||||
it('Remove from archive', async function() {
|
||||
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_title', 'test_user');
|
||||
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_title', 'test_user');
|
||||
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_title', 'test_user');
|
||||
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
|
||||
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
|
||||
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
|
||||
|
||||
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
|
||||
assert(success);
|
||||
@@ -827,7 +757,7 @@ describe('Utils', async function() {
|
||||
|
||||
describe('Categories', async function() {
|
||||
beforeEach(async function() {
|
||||
// await db_api.connectToDB();
|
||||
await db_api.connectToDB();
|
||||
const new_category = {
|
||||
name: 'test_category',
|
||||
uid: uuid(),
|
||||
@@ -875,6 +805,7 @@ describe('Categories', async function() {
|
||||
});
|
||||
|
||||
const category = await categories_api.categorize([sample_video_json]);
|
||||
console.log(category);
|
||||
assert(category && category.name === 'test_category');
|
||||
});
|
||||
|
||||
@@ -927,14 +858,4 @@ describe('Categories', async function() {
|
||||
const category = await categories_api.categorize([sample_video_json]);
|
||||
assert(category);
|
||||
});
|
||||
});
|
||||
|
||||
const generateEmptyVideoFile = async (file_path) => {
|
||||
if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
|
||||
return await exec(`ffmpeg -t 1 -f lavfi -i color=c=black:s=640x480 -c:v libx264 -tune stillimage -pix_fmt yuv420p "${file_path}"`);
|
||||
}
|
||||
|
||||
const generateEmptyAudioFile = async (file_path) => {
|
||||
if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
|
||||
return await exec(`ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 1 -q:a 9 -acodec libmp3lame ${file_path}`);
|
||||
}
|
||||
});
|
||||
@@ -1,32 +1,14 @@
|
||||
const config_api = require('./config');
|
||||
const logger = require('./logger');
|
||||
const utils = require('./utils');
|
||||
|
||||
const moment = require('moment');
|
||||
const fs = require('fs-extra')
|
||||
const axios = require('axios');
|
||||
const { EmoteFetcher } = require('@tzahi12345/twitch-emoticons');
|
||||
const path = require('path');
|
||||
const { promisify } = require('util');
|
||||
const child_process = require('child_process');
|
||||
const commandExistsSync = require('command-exists').sync;
|
||||
|
||||
let auth_timeout = null;
|
||||
let cached_oauth = null;
|
||||
|
||||
exports.ensureTwitchAuth = async () => {
|
||||
const TIMEOUT_MARGIN_MS = 60*1000;
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
|
||||
if (cached_oauth && auth_timeout && (Date.now() - TIMEOUT_MARGIN_MS) < auth_timeout) return cached_oauth;
|
||||
|
||||
const {token, expires_in} = await exports.getTwitchOAuthToken(twitch_client_id, twitch_client_secret);
|
||||
cached_oauth = token;
|
||||
auth_timeout = Date.now() + expires_in;
|
||||
return token;
|
||||
}
|
||||
|
||||
exports.getCommentsForVOD = async (vodId) => {
|
||||
async function getCommentsForVOD(vodId) {
|
||||
const exec = promisify(child_process.exec);
|
||||
|
||||
// Reject invalid params to prevent command injection attack
|
||||
@@ -70,7 +52,7 @@ exports.getCommentsForVOD = async (vodId) => {
|
||||
return new_json;
|
||||
}
|
||||
|
||||
exports.getTwitchChatByFileID = async (id, type, user_uid, uuid, sub) => {
|
||||
async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
|
||||
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
||||
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
||||
let file_path = null;
|
||||
@@ -98,10 +80,10 @@ exports.getTwitchChatByFileID = async (id, type, user_uid, uuid, sub) => {
|
||||
return chat_file;
|
||||
}
|
||||
|
||||
exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, customFileFolderPath = null) => {
|
||||
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
|
||||
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
||||
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
|
||||
const chat = await exports.getCommentsForVOD(vodId);
|
||||
const chat = await getCommentsForVOD(vodId);
|
||||
|
||||
// save file if needed params are included
|
||||
let file_path = null;
|
||||
@@ -126,114 +108,6 @@ exports.downloadTwitchChatByVODID = async (vodId, id, type, user_uid, sub, custo
|
||||
return chat;
|
||||
}
|
||||
|
||||
exports.getTwitchEmotes = async (file_uid) => {
|
||||
const emotes_path = path.join('appdata', 'emotes', file_uid, 'emotes.json')
|
||||
if (!fs.existsSync(emotes_path)) return null;
|
||||
const emote_objs = fs.readJSONSync(emotes_path);
|
||||
// inject custom url
|
||||
for (const emote_obj of emote_objs) {
|
||||
emote_obj.custom_url = `${utils.getBaseURL()}/api/emote/${file_uid}/${emote_obj.id}.${emote_obj.ext}`
|
||||
}
|
||||
return emote_objs;
|
||||
}
|
||||
|
||||
exports.downloadTwitchEmotes = async (channel_name, file_uid) => {
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
|
||||
|
||||
const channel_id = await exports.getChannelID(channel_name);
|
||||
|
||||
const fetcher = new EmoteFetcher(twitch_client_id, twitch_client_secret);
|
||||
|
||||
try {
|
||||
await Promise.allSettled([
|
||||
fetcher.fetchTwitchEmotes(),
|
||||
fetcher.fetchTwitchEmotes(channel_id),
|
||||
fetcher.fetchBTTVEmotes(),
|
||||
fetcher.fetchBTTVEmotes(channel_id),
|
||||
fetcher.fetchSevenTVEmotes(),
|
||||
fetcher.fetchSevenTVEmotes(channel_id),
|
||||
fetcher.fetchFFZEmotes(),
|
||||
fetcher.fetchFFZEmotes(channel_id)
|
||||
]);
|
||||
|
||||
const emotes_dir = path.join('appdata', 'emotes', file_uid);
|
||||
const emote_json_path = path.join(emotes_dir, `emotes.json`);
|
||||
fs.ensureDirSync(emotes_dir);
|
||||
|
||||
const emote_objs = [];
|
||||
let failed_emote_count = 0;
|
||||
for (const [, emote] of fetcher.emotes) {
|
||||
const emote_obj = emote.toObject();
|
||||
|
||||
const ext = emote.imageType;
|
||||
const emote_image_path = path.join(emotes_dir, `${emote.id}.${ext}`);
|
||||
|
||||
try {
|
||||
const link = emote.toLink();
|
||||
if (!fs.existsSync(emote_image_path)) await utils.fetchFile(link, emote_image_path);
|
||||
emote_obj['ext'] = ext;
|
||||
emote_objs.push(emote_obj);
|
||||
} catch (err) {
|
||||
failed_emote_count++;
|
||||
}
|
||||
}
|
||||
if (failed_emote_count) logger.warn(`${failed_emote_count} emotes failed to download for channel ${channel_name}`);
|
||||
await fs.writeJSON(emote_json_path, emote_objs);
|
||||
return emote_objs;
|
||||
} catch (err) {
|
||||
logger.error(err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
exports.getTwitchOAuthToken = async (client_id, client_secret) => {
|
||||
logger.verbose('Generating new Twitch auth token');
|
||||
const url = `https://id.twitch.tv/oauth2/token`;
|
||||
|
||||
try {
|
||||
const response = await axios.post(url, {client_id: client_id, client_secret: client_secret, grant_type: 'client_credentials'});
|
||||
const token = response['data']['access_token'];
|
||||
const expires_in = response['data']['expires_in'];
|
||||
if (token) return {token, expires_in};
|
||||
|
||||
logger.error(`Failed to get token.`);
|
||||
return null;
|
||||
} catch (err) {
|
||||
logger.error(`Failed to get token.`);
|
||||
logger.error(err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
exports.getChannelID = async (channel_name) => {
|
||||
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
|
||||
const token = await exports.ensureTwitchAuth();
|
||||
const url = `https://api.twitch.tv/helix/users?login=${channel_name}`;
|
||||
const headers = {
|
||||
'Client-ID': twitch_client_id,
|
||||
'Authorization': 'Bearer ' + token,
|
||||
// Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8'
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await axios.get(url, {headers: headers});
|
||||
const data = response.data.data;
|
||||
|
||||
if (data && data.length > 0) {
|
||||
const channelID = data[0].id;
|
||||
return channelID;
|
||||
}
|
||||
|
||||
logger.error(`Failed to get channel ID for user ${channel_name}`);
|
||||
if (data.error) logger.error(data.error);
|
||||
return null; // User not found
|
||||
} catch (err) {
|
||||
logger.error(`Failed to get channel ID for user ${channel_name}`);
|
||||
logger.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
|
||||
.toISOString()
|
||||
.replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
|
||||
@@ -241,3 +115,9 @@ const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
|
||||
const seg = v => v ? v.padStart(2, '0') : '00';
|
||||
return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
getCommentsForVOD: getCommentsForVOD,
|
||||
getTwitchChatByFileID: getTwitchChatByFileID,
|
||||
downloadTwitchChatByVODID: downloadTwitchChatByVODID
|
||||
}
|
||||
@@ -13,7 +13,7 @@ const CONSTS = require('./consts');
|
||||
const is_windows = process.platform === 'win32';
|
||||
|
||||
// replaces .webm with appropriate extension
|
||||
exports.getTrueFileName = (unfixed_path, type, force_ext = null) => {
|
||||
exports.getTrueFileName = (unfixed_path, type) => {
|
||||
let fixed_path = unfixed_path;
|
||||
|
||||
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
|
||||
@@ -22,7 +22,7 @@ exports.getTrueFileName = (unfixed_path, type, force_ext = null) => {
|
||||
|
||||
|
||||
if (old_ext !== new_ext) {
|
||||
unfixed_parts[unfixed_parts.length-1] = force_ext || new_ext;
|
||||
unfixed_parts[unfixed_parts.length-1] = new_ext;
|
||||
fixed_path = unfixed_parts.join('.');
|
||||
}
|
||||
return fixed_path;
|
||||
@@ -347,7 +347,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
|
||||
if (!err) {
|
||||
clearTimeout(timer);
|
||||
if (watcher) watcher.close();
|
||||
resolve(true);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -357,36 +357,33 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
|
||||
if (eventType === 'rename' && filename === basename) {
|
||||
clearTimeout(timer);
|
||||
if (watcher) watcher.close();
|
||||
resolve(true);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// helper function to download file using fetch
|
||||
exports.fetchFile = async (url, output_path, file_label = null) => {
|
||||
exports.fetchFile = async (url, path, file_label) => {
|
||||
var len = null;
|
||||
const res = await fetch(url);
|
||||
let bar = null;
|
||||
if (file_label) {
|
||||
len = parseInt(res.headers.get("Content-Length"), 10);
|
||||
|
||||
bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
|
||||
complete: '=',
|
||||
incomplete: ' ',
|
||||
width: 20,
|
||||
total: len
|
||||
});
|
||||
}
|
||||
len = parseInt(res.headers.get("Content-Length"), 10);
|
||||
|
||||
const fileStream = fs.createWriteStream(output_path);
|
||||
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
|
||||
complete: '=',
|
||||
incomplete: ' ',
|
||||
width: 20,
|
||||
total: len
|
||||
});
|
||||
const fileStream = fs.createWriteStream(path);
|
||||
await new Promise((resolve, reject) => {
|
||||
res.body.pipe(fileStream);
|
||||
res.body.on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
res.body.on('data', function (chunk) {
|
||||
if (file_label) bar.tick(chunk.length);
|
||||
bar.tick(chunk.length);
|
||||
});
|
||||
fileStream.on("finish", function() {
|
||||
resolve();
|
||||
@@ -533,40 +530,6 @@ exports.getDirectoriesInDirectory = async (basePath) => {
|
||||
}
|
||||
}
|
||||
|
||||
exports.parseOutputJSON = (output, err) => {
|
||||
let split_output = [];
|
||||
// const output_jsons = [];
|
||||
if (err && !output) {
|
||||
if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) {
|
||||
return null;
|
||||
}
|
||||
logger.info('An error was encountered with at least one video, backup method will be used.')
|
||||
try {
|
||||
split_output = err.stdout.split(/\r\n|\r|\n/);
|
||||
} catch (e) {
|
||||
logger.error('Backup method failed. See error below:');
|
||||
logger.error(e);
|
||||
return null;
|
||||
}
|
||||
} else if (output.length === 0 || (output.length === 1 && output[0].length === 0)) {
|
||||
// output is '' or ['']
|
||||
return [];
|
||||
} else {
|
||||
for (const output_item of output) {
|
||||
// we have to do this because sometimes there will be leading characters before the actual json
|
||||
const start_idx = output_item.indexOf('{"');
|
||||
const clean_output = output_item.slice(start_idx, output_item.length);
|
||||
split_output.push(clean_output);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return split_output.map(split_output_str => JSON.parse(split_output_str));
|
||||
} catch(e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// objects
|
||||
|
||||
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
|
||||
|
||||
@@ -5,7 +5,6 @@ const logger = require('./logger');
|
||||
const utils = require('./utils');
|
||||
const CONSTS = require('./consts');
|
||||
const config_api = require('./config.js');
|
||||
const youtubedl = require('youtube-dl');
|
||||
|
||||
const OUTDATED_VERSION = "2020.00.00";
|
||||
|
||||
@@ -26,15 +25,6 @@ const download_sources = {
|
||||
}
|
||||
}
|
||||
|
||||
exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
|
||||
return new Promise(resolve => {
|
||||
downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
|
||||
const parsed_output = utils.parseOutputJSON(output, err);
|
||||
resolve({parsed_output, err});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
exports.checkForYoutubeDLUpdate = async () => {
|
||||
return new Promise(async resolve => {
|
||||
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
|
||||
|
||||
24081
package-lock.json
generated
24081
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
39
package.json
39
package.json
@@ -21,21 +21,20 @@
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular-devkit/core": "^16.0.3",
|
||||
"@angular/animations": "^16.0.3",
|
||||
"@angular/cdk": "^16.0.2",
|
||||
"@angular/common": "^16.0.3",
|
||||
"@angular/compiler": "^16.0.3",
|
||||
"@angular/core": "^16.0.3",
|
||||
"@angular/forms": "^16.0.3",
|
||||
"@angular/localize": "^16.0.3",
|
||||
"@angular/material": "^16.0.2",
|
||||
"@angular/platform-browser": "^16.0.3",
|
||||
"@angular/platform-browser-dynamic": "^16.0.3",
|
||||
"@angular/router": "^16.0.3",
|
||||
"@angular-devkit/core": "^15.0.1",
|
||||
"@angular/animations": "^15.0.1",
|
||||
"@angular/cdk": "^15.0.0",
|
||||
"@angular/common": "^15.0.1",
|
||||
"@angular/compiler": "^15.0.1",
|
||||
"@angular/core": "^15.0.1",
|
||||
"@angular/forms": "^15.0.1",
|
||||
"@angular/localize": "^15.0.1",
|
||||
"@angular/material": "^15.0.0",
|
||||
"@angular/platform-browser": "^15.0.1",
|
||||
"@angular/platform-browser-dynamic": "^15.0.1",
|
||||
"@angular/router": "^15.0.1",
|
||||
"@fontsource/material-icons": "^4.5.4",
|
||||
"@ngneat/content-loader": "^7.0.0",
|
||||
"@tzahi12345/twitch-emoticons": "^1.0.9",
|
||||
"@videogular/ngx-videogular": "^6.0.0",
|
||||
"core-js": "^2.4.1",
|
||||
"crypto-js": "^4.1.1",
|
||||
@@ -44,20 +43,20 @@
|
||||
"fs-extra": "^10.0.0",
|
||||
"material-icons": "^1.10.8",
|
||||
"nan": "^2.14.1",
|
||||
"ngx-avatars": "^1.6.1",
|
||||
"ngx-avatars": "^1.4.1",
|
||||
"ngx-file-drop": "^15.0.0",
|
||||
"rxjs": "^6.6.3",
|
||||
"rxjs-compat": "^6.6.7",
|
||||
"tslib": "^2.0.0",
|
||||
"typescript": "~5.0.4",
|
||||
"typescript": "~4.8.4",
|
||||
"xliff-to-json": "^1.0.4",
|
||||
"zone.js": "~0.13.0"
|
||||
"zone.js": "~0.11.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-devkit/build-angular": "^16.0.3",
|
||||
"@angular/cli": "^16.0.3",
|
||||
"@angular/compiler-cli": "^16.0.3",
|
||||
"@angular/language-service": "^16.0.3",
|
||||
"@angular-devkit/build-angular": "^15.0.1",
|
||||
"@angular/cli": "^15.0.1",
|
||||
"@angular/compiler-cli": "^15.0.1",
|
||||
"@angular/language-service": "^15.0.1",
|
||||
"@types/core-js": "^2.5.2",
|
||||
"@types/file-saver": "^2.0.1",
|
||||
"@types/jasmine": "^4.3.1",
|
||||
|
||||
@@ -78,6 +78,12 @@ export class DownloadsComponent implements OnInit, OnDestroy {
|
||||
show: (download: Download) => !download.finished && download.paused && download.finished_step,
|
||||
icon: 'play_arrow'
|
||||
},
|
||||
{
|
||||
tooltip: $localize`Resume`,
|
||||
action: (download: Download) => this.resumeDownload(download),
|
||||
show: (download: Download) => !download.finished && download.paused && download.finished_step,
|
||||
icon: 'play_arrow'
|
||||
},
|
||||
{
|
||||
tooltip: $localize`Cancel`,
|
||||
action: (download: Download) => this.cancelDownload(download),
|
||||
|
||||
@@ -380,8 +380,8 @@ export class RecentVideosComponent implements OnInit {
|
||||
|
||||
fileSelectionChanged(event: MatSelectionListChange): void {
|
||||
// TODO: make sure below line is possible (_selected is private)
|
||||
const adding = event.options[0]['_selected'];
|
||||
const value = event.options[0].value;
|
||||
const adding = event.option['_selected'];
|
||||
const value = event.option.value;
|
||||
if (adding) {
|
||||
this.selected_data.push(value.uid);
|
||||
this.selected_data_objs.push(value);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<div class="chat-container" #scrollContainer *ngIf="visible_chat">
|
||||
<div style="width: 250px; text-align: center;"><strong>Twitch Chat</strong></div>
|
||||
<div #chat style="max-width: 250px" *ngFor="let chat of visible_chat; let last = last">
|
||||
{{chat.timestamp_str}} - <strong [style.color]="chat.user_color ? chat.user_color : null">{{chat.name}}</strong>: <span [innerHTML]="chat.message"></span>
|
||||
{{chat.timestamp_str}} - <strong [style.color]="chat.user_color ? chat.user_color : null">{{chat.name}}</strong>: {{chat.message}}
|
||||
{{last ? scrollToBottom() : ''}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { Component, ElementRef, Input, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
|
||||
import { DatabaseFile } from 'api-types';
|
||||
import { PostsService } from 'app/posts.services';
|
||||
import { EmoteFetcher, EmoteObject, EmoteParser } from '@tzahi12345/twitch-emoticons';
|
||||
|
||||
@Component({
|
||||
selector: 'app-twitch-chat',
|
||||
@@ -14,7 +13,6 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
|
||||
visible_chat = null;
|
||||
chat_response_received = false;
|
||||
downloading_chat = false;
|
||||
got_emotes = false;
|
||||
|
||||
current_chat_index = null;
|
||||
|
||||
@@ -23,9 +21,6 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
|
||||
|
||||
scrollContainer = null;
|
||||
|
||||
fetcher: EmoteFetcher;
|
||||
parser: EmoteParser;
|
||||
|
||||
@Input() db_file: DatabaseFile = null;
|
||||
@Input() sub = null;
|
||||
@Input() current_timestamp = null;
|
||||
@@ -37,7 +32,6 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
|
||||
|
||||
ngOnInit(): void {
|
||||
this.getFullChat();
|
||||
this.getEmotes();
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
@@ -75,12 +69,10 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
|
||||
const latest_chat_timestamp = this.visible_chat.length ? this.visible_chat[this.visible_chat.length - 1]['timestamp'] : 0;
|
||||
|
||||
for (let i = this.current_chat_index + 1; i < this.full_chat.length; i++) {
|
||||
const new_chat = this.full_chat[i];
|
||||
if (new_chat['timestamp'] >= latest_chat_timestamp && new_chat['timestamp'] <= this.current_timestamp) {
|
||||
new_chat['message'] = this.got_emotes ? this.parseChat(new_chat['message']) : new_chat['message'];
|
||||
this.visible_chat.push(new_chat);
|
||||
if (this.full_chat[i]['timestamp'] >= latest_chat_timestamp && this.full_chat[i]['timestamp'] <= this.current_timestamp) {
|
||||
this.visible_chat.push(this.full_chat[i]);
|
||||
this.current_chat_index = i;
|
||||
} else if (new_chat['timestamp'] > this.current_timestamp) {
|
||||
} else if (this.full_chat[i]['timestamp'] > this.current_timestamp) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -126,29 +118,6 @@ export class TwitchChatComponent implements OnInit, OnDestroy {
|
||||
this.chat_check_interval_obj = setInterval(() => this.addNewChatMessages(), this.CHAT_CHECK_INTERVAL_MS);
|
||||
}
|
||||
|
||||
getEmotes() {
|
||||
this.postsService.getTwitchEmotes(this.db_file['uid']).subscribe(res => {
|
||||
const emotes = res['emotes'];
|
||||
this.processEmotes(emotes);
|
||||
});
|
||||
}
|
||||
|
||||
processEmotes(emotes: EmoteObject[]) {
|
||||
this.fetcher = new EmoteFetcher();
|
||||
this.parser = new EmoteParser(this.fetcher, {
|
||||
// Custom HTML format
|
||||
template: `<img class="emote" alt="{name}" src="{link}">`,
|
||||
// Match without :colons:
|
||||
match: /(\w+)+?/g
|
||||
});
|
||||
this.fetcher.fromObject(emotes);
|
||||
this.got_emotes = true;
|
||||
}
|
||||
|
||||
parseChat(chat_message: string) {
|
||||
return this.parser.parse(chat_message);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function binarySearch(arr, key, n) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import 'rxjs/add/operator/map';
|
||||
import 'rxjs/add/operator/catch';
|
||||
import 'rxjs/add/observable/throw';
|
||||
import { THEMES_CONFIG } from '../themes';
|
||||
import { Router, ActivatedRouteSnapshot } from '@angular/router';
|
||||
import { Router, CanActivate, ActivatedRouteSnapshot } from '@angular/router';
|
||||
import { DOCUMENT } from '@angular/common';
|
||||
import { BehaviorSubject, Observable } from 'rxjs';
|
||||
import { MatSnackBar } from '@angular/material/snack-bar';
|
||||
@@ -118,10 +118,9 @@ import {
|
||||
import { isoLangs } from './dialogs/user-profile-dialog/locales_list';
|
||||
import { Title } from '@angular/platform-browser';
|
||||
import { MatDrawerMode } from '@angular/material/sidenav';
|
||||
import type { EmoteObject } from '@tzahi12345/twitch-emoticons';
|
||||
|
||||
@Injectable()
|
||||
export class PostsService {
|
||||
export class PostsService implements CanActivate {
|
||||
path = '';
|
||||
|
||||
// local settings
|
||||
@@ -408,10 +407,6 @@ export class PostsService {
|
||||
return this.http.post<DownloadTwitchChatByVODIDResponse>(this.path + 'downloadTwitchChatByVODID', body, this.httpOptions);
|
||||
}
|
||||
|
||||
getTwitchEmotes(uid: string) {
|
||||
return this.http.post<{emotes: EmoteObject[]}>(this.path + 'getTwitchEmotes', {uid: uid}, this.httpOptions);
|
||||
}
|
||||
|
||||
downloadPlaylistFromServer(playlist_id, uuid = null) {
|
||||
const body: DownloadFileRequest = {uuid: uuid, playlist_id: playlist_id};
|
||||
return this.http.post(this.path + 'downloadFileFromServer', body, {responseType: 'blob', params: this.httpOptions.params});
|
||||
|
||||
@@ -252,25 +252,9 @@
|
||||
<mat-hint><a target="_blank" href="https://developers.google.com/youtube/v3/getting-started"><ng-container i18n="Youtube API Key setting hint">Generating a key is easy!</ng-container></a></mat-hint>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<div class="col-12 mt-3">
|
||||
<mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_twitch_API']"><ng-container i18n="Use Twitch API setting">Use Twitch API</ng-container></mat-checkbox>
|
||||
</div>
|
||||
<div class="col-12 mt-1">
|
||||
<mat-checkbox color="accent" [(ngModel)]="new_config['API']['twitch_auto_download_chat']"><ng-container i18n="Auto download Twitch Chat setting">Auto-download Twitch Chat</ng-container></mat-checkbox>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<mat-form-field class="text-field" color="accent">
|
||||
<mat-label i18n="Twitch Client ID">Twitch Client ID</mat-label>
|
||||
<input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_ID']" matInput required>
|
||||
<mat-hint><a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch Client ID setting hint">Generating an ID/secret is easy!</ng-container></a></mat-hint>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<div class="col-12 mt-2">
|
||||
<mat-form-field class="text-field" color="accent">
|
||||
<mat-label i18n="Twitch Client Secret">Twitch Client Secret</mat-label>
|
||||
<input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_secret']" matInput required>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<div class="col-12 mt-2">
|
||||
<mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_sponsorblock_API']" matTooltip="Enables a button to skip ads when viewing supported videos." i18n-matTooltip="SponsorBlock API tooltip"><ng-container i18n="Use SponsorBlock API setting">Use SponsorBlock API</ng-container></mat-checkbox>
|
||||
</div>
|
||||
|
||||
@@ -4133,60 +4133,6 @@
|
||||
<context context-type="linenumber">18</context>
|
||||
</context-group>
|
||||
</trans-unit>
|
||||
<trans-unit id="c748ac656af9f13998206ef2c52018dd418b0483" datatype="html">
|
||||
<source>Archives</source>
|
||||
<target state="translated">Arsip</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/app.component.html</context>
|
||||
<context context-type="linenumber">26</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">Archives menu label</note>
|
||||
</trans-unit>
|
||||
<trans-unit id="5ca707824ab93066c7d9b44e1b8bf216725c2c22" datatype="html">
|
||||
<source>Filter</source>
|
||||
<target state="translated">Penyaringan</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/archive-viewer/archive-viewer.component.html</context>
|
||||
<context context-type="linenumber">3</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">Filter</note>
|
||||
</trans-unit>
|
||||
<trans-unit id="45cc8ca94b5a50842a9a8ef804a5ab089a38ae5c" datatype="html">
|
||||
<source>ID</source>
|
||||
<target state="translated">ID</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/archive-viewer/archive-viewer.component.html</context>
|
||||
<context context-type="linenumber">47</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">ID</note>
|
||||
</trans-unit>
|
||||
<trans-unit id="28da11220a3377ddce3c7948825d33101f142782" datatype="html">
|
||||
<source>Extractor</source>
|
||||
<target state="needs-translation">Extractor</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/archive-viewer/archive-viewer.component.html</context>
|
||||
<context context-type="linenumber">57</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">Extractor</note>
|
||||
</trans-unit>
|
||||
<trans-unit id="c150a30bbbdb175b4d08820196a9acb66b167653" datatype="html">
|
||||
<source>Archives empty</source>
|
||||
<target state="translated">Arsip kosong</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/archive-viewer/archive-viewer.component.html</context>
|
||||
<context context-type="linenumber">72</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">Archives empty</note>
|
||||
</trans-unit>
|
||||
<trans-unit id="51a161ce175abcd44f6c6cbd0e996681bf553ac3" datatype="html">
|
||||
<source>Delete selected</source>
|
||||
<target state="translated">Hapus yang dipilih</target>
|
||||
<context-group purpose="location">
|
||||
<context context-type="sourcefile">src/app/components/archive-viewer/archive-viewer.component.html</context>
|
||||
<context context-type="linenumber">77</context>
|
||||
</context-group>
|
||||
<note priority="1" from="description">Delete selected</note>
|
||||
</trans-unit>
|
||||
</body>
|
||||
</file>
|
||||
</xliff>
|
||||
|
||||
Reference in New Issue
Block a user