TheLanc3/YoutubeDL-Material
1
0
Fork 0
mirror of https://github.com/Tzahi12345/YoutubeDL-Material.git synced 2026-03-07 20:10:03 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #936 from Tzahi12345/test-improvements

Browse Source 
Test improvements
This commit is contained in:
Tzahi12345
2023-05-28 19:33:53 -04:00
committed by GitHub
parent 0427f91cfc be71a9bd8c
commit 098c5a3c25
12 changed files with 260 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
.github/workflows/mocha.yml vendored Normal file
View File

@@ -0,0 +1,40 @@
name: Tests
'on':
push:
branches:
- master
pull_request:
types:
- opened
- synchronize
- reopened
jobs:
test:
name: 'Backend - mocha'
runs-on: ubuntu-latest
strategy:
matrix:
node:
- 16
steps:
- uses: actions/setup-node@v3
with:
node-version: '${{ matrix.node }}'
- uses: actions/checkout@v2
- name: 'Cache node_modules'
uses: actions/cache@v2
with:
path: ~/.npm
key: ${{ runner.os }}-node-v${{ matrix.node }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-v${{ matrix.node }}-
working-directory: ./backend
- uses: FedericoCarboni/setup-ffmpeg@v2
id: setup-ffmpeg
- name: Install Dependencies
run: npm install
working-directory: ./backend
- name: Run All Node.js Tests
run: npm run test
working-directory: ./backend

2
.vscode/settings.json vendored
View File

@@ -3,6 +3,6 @@
"mochaExplorer.cwd": "backend",
"mochaExplorer.globImplementation": "vscode",
"mochaExplorer.env": {
"YTDL_MODE": "debug"
// "YTDL_MODE": "debug"
}
}

16
backend/db.js
View File

@@ -11,9 +11,8 @@ const logger = require('./logger');
const low = require('lowdb')
const FileSync = require('lowdb/adapters/FileSync');
const { BehaviorSubject } = require('rxjs');
const local_adapter = new FileSync('./appdata/local_db.json');
const local_db = low(local_adapter);
let local_db = null;
let database = null;
exports.database_initialized = false;
exports.database_initialized_bs = new BehaviorSubject(false);
@@ -73,10 +72,6 @@ const tables = {
const tables_list = Object.keys(tables);
const local_db_defaults = {}
tables_list.forEach(table => {local_db_defaults[table] = []});
local_db.defaults(local_db_defaults).write();
let using_local_db = null;
function setDB(input_db, input_users_db) {
@@ -85,11 +80,18 @@ function setDB(input_db, input_users_db) {
exports.users_db = input_users_db
}
exports.initialize = (input_db, input_users_db) => {
exports.initialize = (input_db, input_users_db, db_name = 'local_db.json') => {
setDB(input_db, input_users_db);
// must be done here to prevent getConfigItem from being called before init
using_local_db = config_api.getConfigItem('ytdl_use_local_db');
const local_adapter = new FileSync(`./appdata/${db_name}`);
local_db = low(local_adapter);
const local_db_defaults = {}
tables_list.forEach(table => {local_db_defaults[table] = []});
local_db.defaults(local_db_defaults).write();
}
exports.connectToDB = async (retries = 5, no_fallback = false, custom_connection_string = null) => {

212
backend/downloader.js
View File

@@ -7,6 +7,7 @@ const Mutex = require('async-mutex').Mutex;
const youtubedl = require('youtube-dl');
const logger = require('./logger');
const youtubedl_api = require('./youtube-dl');
const config_api = require('./config');
const twitch_api = require('./twitch');
const { create } = require('xmlbuilder2');
@@ -21,10 +22,10 @@ const mutex = new Mutex();
let should_check_downloads = true;
if (db_api.database_initialized) {
setupDownloads();
exports.setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
if (init) exports.setupDownloads();
});
}
@@ -47,7 +48,7 @@ We use checkDownloads() to move downloads through the steps and call their respe
*/
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null) => {
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null, prefetched_info = null, paused = false) => {
return await mutex.runExclusive(async () => {
const download = {
url: url,
@@ -60,7 +61,7 @@ exports.createDownload = async (url, type, options, user_uid = null, sub_id = nu
options: options,
uid: uuid(),
step_index: 0,
paused: false,
paused: paused,
running: false,
finished_step: true,
error: null,
@@ -133,7 +134,7 @@ async function handleDownloadError(download, error_message, error_type = null) {
await db_api.updateRecord('download_queue', {uid: download['uid']}, {error: error_message, finished: true, running: false, error_type: error_type});
}
async function setupDownloads() {
exports.setupDownloads = async () => {
await fixDownloadState();
setInterval(checkDownloads, 1000);
}
@@ -188,7 +189,7 @@ async function checkDownloads() {
if (waiting_download['step_index'] === 0) {
collectInfo(waiting_download['uid']);
} else if (waiting_download['step_index'] === 1) {
downloadQueuedFile(waiting_download['uid']);
exports.downloadQueuedFile(waiting_download['uid']);
}
}
}
@@ -240,7 +241,7 @@ async function collectInfo(download_uid) {
let category = null;
// check if it fits into a category. If so, then get info again using new args
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
if (info.length === 0 || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
@@ -259,13 +260,9 @@ async function collectInfo(download_uid) {
const files_to_check_for_progress = [];
// store info in download for future use
if (Array.isArray(info)) {
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
} else {
files_to_check_for_progress.push(utils.removeFileExtension(info['_filename']));
}
for (let info_obj of info) files_to_check_for_progress.push(utils.removeFileExtension(info_obj['_filename']));
const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null;
const playlist_title = info.length > 0 ? info[0]['playlist_title'] || info[0]['playlist'] : null;
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args,
finished_step: true,
running: false,
@@ -278,7 +275,7 @@ async function collectInfo(download_uid) {
});
}
async function downloadQueuedFile(download_uid) {
exports.downloadQueuedFile = async(download_uid, downloadMethod = youtubedl.exec) => {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
if (download['paused']) {
return;
@@ -306,105 +303,102 @@ async function downloadQueuedFile(download_uid) {
const start_time = Date.now();
const download_checker = setInterval(() => checkDownloadPercent(download['uid']), 1000);
const file_objs = [];
// download file
youtubedl.exec(url, args, {maxBuffer: Infinity}, async function(err, output) {
const file_objs = [];
let end_time = Date.now();
let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
clearInterval(download_checker);
const parsed_output = utils.parseOutputJSON(output, err);
if (!parsed_output) {
logger.error(err.stderr);
await handleDownloadError(download, err.stderr, 'unknown_error');
const {parsed_output, err} = await youtubedl_api.runYoutubeDL(url, args, downloadMethod);
clearInterval(download_checker);
let end_time = Date.now();
let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
if (!parsed_output) {
logger.error(err.stderr);
await handleDownloadError(download, err.stderr, 'unknown_error');
resolve(false);
return;
} else if (parsed_output) {
if (parsed_output.length === 0 || parsed_output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download, error_message, 'no_output');
logger.warn(error_message);
resolve(false);
return;
} else if (parsed_output) {
if (parsed_output.length === 0 || parsed_output[0].length === 0) {
// ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download, error_message, 'no_output');
logger.warn(error_message);
resolve(false);
return;
}
for (const output_json of parsed_output) {
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
const ext = type === 'audio' ? '.mp3' : '.mp4';
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj);
}
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download, error_message, 'no_metadata');
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
resolve();
}
});
for (const output_json of parsed_output) {
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
const ext = type === 'audio' ? '.mp3' : '.mp4';
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename']) && fs.existsSync(output_json['_filename'] + '.webm')) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
logger.error(`Failed to rename file ${output_json['_filename']} to its appropriate extension.`);
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (config_api.getConfigItem('ytdl_generate_nfo_files')) {
exports.generateNFOFile(output_json, `${filepath_no_extension}.nfo`);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await files_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj);
}
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
container = await files_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), download['user_uid']);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
const error_message = 'Downloaded file failed to result in metadata object.';
logger.error(error_message);
await handleDownloadError(download, error_message, 'no_metadata');
}
const file_uids = file_objs.map(file_obj => file_obj.uid);
await db_api.updateRecord('download_queue', {uid: download_uid}, {finished_step: true, finished: true, running: false, step_index: 3, percent_complete: 100, file_uids: file_uids, container: container});
resolve(file_uids);
}
});
}

2
backend/package.json
View File

@@ -4,7 +4,7 @@
"description": "backend for YoutubeDL-Material",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"test": "mocha test --exit -s 1000",
"start": "pm2-runtime --raw pm2.config.js",
"debug": "set YTDL_MODE=debug && node app.js"
},

2
backend/subscriptions.js
View File

@@ -287,7 +287,7 @@ async function handleOutputJSON(output_jsons, sub, user_uid) {
for (let j = 0; j < files_to_download.length; j++) {
const file_to_download = files_to_download[j];
file_to_download['formats'] = utils.stripPropertiesFromObject(file_to_download['formats'], ['format_id', 'filesize', 'filesize_approx']); // prevent download object from blowing up in size
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, file_to_download);
await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name, [file_to_download]);
}
return files_to_download;

1
backend/test/sample.info.json
View File

File diff suppressed because one or more lines are too long

1
backend/test/sample_mp3.info.json Normal file
View File

File diff suppressed because one or more lines are too long

1
backend/test/sample_mp4.info.json Normal file
View File

File diff suppressed because one or more lines are too long

141
backend/test/tests.js
View File

@@ -3,7 +3,8 @@ const assert = require('assert');
const low = require('lowdb')
const winston = require('winston');
const path = require('path');
const util = require('util');
const exec = util.promisify(require('child_process').exec);
const FileSync = require('lowdb/adapters/FileSync');
@@ -41,11 +42,13 @@ const subscriptions_api = require('../subscriptions');
const archive_api = require('../archive');
const categories_api = require('../categories');
const files_api = require('../files');
const youtubedl_api = require('../youtube-dl');
const config_api = require('../config');
const fs = require('fs-extra');
const { uuid } = require('uuidv4');
const NodeID3 = require('node-id3');
db_api.initialize(db, users_db);
db_api.initialize(db, users_db, 'local_db_test.json');
const sample_video_json = {
id: "Sample Video",
@@ -68,9 +71,9 @@ const sample_video_json = {
}
describe('Database', async function() {
describe('Import', async function() {
describe.skip('Import', async function() {
// it('Migrate', async function() {
// await db_api.connectToDB();
// // await db_api.connectToDB();
// await db_api.removeAllRecords();
// const success = await db_api.importJSONToDB(db.value(), users_db.value());
// assert(success);
@@ -86,7 +89,7 @@ describe('Database', async function() {
});
it('Transfer to local', async function() {
await db_api.connectToDB();
// await db_api.connectToDB();
await db_api.removeAllRecords('test');
await db_api.insertRecordIntoTable('test', {test: 'test'});
@@ -114,7 +117,8 @@ describe('Database', async function() {
for (const local_db_mode of local_db_modes) {
let use_local_db = local_db_mode;
describe(`Use local DB - ${use_local_db}`, async function() {
const describe_skippable = use_local_db ? describe : describe.skip;
describe_skippable(`Use local DB - ${use_local_db}`, async function() {
beforeEach(async function() {
if (!use_local_db) {
this.timeout(120000);
@@ -167,7 +171,7 @@ describe('Database', async function() {
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
assert(duplicates && duplicates.length === 2 && duplicates[0]['key'] === '2' && duplicates[1]['key'] === '4')
});
it('Update record', async function() {
@@ -279,7 +283,7 @@ describe('Database', async function() {
assert(stats);
});
it('Query speed', async function() {
it.skip('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
@@ -337,12 +341,13 @@ describe('Database', async function() {
});
describe('Multi User', async function() {
this.timeout(120000);
const user_to_test = 'test_user';
const user_password = 'test_pass';
const sub_to_test = '';
const playlist_to_test = '';
beforeEach(async function() {
await db_api.connectToDB();
// await db_api.connectToDB();
await auth_api.deleteUser(user_to_test);
});
describe('Basic', function() {
@@ -369,17 +374,17 @@ describe('Multi User', async function() {
it('Video access - disallowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: false});
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
assert(!video_obj);
});
it('Video access - allowed', async function() {
await db_api.setVideoProperty(video_to_test, {sharingEnabled: true}, user_to_test);
const video_obj = auth_api.getUserVideo(user_to_test, video_to_test, true);
const video_obj = await auth_api.getUserVideo(user_to_test, video_to_test, true);
assert(video_obj);
});
});
describe('Zip generators', function() {
describe.skip('Zip generators', function() {
it('Playlist zip generator', async function() {
const playlist = await files_api.getPlaylist(playlist_to_test, user_to_test);
assert(playlist);
@@ -435,35 +440,50 @@ describe('Multi User', async function() {
describe('Downloader', function() {
const downloader_api = require('../downloader');
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
const url = 'https://www.youtube.com/watch?v=hpigjnKl7nI';
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const options = {
ui_uid: uuid(),
user: 'admin'
ui_uid: uuid()
}
before(async function() {
const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
config_api.setConfigItem('ytdl_max_concurrent_downloads', 0);
});
beforeEach(async function() {
await db_api.connectToDB();
// await db_api.connectToDB();
await db_api.removeAllRecords('download_queue');
});
it('Get file info', async function() {
this.timeout(300000);
const info = await downloader_api.getVideoInfoByURL(url);
assert(!!info);
assert(!!info && info.length > 0);
});
it('Download file', async function() {
this.timeout(300000);
this.timeout(300000);
await downloader_api.setupDownloads();
const args = await downloader_api.generateArgs(url, 'video', options, null, true);
const [info] = await downloader_api.getVideoInfoByURL(url, args);
if (fs.existsSync(info['_filename'])) fs.unlinkSync(info['_filename']);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
assert(returned_download);
const custom_download_method = async (url, args, options, callback) => {
fs.writeJSONSync(utils.getTrueFileName(info['_filename'], 'video', '.info.json'), info);
await generateEmptyVideoFile(info['_filename']);
return await callback(null, [JSON.stringify(info)]);
}
const success = await downloader_api.downloadQueuedFile(returned_download['uid'], custom_download_method);
assert(success);
});
it('Tag file', async function() {
const audio_path = './test/sample.mp3';
const sample_json = fs.readJSONSync('./test/sample.info.json');
const success = await generateEmptyAudioFile('test/sample_mp3.mp3');
const audio_path = './test/sample_mp3.mp3';
const sample_json = fs.readJSONSync('./test/sample_mp3.info.json');
const tags = {
title: sample_json['title'],
artist: sample_json['artist'] ? sample_json['artist'] : sample_json['uploader'],
@@ -471,14 +491,13 @@ describe('Downloader', function() {
}
NodeID3.write(tags, audio_path);
const written_tags = NodeID3.read(audio_path);
assert(written_tags['raw']['TRCK'] === '27');
assert(success && written_tags['raw']['TRCK'] === '27');
});
it('Queue file', async function() {
this.timeout(300000);
const returned_download = await downloader_api.createDownload(url, 'video', options);
console.log(returned_download);
await utils.wait(20000);
const returned_download = await downloader_api.createDownload(url, 'video', options, null, null, null, null, true);
assert(returned_download);
});
it('Pause file', async function() {
@@ -493,7 +512,7 @@ describe('Downloader', function() {
assert(args.length > 0);
});
it('Generate args - subscription', async function() {
it.skip('Generate args - subscription', async function() {
const sub = await subscriptions_api.getSubscription(sub_id);
const sub_options = subscriptions_api.generateOptionsForSubscriptionDownload(sub, 'admin');
const args_normal = await downloader_api.generateArgs(url, 'video', options);
@@ -506,7 +525,7 @@ describe('Downloader', function() {
if (fs.existsSync(nfo_file_path)) {
fs.unlinkSync(nfo_file_path);
}
const sample_json = fs.readJSONSync('./test/sample.info.json');
const sample_json = fs.readJSONSync('./test/sample_mp4.info.json');
downloader_api.generateNFOFile(sample_json, nfo_file_path);
assert(fs.existsSync(nfo_file_path), true);
fs.unlinkSync(nfo_file_path);
@@ -534,10 +553,18 @@ describe('Downloader', function() {
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1710641401';
it('Download VOD', async function() {
it('Download VOD chat', async function() {
this.timeout(300000);
if (!fs.existsSync('TwitchDownloaderCLI')) {
try {
await exec('sh ../docker-utils/fetch-twitchdownloader.sh');
fs.copyFileSync('../docker-utils/TwitchDownloaderCLI', 'TwitchDownloaderCLI');
} catch (e) {
logger.info('TwitchDownloaderCLI fetch failed, file may exist regardless.');
}
}
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
this.timeout(300000);
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
assert(fs.existsSync(sample_path));
@@ -550,7 +577,7 @@ describe('Downloader', function() {
describe('Tasks', function() {
const tasks_api = require('../tasks');
beforeEach(async function() {
await db_api.connectToDB();
// await db_api.connectToDB();
await db_api.removeAllRecords('tasks');
const dummy_task = {
@@ -569,7 +596,7 @@ describe('Tasks', function() {
await tasks_api.executeTask('backup_local_db');
const backups_new = await utils.recFindByExt('appdata', 'bak');
const new_length = backups_new.length;
assert(original_length, new_length-1);
assert(original_length === new_length-1);
});
it('Check for missing files', async function() {
@@ -579,7 +606,7 @@ describe('Tasks', function() {
await db_api.insertRecordIntoTable('files', test_missing_file);
await tasks_api.executeTask('missing_files_check');
const missing_file_db_record = await db_api.getRecord('files', {uid: 'test'});
assert(!missing_file_db_record, true);
assert(!missing_file_db_record);
});
it('Check for duplicate files', async function() {
@@ -599,27 +626,29 @@ describe('Tasks', function() {
await tasks_api.executeTask('duplicate_files_check');
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
assert(duplicated_record_count == 1, true);
assert(duplicated_record_count === 1);
});
it('Import unregistered files', async function() {
this.timeout(300000);
const success = await generateEmptyVideoFile('test/sample_mp4.mp4');
// pre-test cleanup
await db_api.removeAllRecords('files', {title: 'Sample File'});
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
if (fs.existsSync('video/sample_mp4.mp4')) fs.unlinkSync('video/sample_mp4.mp4');
// copies in files
fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
fs.copyFileSync('test/sample_mp4.info.json', 'video/sample_mp4.info.json');
fs.copyFileSync('test/sample_mp4.mp4', 'video/sample_mp4.mp4');
await tasks_api.executeTask('missing_db_records');
const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
assert(!!imported_file === true);
assert(success && !!imported_file);
// post-test cleanup
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
if (fs.existsSync('video/sample_mp4.info.json')) fs.unlinkSync('video/sample_mp4.info.json');
if (fs.existsSync('video/sample_mp4.mp4')) fs.unlinkSync('video/sample_mp4.mp4');
});
it('Schedule and cancel task', async function() {
@@ -659,12 +688,12 @@ describe('Tasks', function() {
describe('Archive', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
// await db_api.connectToDB();
await db_api.removeAllRecords('archives');
});
afterEach(async function() {
await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
await db_api.removeAllRecords('archives');
});
it('Import archive', async function() {
@@ -678,7 +707,6 @@ describe('Archive', async function() {
const count = await archive_api.importArchiveFile(archive_text, 'video', 'test_user', 'test_sub');
assert(count === 4)
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
console.log(archive_items);
assert(archive_items.length === 4);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
@@ -709,9 +737,9 @@ describe('Archive', async function() {
});
it('Remove from archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_title', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_title', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_title', 'test_user');
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
assert(success);
@@ -757,7 +785,7 @@ describe('Utils', async function() {
describe('Categories', async function() {
beforeEach(async function() {
await db_api.connectToDB();
// await db_api.connectToDB();
const new_category = {
name: 'test_category',
uid: uuid(),
@@ -805,7 +833,6 @@ describe('Categories', async function() {
});
const category = await categories_api.categorize([sample_video_json]);
console.log(category);
assert(category && category.name === 'test_category');
});
@@ -858,4 +885,14 @@ describe('Categories', async function() {
const category = await categories_api.categorize([sample_video_json]);
assert(category);
});
});
});
const generateEmptyVideoFile = async (file_path) => {
if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
return await exec(`ffmpeg -t 1 -f lavfi -i color=c=black:s=640x480 -c:v libx264 -tune stillimage -pix_fmt yuv420p "${file_path}"`);
}
const generateEmptyAudioFile = async (file_path) => {
if (fs.existsSync(file_path)) fs.unlinkSync(file_path);
return await exec(`ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 1 -q:a 9 -acodec libmp3lame ${file_path}`);
}

8
backend/utils.js
View File

@@ -13,7 +13,7 @@ const CONSTS = require('./consts');
const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
exports.getTrueFileName = (unfixed_path, type) => {
exports.getTrueFileName = (unfixed_path, type, force_ext = null) => {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
@@ -22,7 +22,7 @@ exports.getTrueFileName = (unfixed_path, type) => {
if (old_ext !== new_ext) {
unfixed_parts[unfixed_parts.length-1] = new_ext;
unfixed_parts[unfixed_parts.length-1] = force_ext || new_ext;
fixed_path = unfixed_parts.join('.');
}
return fixed_path;
@@ -347,7 +347,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
resolve(true);
}
});
@@ -357,7 +357,7 @@ exports.checkExistsWithTimeout = async (filePath, timeout) => {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
resolve(true);
}
});
});

10
backend/youtube-dl.js
View File

@@ -5,6 +5,7 @@ const logger = require('./logger');
const utils = require('./utils');
const CONSTS = require('./consts');
const config_api = require('./config.js');
const youtubedl = require('youtube-dl');
const OUTDATED_VERSION = "2020.00.00";
@@ -25,6 +26,15 @@ const download_sources = {
}
}
exports.runYoutubeDL = async (url, args, downloadMethod = youtubedl.exec) => {
return new Promise(resolve => {
downloadMethod(url, args, {maxBuffer: Infinity}, async function(err, output) {
const parsed_output = utils.parseOutputJSON(output, err);
resolve({parsed_output, err});
});
});
}
exports.checkForYoutubeDLUpdate = async () => {
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');