From e2c31319cf236bff1d5d1b26b74d101112f5f1b9 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 23 May 2021 03:59:38 -0600 Subject: [PATCH] Migrated playlist and subscription (per video and sub-wide) video downloading functionality to new schema Migrated modify playlist component to new schema Moved wait function and playlist generation function(s) to utils - added tests for zip generation --- backend/app.js | 95 ++++++------------- backend/db.js | 1 - backend/test/tests.js | 36 +++++++ backend/utils.js | 54 ++++++++++- .../custom-playlists.component.ts | 2 +- .../create-playlist.component.html | 4 +- .../modify-playlist.component.html | 54 ++++++----- .../modify-playlist.component.ts | 30 +++--- src/app/file-card/file-card.component.ts | 2 +- src/app/main/main.component.ts | 2 +- src/app/player/player.component.ts | 5 +- src/app/posts.services.ts | 18 +++- .../subscription/subscription.component.ts | 5 +- 13 files changed, 189 insertions(+), 119 deletions(-) diff --git a/backend/app.js b/backend/app.js index 8ae57ac..065b339 100644 --- a/backend/app.js +++ b/backend/app.js @@ -193,16 +193,6 @@ app.use(auth_api.passport.initialize()); // actual functions -/** - * setTimeout, but its a promise. - * @param {number} ms - */ -async function wait(ms) { - await new Promise(resolve => { - setTimeout(resolve, ms); - }); -} - async function checkMigrations() { // 3.5->3.6 migration const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); @@ -529,7 +519,7 @@ async function backupServerLite() { }); // wait a tiny bit for the zip to reload in fs - await wait(100); + await utils.wait(100); return true; } @@ -597,7 +587,7 @@ async function killAllDownloads() { async function setPortItemFromENV() { config_api.setConfigItem('ytdl_port', backendPort.toString()); - await wait(100); + await utils.wait(100); return true; } @@ -611,7 +601,7 @@ async function setConfigFromEnv() { let success = config_api.setConfigItems(config_items); if (success) { logger.info('Config items set using ENV variables.'); - await wait(100); + await utils.wait(100); return true; } else { logger.error('ERROR: Failed to set config items using ENV variables.'); @@ -847,47 +837,6 @@ function getVideoFormatID(name) } } -async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) { - let zipFolderPath = null; - - if (!fullPathProvided) { - zipFolderPath = (type === 'audio') ? audioFolderPath : videoFolderPath - if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath); - } else { - zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path')); - } - - let ext = (type === 'audio') ? '.mp3' : '.mp4'; - - let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip')); - - var archive = archiver('zip', { - gzip: true, - zlib: { level: 9 } // Sets the compression level. - }); - - archive.on('error', function(err) { - logger.error(err); - throw err; - }); - - // pipe archive data to the output file - archive.pipe(output); - - for (let i = 0; i < fileNames.length; i++) { - let fileName = fileNames[i]; - let fileNamePathRemoved = path.parse(fileName).base; - let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName; - archive.file(file_path, {name: fileNamePathRemoved + ext}) - } - - await archive.finalize(); - - // wait a tiny bit for the zip to reload in fs - await wait(100); - return path.join(zipFolderPath,outputName + '.zip'); -} - // TODO: add to db_api and support multi-user mode async function deleteFile(uid, uuid = null, blacklistMode = false) { const file_obj = await db_api.getVideo(uid, uuid); @@ -2523,18 +2472,19 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { let include_file_metadata = req.body.include_file_metadata; const playlist = await db_api.getPlaylist(playlist_id, uuid); + const file_objs = []; if (playlist && include_file_metadata) { - playlist['file_objs'] = []; for (let i = 0; i < playlist['uids'].length; i++) { const uid = playlist['uids'][i]; const file_obj = await db_api.getVideo(uid, uuid); - playlist['file_objs'].push(file_obj); + file_objs.push(file_obj); } } res.send({ playlist: playlist, + file_objs: file_objs, type: playlist && playlist.type, success: !!playlist }); @@ -2616,32 +2566,47 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => { app.post('/api/downloadFile', optionalJwt, async (req, res) => { let uid = req.body.uid; - let is_playlist = req.body.is_playlist; let uuid = req.body.uuid; + let playlist_id = req.body.playlist_id; + let sub_id = req.body.sub_id; let file_path_to_download = null; if (!uuid && req.user) uuid = req.user.uid; - if (is_playlist) { + + let zip_file_generated = false; + if (playlist_id) { + zip_file_generated = true; const playlist_files_to_download = []; - const playlist = db_api.getPlaylist(uid, uuid); + const playlist = await db_api.getPlaylist(playlist_id, uuid); for (let i = 0; i < playlist['uids'].length; i++) { - const uid = playlist['uids'][i]; - const file_obj = await db_api.getVideo(uid, uuid); - playlist_files_to_download.push(file_obj.path); + const playlist_file_uid = playlist['uids'][i]; + const file_obj = await db_api.getVideo(playlist_file_uid, uuid); + playlist_files_to_download.push(file_obj); } // generate zip - file_path_to_download = await createPlaylistZipFile(playlist_files_to_download, playlist.type, playlist.name); + file_path_to_download = await utils.createContainerZipFile(playlist, playlist_files_to_download); + } else if (sub_id && !uid) { + zip_file_generated = true; + const sub_files_to_download = []; + const sub = subscriptions_api.getSubscription(sub_id, uuid); + for (let i = 0; i < sub['videos'].length; i++) { + const sub_file = sub['videos'][i]; + sub_files_to_download.push(sub_file); + } + + // generate zip + file_path_to_download = await utils.createContainerZipFile(sub, sub_files_to_download); } else { - const file_obj = await db_api.getVideo(uid, uuid) + const file_obj = await db_api.getVideo(uid, uuid, sub_id) file_path_to_download = file_obj.path; } if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download); res.sendFile(file_path_to_download, function (err) { if (err) { logger.error(err); - } else if (is_playlist) { + } else if (zip_file_generated) { try { // delete generated zip file fs.unlinkSync(file_path_to_download); diff --git a/backend/db.js b/backend/db.js index 061280b..106c3f2 100644 --- a/backend/db.js +++ b/backend/db.js @@ -245,7 +245,6 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal if (uid) playlist['uids'].push(uid); else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`); } - delete playlist['fileNames']; exports.updatePlaylist(playlist, user_uid); } diff --git a/backend/test/tests.js b/backend/test/tests.js index c9726a0..9697a36 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -35,13 +35,19 @@ const logger = winston.createLogger({ var auth_api = require('../authentication/auth'); var db_api = require('../db'); +const utils = require('../utils'); +const subscriptions_api = require('../subscriptions'); +const fs = require('fs-extra'); db_api.initialize(db, users_db, logger); auth_api.initialize(db, users_db, logger); +subscriptions_api.initialize(db, users_db, logger, db_api); describe('Multi User', async function() { let user = null; const user_to_test = 'admin'; + const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; + const playlist_to_test = 'ysabVZz4x'; before(async function() { user = await auth_api.login('admin', 'pass'); console.log('hi') @@ -70,6 +76,36 @@ describe('Multi User', async function() { assert(video_obj); }); }); + describe('Zip generators', function() { + it('Playlist zip generator', async function() { + const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test); + assert(playlist); + const playlist_files_to_download = []; + for (let i = 0; i < playlist['uids'].length; i++) { + const uid = playlist['uids'][i]; + const playlist_file = await db_api.getVideo(uid, user_to_test); + playlist_files_to_download.push(playlist_file); + } + const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download); + const zip_exists = fs.pathExistsSync(zip_path); + assert(zip_exists); + if (zip_exists) fs.unlinkSync(zip_path); + }); + + it('Subscription zip generator', async function() { + const sub = subscriptions_api.getSubscription(sub_to_test, user_to_test); + assert(sub); + const sub_files_to_download = []; + for (let i = 0; i < sub['videos'].length; i++) { + const sub_file = sub['videos'][i]; + sub_files_to_download.push(sub_file); + } + const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download); + const zip_exists = fs.pathExistsSync(zip_path); + assert(zip_exists); + if (zip_exists) fs.unlinkSync(zip_path); + }); + }); // describe('Video player - subscription', function() { // const sub_to_test = ''; // const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e'; diff --git a/backend/utils.js b/backend/utils.js index cd7c23d..2b825dd 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -1,6 +1,7 @@ -var fs = require('fs-extra') -var path = require('path') +const fs = require('fs-extra') +const path = require('path') const config_api = require('./config'); +const archiver = require('archiver'); const is_windows = process.platform === 'win32'; @@ -52,6 +53,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) { return files; } +async function createContainerZipFile(container_obj, container_file_objs) { + const container_files_to_download = []; + for (let i = 0; i < container_file_objs.length; i++) { + const container_file_obj = container_file_objs[i]; + container_files_to_download.push(container_file_obj.path); + } + return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download); +} + +async function createZipFile(zip_file_path, file_paths) { + let output = fs.createWriteStream(zip_file_path); + + var archive = archiver('zip', { + gzip: true, + zlib: { level: 9 } // Sets the compression level. + }); + + archive.on('error', function(err) { + logger.error(err); + throw err; + }); + + // pipe archive data to the output file + archive.pipe(output); + + for (let file_path of file_paths) { + const file_name = path.parse(file_path).base; + archive.file(file_path, {name: file_name}) + } + + await archive.finalize(); + + // wait a tiny bit for the zip to reload in fs + await wait(100); + return zip_file_path; +} + function getJSONMp4(name, customPath, openReadPerms = false) { var obj = null; // output if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path'); @@ -193,6 +231,16 @@ function removeFileExtension(filename) { return filename_parts.join('.'); } +/** + * setTimeout, but its a promise. + * @param {number} ms + */ + async function wait(ms) { + await new Promise(resolve => { + setTimeout(resolve, ms); + }); +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { @@ -221,7 +269,9 @@ module.exports = { fixVideoMetadataPerms: fixVideoMetadataPerms, deleteJSONFile: deleteJSONFile, getDownloadedFilesByType: getDownloadedFilesByType, + createContainerZipFile: createContainerZipFile, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, + wait: wait, File: File } diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index 8870a8e..69c7908 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -97,7 +97,7 @@ export class CustomPlaylistsComponent implements OnInit { const index = args.index; const dialogRef = this.dialog.open(ModifyPlaylistComponent, { data: { - playlist: playlist, + playlist_id: playlist.id, width: '65vw' } }); diff --git a/src/app/create-playlist/create-playlist.component.html b/src/app/create-playlist/create-playlist.component.html index d9f108a..ad21ec0 100644 --- a/src/app/create-playlist/create-playlist.component.html +++ b/src/app/create-playlist/create-playlist.component.html @@ -20,8 +20,8 @@ Videos {{file.id}} - {{file.id}} - {{file.id}} + {{file.id}} + {{file.id}} diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.html b/src/app/dialogs/modify-playlist/modify-playlist.component.html index 69f4cad..a8471bb 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.html +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.html @@ -1,38 +1,40 @@

Modify playlist

- -
- - - -
- -
-
- Normal order  - Reverse order  - +
+ +
+ + +
-
- -
-
+
+
+ Normal order  + Reverse order  + +
- - - -
{{playlist_item}}
 - - - - - +
+ +
+
+ + + + +
{{playlist_item.title}}
 + + + + + +
- + \ No newline at end of file diff --git a/src/app/dialogs/modify-playlist/modify-playlist.component.ts b/src/app/dialogs/modify-playlist/modify-playlist.component.ts index 414fc92..161cab8 100644 --- a/src/app/dialogs/modify-playlist/modify-playlist.component.ts +++ b/src/app/dialogs/modify-playlist/modify-playlist.component.ts @@ -10,8 +10,12 @@ import { PostsService } from 'app/posts.services'; }) export class ModifyPlaylistComponent implements OnInit { + playlist_id = null; + original_playlist = null; playlist = null; + playlist_file_objs = null; + available_files = []; all_files = []; playlist_updated = false; @@ -23,9 +27,8 @@ export class ModifyPlaylistComponent implements OnInit { ngOnInit(): void { if (this.data) { - this.playlist = JSON.parse(JSON.stringify(this.data.playlist)); - this.original_playlist = JSON.parse(JSON.stringify(this.data.playlist)); - this.getFiles(); + this.playlist_id = this.data.playlist_id; + this.getPlaylist(); } this.reverse_order = localStorage.getItem('default_playlist_order_reversed') === 'true'; @@ -44,11 +47,12 @@ export class ModifyPlaylistComponent implements OnInit { } processFiles(new_files = null) { - if (new_files) { this.all_files = new_files.map(file => file.id); } - this.available_files = this.all_files.filter(e => !this.playlist.fileNames.includes(e)) + if (new_files) { this.all_files = new_files; } + this.available_files = this.all_files.filter(e => !this.playlist_file_objs.includes(e)) } updatePlaylist() { + this.playlist['uids'] = this.playlist_file_objs.map(playlist_file_obj => playlist_file_obj['uid']) this.postsService.updatePlaylist(this.playlist).subscribe(res => { this.playlist_updated = true; this.postsService.openSnackBar('Playlist updated successfully.'); @@ -61,24 +65,26 @@ export class ModifyPlaylistComponent implements OnInit { } getPlaylist() { - this.postsService.getPlaylist(this.playlist.id, this.playlist.type, null).subscribe(res => { + this.postsService.getPlaylist(this.playlist_id, null, true).subscribe(res => { if (res['playlist']) { this.playlist = res['playlist']; + this.playlist_file_objs = res['file_objs']; this.original_playlist = JSON.parse(JSON.stringify(this.playlist)); + this.getFiles(); } }); } addContent(file) { - this.playlist.fileNames.push(file); + this.playlist_file_objs.push(file); this.processFiles(); } removeContent(index) { if (this.reverse_order) { - index = this.playlist.fileNames.length - 1 - index; + index = this.playlist_file_objs.length - 1 - index; } - this.playlist.fileNames.splice(index, 1); + this.playlist_file_objs.splice(index, 1); this.processFiles(); } @@ -89,10 +95,10 @@ export class ModifyPlaylistComponent implements OnInit { drop(event: CdkDragDrop) { if (this.reverse_order) { - event.previousIndex = this.playlist.fileNames.length - 1 - event.previousIndex; - event.currentIndex = this.playlist.fileNames.length - 1 - event.currentIndex; + event.previousIndex = this.playlist_file_objs.length - 1 - event.previousIndex; + event.currentIndex = this.playlist_file_objs.length - 1 - event.currentIndex; } - moveItemInArray(this.playlist.fileNames, event.previousIndex, event.currentIndex); + moveItemInArray(this.playlist_file_objs, event.previousIndex, event.currentIndex); } } diff --git a/src/app/file-card/file-card.component.ts b/src/app/file-card/file-card.component.ts index 90b906c..5596eec 100644 --- a/src/app/file-card/file-card.component.ts +++ b/src/app/file-card/file-card.component.ts @@ -84,7 +84,7 @@ export class FileCardComponent implements OnInit { editPlaylistDialog() { const dialogRef = this.dialog.open(ModifyPlaylistComponent, { data: { - playlist: this.playlist, + playlist_id: this.playlist.id, width: '65vw' } }); diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index e37d041..cd1e3ab 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -657,7 +657,7 @@ export class MainComponent implements OnInit { } downloadPlaylist(playlist) { - this.postsService.downloadFileFromServer(playlist.id, null, true).subscribe(res => { + this.postsService.downloadPlaylistFromServer(playlist.id).subscribe(res => { if (playlist.id) { this.downloading_content[playlist.type][playlist.id] = false }; const blob: Blob = res; saveAs(blob, playlist.name + '.zip'); diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 601271d..d0dd480 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -187,6 +187,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { this.postsService.getPlaylist(this.playlist_id, this.uuid, true).subscribe(res => { if (res['playlist']) { this.db_playlist = res['playlist']; + this.db_playlist['file_objs'] = res['file_objs']; this.uids = this.db_playlist.uids; this.type = res['type']; this.show_player = true; @@ -316,7 +317,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0]; this.downloading = true; - this.postsService.downloadFileFromServer(this.playlist_id, this.uuid, true).subscribe(res => { + this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => { this.downloading = false; const blob: Blob = res; saveAs(blob, zipName + '.zip'); @@ -330,7 +331,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { const ext = (this.type === 'audio') ? '.mp3' : '.mp4'; const filename = this.playlist[0].title; this.downloading = true; - this.postsService.downloadFileFromServer(this.uid, this.uuid, false).subscribe(res => { + this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id).subscribe(res => { this.downloading = false; const blob: Blob = res; saveAs(blob, filename + ext); diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index cf04447..f0fc20e 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -247,17 +247,29 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions); } - downloadFileFromServer(uid, uuid = null, is_playlist = false) { + downloadFileFromServer(uid, uuid = null, sub_id = null) { return this.http.post(this.path + 'downloadFile', { uid: uid, uuid: uuid, - is_playlist: is_playlist + sub_id: sub_id }, {responseType: 'blob', params: this.httpOptions.params}); } downloadPlaylistFromServer(playlist_id, uuid = null) { - return this.http.post(this.path + 'downloadPlaylist', {playlist_id: playlist_id, uuid: uuid}); + return this.http.post(this.path + 'downloadFile', { + uuid: uuid, + playlist_id: playlist_id + }, + {responseType: 'blob', params: this.httpOptions.params}); + } + + downloadSubFromServer(sub_id, uuid = null) { + return this.http.post(this.path + 'downloadFile', { + uuid: uuid, + sub_id: sub_id + }, + {responseType: 'blob', params: this.httpOptions.params}); } checkConcurrentStream(uid) { diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index af08088..cb40b1c 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -153,15 +153,14 @@ export class SubscriptionComponent implements OnInit, OnDestroy { } this.downloading = true; - // TODO: add download subscription route - /*this.postsService.downloadFileFromServer(fileNames, 'video', this.subscription.name, true).subscribe(res => { + this.postsService.downloadSubFromServer(this.subscription.id).subscribe(res => { this.downloading = false; const blob: Blob = res; saveAs(blob, this.subscription.name + '.zip'); }, err => { console.log(err); this.downloading = false; - });*/ + }); } editSubscription() {