Migrated playlist and subscription (per video and sub-wide) video downloading functionality to new schema

Migrated modify playlist component to new schema

Moved wait function and playlist generation function(s) to utils
- added tests for zip generation

backend/app.js

@@ -193,16 +193,6 @@ app.use(auth_api.passport.initialize());
 
 // actual functions
 
-/**
- * setTimeout, but its a promise.
- * @param {number} ms
- */
-async function wait(ms) {
-    await new Promise(resolve => {
-        setTimeout(resolve, ms);
-    });
-}
-
 async function checkMigrations() {
     // 3.5->3.6 migration
     const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
@@ -529,7 +519,7 @@ async function backupServerLite() {
     });
 
     // wait a tiny bit for the zip to reload in fs
-    await wait(100);
+    await utils.wait(100);
     return true;
 }
 
@@ -597,7 +587,7 @@ async function killAllDownloads() {
 
 async function setPortItemFromENV() {
     config_api.setConfigItem('ytdl_port', backendPort.toString());
-    await wait(100);
+    await utils.wait(100);
     return true;
 }
 
@@ -611,7 +601,7 @@ async function setConfigFromEnv() {
     let success = config_api.setConfigItems(config_items);
     if (success) {
         logger.info('Config items set using ENV variables.');
-        await wait(100);
+        await utils.wait(100);
         return true;
     } else {
         logger.error('ERROR: Failed to set config items using ENV variables.');
@@ -847,47 +837,6 @@ function getVideoFormatID(name)
     }
 }
 
-async function createPlaylistZipFile(fileNames, type, outputName, fullPathProvided = null, user_uid = null) {
-    let zipFolderPath = null;
-
-    if (!fullPathProvided) {
-        zipFolderPath = (type === 'audio') ? audioFolderPath : videoFolderPath
-        if (user_uid) zipFolderPath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, zipFolderPath);
-    } else {
-        zipFolderPath = path.join(__dirname, config_api.getConfigItem('ytdl_subscriptions_base_path'));
-    }
-
-    let ext = (type === 'audio') ? '.mp3' : '.mp4';
-
-    let output = fs.createWriteStream(path.join(zipFolderPath, outputName + '.zip'));
-
-    var archive = archiver('zip', {
-        gzip: true,
-        zlib: { level: 9 } // Sets the compression level.
-    });
-
-    archive.on('error', function(err) {
-        logger.error(err);
-        throw err;
-    });
-
-    // pipe archive data to the output file
-    archive.pipe(output);
-
-    for (let i = 0; i < fileNames.length; i++) {
-        let fileName = fileNames[i];
-        let fileNamePathRemoved = path.parse(fileName).base;
-        let file_path = !fullPathProvided ? path.join(zipFolderPath, fileName + ext) : fileName;
-        archive.file(file_path, {name: fileNamePathRemoved + ext})
-    }
-
-    await archive.finalize();
-
-    // wait a tiny bit for the zip to reload in fs
-    await wait(100);
-    return path.join(zipFolderPath,outputName + '.zip');
-}
-
 // TODO: add to db_api and support multi-user mode
 async function deleteFile(uid, uuid = null, blacklistMode = false) {
     const file_obj = await db_api.getVideo(uid, uuid);
@@ -2523,18 +2472,19 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
     let include_file_metadata = req.body.include_file_metadata;
 
     const playlist = await db_api.getPlaylist(playlist_id, uuid);
+    const file_objs = [];
 
     if (playlist && include_file_metadata) {
-        playlist['file_objs'] = [];
         for (let i = 0; i < playlist['uids'].length; i++) {
             const uid = playlist['uids'][i];
             const file_obj = await db_api.getVideo(uid, uuid);
-            playlist['file_objs'].push(file_obj);
+            file_objs.push(file_obj);
         }
     }
 
     res.send({
         playlist: playlist,
+        file_objs: file_objs,
         type: playlist && playlist.type,
         success: !!playlist
     });
@@ -2616,32 +2566,47 @@ app.post('/api/deleteFile', optionalJwt, async (req, res) => {
 
 app.post('/api/downloadFile', optionalJwt, async (req, res) => {
     let uid = req.body.uid;
-    let is_playlist = req.body.is_playlist;
     let uuid = req.body.uuid;
+    let playlist_id = req.body.playlist_id;
+    let sub_id = req.body.sub_id;
 
     let file_path_to_download = null;
 
     if (!uuid && req.user) uuid = req.user.uid;
-    if (is_playlist) {
+
+    let zip_file_generated = false;
+    if (playlist_id) {
+        zip_file_generated = true;
         const playlist_files_to_download = [];
-        const playlist = db_api.getPlaylist(uid, uuid);
+        const playlist = await db_api.getPlaylist(playlist_id, uuid);
         for (let i = 0; i < playlist['uids'].length; i++) {
-            const uid = playlist['uids'][i];
+            const playlist_file_uid = playlist['uids'][i];
-            const file_obj = await db_api.getVideo(uid, uuid);
+            const file_obj = await db_api.getVideo(playlist_file_uid, uuid);
-            playlist_files_to_download.push(file_obj.path);
+            playlist_files_to_download.push(file_obj);
         }
 
         // generate zip
-        file_path_to_download = await createPlaylistZipFile(playlist_files_to_download, playlist.type, playlist.name);
+        file_path_to_download = await utils.createContainerZipFile(playlist, playlist_files_to_download);
+    } else if (sub_id && !uid) {
+        zip_file_generated = true;
+        const sub_files_to_download = [];
+        const sub = subscriptions_api.getSubscription(sub_id, uuid);
+        for (let i = 0; i < sub['videos'].length; i++) {
+            const sub_file = sub['videos'][i];
+            sub_files_to_download.push(sub_file);
+        }
+
+        // generate zip
+        file_path_to_download = await utils.createContainerZipFile(sub, sub_files_to_download);
     } else {
-        const file_obj = await db_api.getVideo(uid, uuid)
+        const file_obj = await db_api.getVideo(uid, uuid, sub_id)
         file_path_to_download = file_obj.path;
     }
     if (!path.isAbsolute(file_path_to_download)) file_path_to_download = path.join(__dirname, file_path_to_download);
     res.sendFile(file_path_to_download, function (err) {
         if (err) {
           logger.error(err);
-        } else if (is_playlist) {
+        } else if (zip_file_generated) {
           try {
             // delete generated zip file
             fs.unlinkSync(file_path_to_download);

backend/db.js

@@ -245,7 +245,6 @@ exports.getPlaylist = async (playlist_id, user_uid = null, require_sharing = fal
             if (uid) playlist['uids'].push(uid);
             else logger.warn(`Failed to convert file with name ${fileName} to its UID while converting playlist ${playlist['name']} to the new UID-based schema. The original file is likely missing/deleted and it will be skipped.`);
         }
-        delete playlist['fileNames'];
         exports.updatePlaylist(playlist, user_uid);
     }
 

backend/test/tests.js

@@ -35,13 +35,19 @@ const logger = winston.createLogger({
 
 var auth_api = require('../authentication/auth');
 var db_api = require('../db');
+const utils = require('../utils');
+const subscriptions_api = require('../subscriptions');
+const fs = require('fs-extra');
 
 db_api.initialize(db, users_db, logger);
 auth_api.initialize(db, users_db, logger);
+subscriptions_api.initialize(db, users_db, logger, db_api);
 
 describe('Multi User', async function() {
     let user = null;
     const user_to_test = 'admin';
+    const sub_to_test = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
+    const playlist_to_test = 'ysabVZz4x';
     before(async function() {
         user = await auth_api.login('admin', 'pass');
         console.log('hi')
@@ -70,6 +76,36 @@ describe('Multi User', async function() {
             assert(video_obj);
         });
     });
+    describe('Zip generators', function() {
+        it('Playlist zip generator', async function() {
+            const playlist = await db_api.getPlaylist(playlist_to_test, user_to_test);
+            assert(playlist);
+            const playlist_files_to_download = [];
+            for (let i = 0; i < playlist['uids'].length; i++) {
+                const uid = playlist['uids'][i];
+                const playlist_file = await db_api.getVideo(uid, user_to_test);
+                playlist_files_to_download.push(playlist_file);
+            }
+            const zip_path = await utils.createContainerZipFile(playlist, playlist_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+
+        it('Subscription zip generator', async function() {
+            const sub = subscriptions_api.getSubscription(sub_to_test, user_to_test);
+            assert(sub);
+            const sub_files_to_download = [];
+            for (let i = 0; i < sub['videos'].length; i++) {
+                const sub_file = sub['videos'][i];
+                sub_files_to_download.push(sub_file);
+            }
+            const zip_path = await utils.createContainerZipFile(sub, sub_files_to_download);
+            const zip_exists = fs.pathExistsSync(zip_path);
+            assert(zip_exists);
+            if (zip_exists) fs.unlinkSync(zip_path);
+        });
+    });
     // describe('Video player - subscription', function() {
     //     const sub_to_test = '';
     //     const video_to_test = 'ebbcfffb-d6f1-4510-ad25-d1ec82e0477e';

backend/utils.js

@@ -1,6 +1,7 @@
-var fs = require('fs-extra')
+const fs = require('fs-extra')
-var path = require('path')
+const path = require('path')
 const config_api = require('./config');
+const archiver = require('archiver');
 
 const is_windows = process.platform === 'win32';
 
@@ -52,6 +53,43 @@ async function getDownloadedFilesByType(basePath, type, full_metadata = false) {
     return files;
 }
 
+async function createContainerZipFile(container_obj, container_file_objs) {
+    const container_files_to_download = [];
+    for (let i = 0; i < container_file_objs.length; i++) {
+        const container_file_obj = container_file_objs[i];
+        container_files_to_download.push(container_file_obj.path);
+    }
+    return await createZipFile(path.join('appdata', container_obj.name + '.zip'), container_files_to_download);
+}
+
+async function createZipFile(zip_file_path, file_paths) {
+    let output = fs.createWriteStream(zip_file_path);
+
+    var archive = archiver('zip', {
+        gzip: true,
+        zlib: { level: 9 } // Sets the compression level.
+    });
+
+    archive.on('error', function(err) {
+        logger.error(err);
+        throw err;
+    });
+
+    // pipe archive data to the output file
+    archive.pipe(output);
+
+    for (let file_path of file_paths) {
+        const file_name = path.parse(file_path).base;
+        archive.file(file_path, {name: file_name})
+    }
+
+    await archive.finalize();
+
+    // wait a tiny bit for the zip to reload in fs
+    await wait(100);
+    return zip_file_path;
+}
+
 function getJSONMp4(name, customPath, openReadPerms = false) {
     var obj = null; // output
     if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path');
@@ -193,6 +231,16 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
+/**
+ * setTimeout, but its a promise.
+ * @param {number} ms
+ */
+ async function wait(ms) {
+    await new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@@ -221,7 +269,9 @@ module.exports = {
     fixVideoMetadataPerms: fixVideoMetadataPerms,
     deleteJSONFile: deleteJSONFile,
     getDownloadedFilesByType: getDownloadedFilesByType,
+    createContainerZipFile: createContainerZipFile,
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
+    wait: wait,
     File: File
 }

src/app/components/custom-playlists/custom-playlists.component.ts

@@ -97,7 +97,7 @@ export class CustomPlaylistsComponent implements OnInit {
     const index = args.index;
     const dialogRef = this.dialog.open(ModifyPlaylistComponent, {
       data: {
-        playlist: playlist,
+        playlist_id: playlist.id,
         width: '65vw'
       }
     });

src/app/create-playlist/create-playlist.component.html

@@ -20,8 +20,8 @@
                 <mat-label *ngIf="type === 'video'"><ng-container i18n="Videos title">Videos</ng-container></mat-label>
                 <mat-select [formControl]="filesSelect" multiple required aria-required>
                     <ng-container *ngIf="filesToSelectFrom"><mat-option *ngFor="let file of filesToSelectFrom" [value]="file.uid">{{file.id}}</mat-option></ng-container>
-                    <ng-container *ngIf="audiosToSelectFrom && type === 'audio'"><mat-option *ngFor="let file of audiosToSelectFrom" [value]="file.id">{{file.id}}</mat-option></ng-container>
+                    <ng-container *ngIf="audiosToSelectFrom && type === 'audio'"><mat-option *ngFor="let file of audiosToSelectFrom" [value]="file.uid">{{file.id}}</mat-option></ng-container>
-                    <ng-container *ngIf="videosToSelectFrom && type === 'video'"><mat-option *ngFor="let file of videosToSelectFrom" [value]="file.id">{{file.id}}</mat-option></ng-container>
+                    <ng-container *ngIf="videosToSelectFrom && type === 'video'"><mat-option *ngFor="let file of videosToSelectFrom" [value]="file.uid">{{file.id}}</mat-option></ng-container>
                 </mat-select>
             </mat-form-field>
             <!-- No videos available -->

src/app/dialogs/modify-playlist/modify-playlist.component.html

@@ -1,6 +1,7 @@
 <h4 mat-dialog-title><ng-container i18n="Modify playlist dialog title">Modify playlist</ng-container></h4>
 
 <mat-dialog-content>
+    <div *ngIf="playlist">
         <!-- Playlist info -->
         <div>
             <mat-form-field color="accent">
@@ -23,16 +24,17 @@
         <!-- Playlist order -->
         <mat-button-toggle-group class="media-list" cdkDropList (cdkDropListDropped)="drop($event)" style="width: 80%; left: 9%" vertical name="videoSelect" aria-label="Video Select" #group="matButtonToggleGroup">
             <!-- The following for loop can be optimized but it requires a pipe https://stackoverflow.com/a/35703364/8088021 -->
-        <mat-button-toggle class="media-box" cdkDrag *ngFor="let playlist_item of (reverse_order ? playlist.fileNames.slice().reverse() : playlist.fileNames); let i = index" [checked]="false"><div><div class="playlist-item-text">{{playlist_item}}</div> <button (click)="removeContent(i)" class="remove-item-button" mat-icon-button><mat-icon>cancel</mat-icon></button></div></mat-button-toggle>
+            <mat-button-toggle class="media-box" cdkDrag *ngFor="let playlist_item of (reverse_order ? playlist_file_objs.slice().reverse() : playlist_file_objs); let i = index" [checked]="false"><div><div class="playlist-item-text">{{playlist_item.title}}</div> <button (click)="removeContent(i)" class="remove-item-button" mat-icon-button><mat-icon>cancel</mat-icon></button></div></mat-button-toggle>
         </mat-button-toggle-group>
         
         <mat-menu #menu="matMenu">
-        <button *ngFor="let file of available_files" (click)="addContent(file)" mat-menu-item>{{file}}</button>
+            <button *ngFor="let file of available_files" (click)="addContent(file)" mat-menu-item>{{file.title}}</button>
         </mat-menu>
+    </div>
 
 </mat-dialog-content>
 
 <mat-dialog-actions>
     <!-- Save -->
-    <button [disabled]="!playlistChanged()" (click)="updatePlaylist()" [disabled]="playlistChanged()" mat-raised-button color="accent"><ng-container i18n="Save">Save</ng-container></button>
+    <button [disabled]="!playlist || !playlistChanged()" (click)="updatePlaylist()" mat-raised-button color="accent"><ng-container i18n="Save">Save</ng-container></button>
 </mat-dialog-actions>

src/app/dialogs/modify-playlist/modify-playlist.component.ts

@@ -10,8 +10,12 @@ import { PostsService } from 'app/posts.services';
 })
 export class ModifyPlaylistComponent implements OnInit {
 
+  playlist_id = null;
+
   original_playlist = null;
   playlist = null;
+  playlist_file_objs = null;
+
   available_files = [];
   all_files = [];
   playlist_updated = false;
@@ -23,9 +27,8 @@ export class ModifyPlaylistComponent implements OnInit {
 
   ngOnInit(): void {
     if (this.data) {
-      this.playlist = JSON.parse(JSON.stringify(this.data.playlist));
+      this.playlist_id = this.data.playlist_id;
-      this.original_playlist = JSON.parse(JSON.stringify(this.data.playlist));
+      this.getPlaylist();
-      this.getFiles();
     }
 
     this.reverse_order = localStorage.getItem('default_playlist_order_reversed') === 'true';
@@ -44,11 +47,12 @@ export class ModifyPlaylistComponent implements OnInit {
   }
 
   processFiles(new_files = null) {
-    if (new_files) { this.all_files = new_files.map(file => file.id); }
+    if (new_files) { this.all_files = new_files; }
-    this.available_files = this.all_files.filter(e => !this.playlist.fileNames.includes(e))
+    this.available_files = this.all_files.filter(e => !this.playlist_file_objs.includes(e))
   }
 
   updatePlaylist() {
+    this.playlist['uids'] = this.playlist_file_objs.map(playlist_file_obj => playlist_file_obj['uid'])
     this.postsService.updatePlaylist(this.playlist).subscribe(res => {
       this.playlist_updated = true;
       this.postsService.openSnackBar('Playlist updated successfully.');
@@ -61,24 +65,26 @@ export class ModifyPlaylistComponent implements OnInit {
   }
 
   getPlaylist() {
-    this.postsService.getPlaylist(this.playlist.id, this.playlist.type, null).subscribe(res => {
+    this.postsService.getPlaylist(this.playlist_id, null, true).subscribe(res => {
       if (res['playlist']) {
         this.playlist = res['playlist'];
+        this.playlist_file_objs = res['file_objs'];
         this.original_playlist = JSON.parse(JSON.stringify(this.playlist));
+        this.getFiles();
       }
     });
   }
 
   addContent(file) {
-    this.playlist.fileNames.push(file);
+    this.playlist_file_objs.push(file);
     this.processFiles();
   }
 
   removeContent(index) {
     if (this.reverse_order) {
-      index = this.playlist.fileNames.length - 1 - index;
+      index = this.playlist_file_objs.length - 1 - index;
     }
-    this.playlist.fileNames.splice(index, 1);
+    this.playlist_file_objs.splice(index, 1);
     this.processFiles();
   }
 
@@ -89,10 +95,10 @@ export class ModifyPlaylistComponent implements OnInit {
 
   drop(event: CdkDragDrop<string[]>) {
     if (this.reverse_order) {
-      event.previousIndex = this.playlist.fileNames.length - 1 - event.previousIndex;
+      event.previousIndex = this.playlist_file_objs.length - 1 - event.previousIndex;
-      event.currentIndex = this.playlist.fileNames.length - 1 - event.currentIndex;
+      event.currentIndex = this.playlist_file_objs.length - 1 - event.currentIndex;
     }
-    moveItemInArray(this.playlist.fileNames, event.previousIndex, event.currentIndex);
+    moveItemInArray(this.playlist_file_objs, event.previousIndex, event.currentIndex);
   }
 
 }

src/app/file-card/file-card.component.ts

@@ -84,7 +84,7 @@ export class FileCardComponent implements OnInit {
   editPlaylistDialog() {
     const dialogRef = this.dialog.open(ModifyPlaylistComponent, {
       data: {
-        playlist: this.playlist,
+        playlist_id: this.playlist.id,
         width: '65vw'
       }
     });

src/app/main/main.component.ts

@@ -657,7 +657,7 @@ export class MainComponent implements OnInit {
   }
 
   downloadPlaylist(playlist) {
-    this.postsService.downloadFileFromServer(playlist.id, null, true).subscribe(res => {
+    this.postsService.downloadPlaylistFromServer(playlist.id).subscribe(res => {
       if (playlist.id) { this.downloading_content[playlist.type][playlist.id] = false };
       const blob: Blob = res;
       saveAs(blob, playlist.name + '.zip');

src/app/player/player.component.ts

@@ -187,6 +187,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
     this.postsService.getPlaylist(this.playlist_id, this.uuid, true).subscribe(res => {
       if (res['playlist']) {
         this.db_playlist = res['playlist'];
+        this.db_playlist['file_objs'] = res['file_objs'];
         this.uids = this.db_playlist.uids;
         this.type = res['type'];
         this.show_player = true;
@@ -316,7 +317,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
 
     const zipName = fileNames[0].split(' ')[0] + fileNames[1].split(' ')[0];
     this.downloading = true;
-    this.postsService.downloadFileFromServer(this.playlist_id, this.uuid, true).subscribe(res => {
+    this.postsService.downloadPlaylistFromServer(this.playlist_id, this.uuid).subscribe(res => {
       this.downloading = false;
       const blob: Blob = res;
       saveAs(blob, zipName + '.zip');
@@ -330,7 +331,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
     const ext = (this.type === 'audio') ? '.mp3' : '.mp4';
     const filename = this.playlist[0].title;
     this.downloading = true;
-    this.postsService.downloadFileFromServer(this.uid, this.uuid, false).subscribe(res => {
+    this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id).subscribe(res => {
       this.downloading = false;
       const blob: Blob = res;
       saveAs(blob, filename + ext);

src/app/posts.services.ts

@@ -247,17 +247,29 @@ export class PostsService implements CanActivate {
         return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid, sub: sub}, this.httpOptions);
     }
 
-    downloadFileFromServer(uid, uuid = null, is_playlist = false) {
+    downloadFileFromServer(uid, uuid = null, sub_id = null) {
         return this.http.post(this.path + 'downloadFile', {
                                                             uid: uid,
                                                             uuid: uuid,
-                                                            is_playlist: is_playlist
+                                                            sub_id: sub_id
                                                             },
                                                           {responseType: 'blob', params: this.httpOptions.params});
     }
 
     downloadPlaylistFromServer(playlist_id, uuid = null) {
-        return this.http.post(this.path + 'downloadPlaylist', {playlist_id: playlist_id, uuid: uuid});
+        return this.http.post(this.path + 'downloadFile', {
+                                                            uuid: uuid,
+                                                            playlist_id: playlist_id
+                                                            },
+                                                          {responseType: 'blob', params: this.httpOptions.params});
+    }
+
+    downloadSubFromServer(sub_id, uuid = null) {
+        return this.http.post(this.path + 'downloadFile', {
+                                                            uuid: uuid,
+                                                            sub_id: sub_id
+                                                            },
+                                                          {responseType: 'blob', params: this.httpOptions.params});
     }
 
     checkConcurrentStream(uid) {

src/app/subscription/subscription/subscription.component.ts

@@ -153,15 +153,14 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
     }
 
     this.downloading = true;
-    // TODO: add download subscription route
+    this.postsService.downloadSubFromServer(this.subscription.id).subscribe(res => {
-    /*this.postsService.downloadFileFromServer(fileNames, 'video', this.subscription.name, true).subscribe(res => {
       this.downloading = false;
       const blob: Blob = res;
       saveAs(blob, this.subscription.name + '.zip');
     }, err => {
       console.log(err);
       this.downloading = false;
-    });*/
+    });
   }
 
   editSubscription() {