Added additional protections to verify that the DB is initialized before downloader does

Began work on watching entire subscriptions as a playlist

Subscriptions now use the new download manager to download files

backend/app.js

@@ -61,7 +61,7 @@ config_api.initialize();
 db_api.initialize(db, users_db);
 auth_api.initialize(db_api);
 downloader_api.initialize(db_api);
-subscriptions_api.initialize(db_api);
+subscriptions_api.initialize(db_api, downloader_api);
 categories_api.initialize(db_api);
 
 // Set some defaults
@@ -533,6 +533,8 @@ async function loadConfig() {
 
     // connect to DB
     await db_api.connectToDB();
+    db_api.database_initialized = true;
+    db_api.database_initialized_bs.next(true);
 
     // creates archive path if missing
     await fs.ensureDir(archivePath);

backend/db.js

@@ -9,10 +9,13 @@ const logger = require('./logger');
 
 const low = require('lowdb')
 const FileSync = require('lowdb/adapters/FileSync');
+const { BehaviorSubject } = require('rxjs');
 const local_adapter = new FileSync('./appdata/local_db.json');
 const local_db = low(local_adapter);
 
 let database = null;
+exports.database_initialized = false;
+exports.database_initialized_bs = new BehaviorSubject(false);
 
 const tables = {
     files: {

backend/downloader.js

@@ -26,16 +26,24 @@ function setDB(input_db_api) { db_api = input_db_api }
 exports.initialize = (input_db_api) => {
     setDB(input_db_api);
     categories_api.initialize(db_api);
-    setupDownloads();
+    if (db_api.database_initialized) {
+        setupDownloads();
+    } else {
+        db_api.database_initialized_bs.subscribe(init => {
+            if (init) setupDownloads();
+        });
+    }
 }
 
-exports.createDownload = async (url, type, options, user_uid = null) => {
+exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {
     return await mutex.runExclusive(async () => {
         const download = {
             url: url,
             type: type,
             title: '',
             user_uid: user_uid,
+            sub_id: sub_id,
+            sub_name: sub_name,
             options: options,
             uid: uuid(),
             step_index: 0,
@@ -116,7 +124,7 @@ async function setupDownloads() {
 async function fixDownloadState() {
     const downloads = await db_api.getRecords('download_queue');
     downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start);
-    const running_downloads = downloads.filter(download => !download['finished_step']);
+    const running_downloads = downloads.filter(download => !download['finished'] && !download['error']);
     for (let i = 0; i < running_downloads.length; i++) {
         const running_download = running_downloads[i];
         const update_obj = {finished_step: true, paused: true};
@@ -143,7 +151,7 @@ async function checkDownloads() {
         return;
     });
 
-    const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step']);
+    const waiting_downloads = downloads.filter(download => !download['paused'] && download['finished_step'] && !download['finished']);
     for (let i = 0; i < waiting_downloads.length; i++) {
         const running_download = waiting_downloads[i];
         if (i === 5/*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/) break;
@@ -322,7 +330,7 @@ async function downloadQueuedFile(download_uid) {
                     }
 
                     // registers file in DB
-                    const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, null, options.cropFileSettings);
+                    const file_obj = await db_api.registerFileDB2(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
 
                     file_objs.push(file_obj);
                 }
@@ -437,13 +445,20 @@ async function generateArgs(url, type, options, user_uid = null) {
 
         let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
         if (useYoutubeDLArchive) {
-            const archive_folder = user_uid ? path.join(fileFolderPath, 'archives') : archivePath;
+            let archive_folder = null;
+            if (options.customArchivePath) {
+                archive_folder = path.join(options.customArchivePath);
+            } else if (user_uid) {
+                archive_folder = path.join(fileFolderPath, 'archives');
+            } else {
+                archive_folder = path.join(archivePath);
+            }
             const archive_path = path.join(archive_folder, `archive_${type}.txt`);
 
             await fs.ensureDir(archive_folder);
             await fs.ensureFile(archive_path);
 
-            let blacklist_path = user_uid ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
+            let blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
             await fs.ensureFile(blacklist_path);
 
             let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
@@ -471,6 +486,10 @@ async function generateArgs(url, type, options, user_uid = null) {
             downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
         }
 
+        if (options.additionalArgs && options.additionalArgs !== '') {
+            downloadConfig = downloadConfig.concat(options.additionalArgs.split(',,'));
+        }
+
         const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
         if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
             downloadConfig.push('-r', rate_limit);

backend/subscriptions.js

@@ -10,11 +10,13 @@ const logger = require('./logger');
 const debugMode = process.env.YTDL_MODE === 'debug';
 
 let db_api = null;
+let downloader_api = null;
 
 function setDB(input_db_api) { db_api = input_db_api }
 
-function initialize(input_db_api) {
+function initialize(input_db_api, input_downloader_api) {
     setDB(input_db_api);
+    downloader_api = input_downloader_api;
 }
 
 async function subscribe(sub, user_uid = null) {
@@ -107,22 +109,6 @@ async function getSubscriptionInfo(sub, user_uid = null) {
                         }
                     }
 
-                    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
-                    if (useArchive && !sub.archive) {
-                        // must create the archive
-                        const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
-                        const archive_path = path.join(archive_dir, 'archive.txt');
-
-                        // creates archive directory and text file if it doesn't exist
-                        fs.ensureDirSync(archive_dir);
-                        fs.ensureFileSync(archive_path);
-
-                        // updates subscription
-                        sub.archive = archive_dir;
-
-                        await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
-                    }
-
                     // TODO: get even more info
 
                     resolve(true);
@@ -256,30 +242,15 @@ async function getVideosForSub(sub, user_uid = null) {
     let appendedBasePath = getAppendedBasePath(sub, basePath);
     fs.ensureDirSync(appendedBasePath);
 
-    let multiUserMode = null;
-    if (user_uid) {
-        multiUserMode = {
-            user: user_uid,
-            file_path: appendedBasePath
-        }
-    }
-
     const downloadConfig = await generateArgsForSubscription(sub, user_uid);
 
     // get videos
     logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);
 
     return new Promise(async resolve => {
-        const preimported_file_paths = [];
-        const PREIMPORT_INTERVAL = 5000;
-        const preregister_check = setInterval(async () => {
-            if (sub.streamingOnly) return;
-            await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
-        }, PREIMPORT_INTERVAL);
         youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
             // cleanup
             updateSubscriptionProperty(sub, {downloading: false}, user_uid);
-            clearInterval(preregister_check);
 
             logger.verbose('Subscription: finished check for ' + sub.name);
             if (err && !output) {
@@ -287,19 +258,21 @@ async function getVideosForSub(sub, user_uid = null) {
                 if (err.stderr.includes('This video is unavailable')) {
                     logger.info('An error was encountered with at least one video, backup method will be used.')
                     try {
-                        const outputs = err.stdout.split(/\r\n|\r|\n/);
+                        // TODO: reimplement
-                        for (let i = 0; i < outputs.length; i++) {
+
-                            const output = JSON.parse(outputs[i]);
+                        // const outputs = err.stdout.split(/\r\n|\r|\n/);
-                            await handleOutputJSON(sub, output, i === 0, multiUserMode)
+                        // for (let i = 0; i < outputs.length; i++) {
-                            if (err.stderr.includes(output['id']) && archive_path) {
+                        //     const output = JSON.parse(outputs[i]);
-                                // we found a video that errored! add it to the archive to prevent future errors
+                        //     await handleOutputJSON(sub, output, i === 0, multiUserMode)
-                                if (sub.archive) {
+                        //     if (err.stderr.includes(output['id']) && archive_path) {
-                                    archive_dir = sub.archive;
+                        //         // we found a video that errored! add it to the archive to prevent future errors
-                                    archive_path = path.join(archive_dir, 'archive.txt')
+                        //         if (sub.archive) {
-                                    fs.appendFileSync(archive_path, output['id']);
+                        //             archive_dir = sub.archive;
-                                }
+                        //             archive_path = path.join(archive_dir, 'archive.txt')
-                            }
+                        //             fs.appendFileSync(archive_path, output['id']);
-                        }
+                        //         }
+                        //     }
+                        // }
                     } catch(e) {
                         logger.error('Backup method failed. See error below:');
                         logger.error(e);
@@ -312,21 +285,30 @@ async function getVideosForSub(sub, user_uid = null) {
                     resolve(true);
                     return;
                 }
+                const output_jsons = [];
                 for (let i = 0; i < output.length; i++) {
                     let output_json = null;
                     try {
                         output_json = JSON.parse(output[i]);
+                        output_jsons.push(output_json);
                     } catch(e) {
                         output_json = null;
                     }
                     if (!output_json) {
                         continue;
                     }
-
-                    const reset_videos = i === 0;
-                    await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
                 }
 
+                const files_to_download = await getFilesToDownload(sub, output_jsons);
+                const base_download_options = generateOptionsForSubscriptionDownload(sub, user_uid);
+
+                for (let j = 0; j < files_to_download.length; j++) {
+                    const file_to_download = files_to_download[j];
+                    await downloader_api.createDownload(file_to_download['webpage_url'], sub.type || 'video', base_download_options, user_uid, sub.id, sub.name);
+                }
+
+                resolve(files_to_download);
+
                 if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
                     await setFreshUploads(sub, user_uid);
                     checkVideosForFreshUploads(sub, user_uid);
@@ -338,10 +320,28 @@ async function getVideosForSub(sub, user_uid = null) {
     }, err => {
         logger.error(err);
         updateSubscriptionProperty(sub, {downloading: false}, user_uid);
-        clearInterval(preregister_check);
     });
 }
 
+function generateOptionsForSubscriptionDownload(sub, user_uid) {
+    let basePath = null;
+    if (user_uid)
+        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
+    else
+        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
+
+    let default_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
+
+    const base_download_options = {
+        selectedHeight: sub.maxQuality && sub.maxQuality !== 'best' ? sub.maxQuality : null,
+        customFileFolderPath: getAppendedBasePath(sub, basePath),
+        customOutput: sub.custom_output ? `${sub.custom_output}` : `${default_output}`,
+        customArchivePath: path.join(__dirname, basePath, 'archives', sub.name)
+    }
+
+    return base_download_options;
+}
+
 async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
     // get basePath
     let basePath = null;
@@ -363,7 +363,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
         fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
     }
 
-    let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
+    let downloadConfig = ['--dump-json', '-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];
 
     let qualityPath = null;
     if (sub.type && sub.type === 'audio') {
@@ -378,7 +378,7 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
     downloadConfig.push(...qualityPath)
 
     if (sub.custom_args) {
-        customArgsArray = sub.custom_args.split(',,');
+        const customArgsArray = sub.custom_args.split(',,');
         if (customArgsArray.indexOf('-f') !== -1) {
             // if custom args has a custom quality, replce the original quality with that of custom args
             const original_output_index = downloadConfig.indexOf('-f');
@@ -466,6 +466,24 @@ async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_vi
     }
 }
 
+async function getFilesToDownload(sub, output_jsons) {
+    const files_to_download = [];
+    for (let i = 0; i < output_jsons.length; i++) {
+        const output_json = output_jsons[i];
+        const file_missing = !(await db_api.getRecord('files', {sub_id: sub.id, url: output_json['webpage_url']}));
+        if (file_missing) {
+            const file_with_path_exists = await db_api.getRecord('files', {sub_id: sub.id, path: output_json['_filename']});
+            if (file_with_path_exists) {
+                // or maybe just overwrite???
+                logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
+            }
+            files_to_download.push(output_json);
+        }
+    }
+    return files_to_download;
+}
+
+
 async function getSubscriptions(user_uid = null) {
     return await db_api.getRecords('subscriptions', {user_uid: user_uid});
 }

src/app/components/downloads/downloads.component.html

@@ -18,6 +18,19 @@
             </mat-cell>
           </ng-container>
 
+          <!-- Subscription Column -->
+          <ng-container matColumnDef="subscription">
+            <mat-header-cell *matHeaderCellDef> <ng-container i18n="Subscription">Subscription</ng-container> </mat-header-cell>
+            <mat-cell *matCellDef="let element">
+              <ng-container *ngIf="element.sub_name">
+                {{element.sub_name}}
+              </ng-container>
+              <ng-container *ngIf="!element.sub_name">
+                  N/A
+              </ng-container>
+            </mat-cell>
+          </ng-container>
+      
           <!-- Stage Column -->
           <ng-container matColumnDef="stage">
             <mat-header-cell *matHeaderCellDef> <ng-container i18n="Stage">Stage</ng-container> </mat-header-cell>
@@ -41,15 +54,17 @@
           <ng-container matColumnDef="actions">
             <mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
             <mat-cell *matCellDef="let element">
-                <div *ngIf="!element.finished">
+                <div>
+                  <ng-container *ngIf="!element.finished">
                     <button (click)="pauseDownload(element.uid)" *ngIf="!element.paused || !element.finished_step" [disabled]="element.paused && !element.finished_step" mat-icon-button matTooltip="Pause" i18n-matTooltip="Pause"><mat-spinner [diameter]="28" *ngIf="element.paused && !element.finished_step" class="icon-button-spinner"></mat-spinner><mat-icon>pause</mat-icon></button>
                     <button (click)="resumeDownload(element.uid)" *ngIf="element.paused && element.finished_step" mat-icon-button matTooltip="Resume" i18n-matTooltip="Resume"><mat-icon>play_arrow</mat-icon></button>
-                    <button (click)="cancelDownload(element.uid)" mat-icon-button matTooltip="Cancel" i18n-matTooltip="Cancel"><mat-icon>cancel</mat-icon></button>
+                    <button *ngIf="!element.paused" (click)="cancelDownload(element.uid)" mat-icon-button matTooltip="Cancel" i18n-matTooltip="Cancel"><mat-icon>cancel</mat-icon></button>
-                </div>
+                  </ng-container>
-                <div *ngIf="element.finished">
+                  <ng-container *ngIf="element.finished">
                     <button (click)="watchContent(element)" mat-icon-button matTooltip="Watch content" i18n-matTooltip="Watch content"><mat-icon>smart_display</mat-icon></button>
                     <button (click)="restartDownload(element.uid)" mat-icon-button matTooltip="Restart" i18n-matTooltip="Restart"><mat-icon>restart_alt</mat-icon></button>
-                    <button (click)="clearDownload(element.uid)" mat-icon-button matTooltip="Clear" i18n-matTooltip="Clear"><mat-icon>delete</mat-icon></button>
+                  </ng-container>
+                  <button *ngIf="element.finished || element.paused" (click)="clearDownload(element.uid)" mat-icon-button matTooltip="Clear" i18n-matTooltip="Clear"><mat-icon>delete</mat-icon></button>
                 </div>
             </mat-cell>
           </ng-container>

src/app/components/downloads/downloads.component.ts

@@ -52,7 +52,7 @@ export class DownloadsComponent implements OnInit, OnDestroy {
       3: 'Complete'
   }
 
-  displayedColumns: string[] = ['date', 'title', 'stage', 'progress', 'actions'];
+  displayedColumns: string[] = ['date', 'title', 'stage', 'subscription', 'progress', 'actions'];
   dataSource = null; // new MatTableDataSource<Download>();
   downloads_retrieved = false;
 

src/app/main/main.component.ts

@@ -246,11 +246,6 @@ export class MainComponent implements OnInit {
     this.useDefaultDownloadingAgent = this.postsService.config['Advanced']['use_default_downloading_agent'];
     this.customDownloadingAgent = this.postsService.config['Advanced']['custom_downloading_agent'];
 
-    if (this.youtubeSearchEnabled && this.youtubeAPIKey) {
-      this.youtubeSearch.initializeAPI(this.youtubeAPIKey);
-      this.attachToInput();
-    }
-
     // set final cache items
 
     localStorage.setItem('cached_filemanager_enabled', this.fileManagerEnabled.toString());
@@ -330,6 +325,13 @@ export class MainComponent implements OnInit {
     this.setCols();
   }
 
+  ngAfterViewInit() {
+    if (this.youtubeSearchEnabled && this.youtubeAPIKey) {
+      this.youtubeSearch.initializeAPI(this.youtubeAPIKey);
+      this.attachToInput();
+    }
+  }
+
   public setCols() {
     if (window.innerWidth <= 350) {
       this.files_cols = 1;

src/app/player/player.component.ts

@@ -166,18 +166,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
       const subscription = res['subscription'];
       this.subscription = subscription;
       this.type === this.subscription.type;
-      subscription.videos.forEach(video => {
+      this.uids = this.subscription.videos.map(video => video['uid']);
-        if (video['uid'] === this.uid) {
+      this.parseFileNames();
-          this.db_file = video;
-          this.postsService.incrementViewCount(this.db_file['uid'], this.sub_id, this.uuid).subscribe(res => {}, err => {
-            console.error('Failed to increment view count');
-            console.error(err);
-          });
-          this.uids = [this.db_file['uid']];
-          this.show_player = true;
-          this.parseFileNames();
-        }
-      });
     }, err => {
       this.openSnackBar(`Failed to find subscription ${this.sub_id}`, 'Dismiss');
     });
@@ -205,7 +195,14 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
     for (let i = 0; i < this.uids.length; i++) {
       const uid = this.uids[i];
 
-      const file_obj = this.playlist_id ? this.db_playlist['file_objs'][i] : this.db_file;
+      let file_obj = null;
+      if (this.playlist_id) {
+        file_obj = this.db_playlist['file_objs'][i];
+      } else if (this.sub_id) {
+        file_obj = this.subscription['videos'][i];
+      } else {
+        file_obj = this.db_file;
+      }
 
       const mime_type = file_obj.isAudio ? 'audio/mp3' : 'video/mp4' 
 

src/app/subscription/subscription/subscription.component.html

@@ -44,5 +44,6 @@
         </div>
     </div>
     <button class="edit-button" color="primary" (click)="editSubscription()" [disabled]="downloading" mat-fab><mat-icon class="save-icon">edit</mat-icon></button>
+    <button class="watch-button" color="primary" (click)="watchSubscription()" mat-fab><mat-icon class="save-icon">video_library</mat-icon></button>
     <button class="save-button" color="primary" (click)="downloadContent()" [disabled]="downloading" mat-fab><mat-icon class="save-icon">save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="50"></mat-spinner></button>
 </div>

src/app/subscription/subscription/subscription.component.scss

@@ -68,3 +68,9 @@
     bottom: 1px;
     position: relative;
 }
+
+.watch-button {
+    left: 90px;
+    position: fixed;
+    bottom: 25px;
+}

src/app/subscription/subscription/subscription.component.ts

@@ -109,8 +109,7 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
     if (this.subscription.streamingOnly) {
       this.router.navigate(['/player', {uid: uid, url: url}]);
     } else {
-      this.router.navigate(['/player', {uid: uid,
+      this.router.navigate(['/player', {uid: uid}]);
-        sub_id: this.subscription.id}]);
     }
   }
 
@@ -171,4 +170,8 @@ export class SubscriptionComponent implements OnInit, OnDestroy {
     });
   }
 
+  watchSubscription() {
+    this.router.navigate(['/player', {sub_id: this.subscription.id}])
+  }
+
 }