Merge branch 'master' of https://github.com/Tzahi12345/YoutubeDL-Material into download-manager

README.md

@@ -124,7 +124,7 @@ Official translators:
 * German - UnlimitedCookies
 * Chinese - TyRoyal
 
-See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project.
+See also the list of [contributors](https://github.com/Tzahi12345/YoutubeDL-Material/graphs/contributors) who participated in this project.
 
 ## License
 

backend/app.js

@@ -560,7 +560,7 @@ async function loadConfig() {
         watchSubscriptionsInterval();
     }
 
-    await db_api.importUnregisteredFiles();
+    db_api.importUnregisteredFiles();
 
     // start the server here
     startServer();
@@ -1067,9 +1067,23 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
     let files = null;
     let playlists = null;
+    let sort = req.body.sort;
+    let range = req.body.range;
+    let text_search = req.body.text_search;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    files = await db_api.getRecords('files', {user_uid: uuid});
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+    
+    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
+    let file_count = await db_api.getRecords('files', filter_obj, true);
     playlists = await db_api.getRecords('playlists', {user_uid: uuid});
 
     const categories = await categories_api.getCategoriesAsPlaylists(files);
@@ -1081,6 +1095,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
 
     res.send({
         files: files,
+        file_count: file_count,
         playlists: playlists
     });
 });
@@ -1498,8 +1513,15 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
 
 app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
     const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const include_categories = req.body.include_categories;
 
     const playlists = await db_api.getRecords('playlists', {user_uid: uuid});
+    if (include_categories) {
+        const categories = await categories_api.getCategoriesAsPlaylists(files);
+        if (categories) {
+            playlists = playlists.concat(categories);
+        }
+    }
 
     res.send({
         playlists: playlists

backend/config.js

@@ -95,13 +95,13 @@ function getConfigItem(key) {
     }
     let path = CONFIG_ITEMS[key]['path'];
     const val = Object.byString(config_json, path);
-    if (val === undefined && Object.byString(DEFAULT_CONFIG, path)) {
+    if (val === undefined && Object.byString(DEFAULT_CONFIG, path) !== undefined) {
         logger.warn(`Cannot find config with key '${key}'. Creating one with the default value...`);
         setConfigItem(key, Object.byString(DEFAULT_CONFIG, path));
         return Object.byString(DEFAULT_CONFIG, path);
     }
     return Object.byString(config_json, path);
-};
+}
 
 function setConfigItem(key, value) {
     let success = false;

backend/db.js

@@ -20,7 +20,12 @@ exports.database_initialized_bs = new BehaviorSubject(false);
 const tables = {
     files: {
         name: 'files',
-        primary_key: 'uid'
+        primary_key: 'uid',
+        text_search: {
+            title: 'text',
+            uploader: 'text',
+            uid: 'text'
+        }
     },
     playlists: {
         name: 'playlists',
@@ -132,8 +137,13 @@ exports._connectToDB = async (custom_connection_string = null) => {
 
         tables_list.forEach(async table => {
             const primary_key = tables[table]['primary_key'];
-            if (!primary_key) return;
+            if (primary_key) {
-            await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
+                await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
+            }
+            const text_search = tables[table]['text_search'];
+            if (text_search) {
+                await database.collection(table).createIndex(text_search);
+            }
         });
         return true;
     } catch(err) {
@@ -624,7 +634,22 @@ exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
         return true;
     }
 
-    if (replaceFilter) await database.collection(table).deleteMany(replaceFilter);
+    if (replaceFilter) {
+        const output = await database.collection(table).bulkWrite([
+            {
+                deleteMany: {
+                    filter: replaceFilter
+                }
+            },
+            {
+                insertOne: {
+                    document: doc
+                }
+            }
+        ]);
+        logger.debug(`Inserted doc into ${table} with filter: ${JSON.stringify(replaceFilter)}`);
+        return !!(output['result']['ok']);
+    }
 
     const output = await database.collection(table).insertOne(doc);
     logger.debug(`Inserted doc into ${table}`);
@@ -681,13 +706,28 @@ exports.getRecord = async (table, filter_obj) => {
     return await database.collection(table).findOne(filter_obj);
 }
 
-exports.getRecords = async (table, filter_obj = null) => {
+exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
     // local db override
     if (using_local_db) {
-        return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        if (sort) {
+            cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
+        }
+        if (range) {
+            cursor = cursor.slice(range[0], range[1]);
+        }
+        return !return_count ? cursor : cursor.length;
     }
 
-    return filter_obj ? await database.collection(table).find(filter_obj).toArray()  : await database.collection(table).find().toArray();
+    const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
+    if (sort) {
+        cursor.sort({[sort['by']]: sort['order']});
+    }
+    if (range) {
+        cursor.skip(range[0]).limit(range[1] - range[0]);
+    }
+
+    return !return_count ? await cursor.toArray() : await cursor.count();
 }
 
 // Update
@@ -1013,7 +1053,13 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
             if (filter_prop_value === undefined || filter_prop_value === null) {
                 filtered &= record[filter_prop] === undefined || record[filter_prop] === null
             } else {
-                filtered &= record[filter_prop] === filter_prop_value;
+                if (typeof filter_prop_value === 'object') {
+                    if (filter_prop_value['$regex']) {
+                        filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
+                    }
+                } else {
+                    filtered &= record[filter_prop] === filter_prop_value;
+                }
             }
         }
         return filtered;

backend/subscriptions.js

@@ -491,7 +491,7 @@ async function getSubscriptions(user_uid = null) {
 async function getAllSubscriptions() {
     const all_subs = await db_api.getRecords('subscriptions');
     const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
-    return all_subs.filter(sub => !!(sub.user_uid) === multiUserMode);
+    return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
 }
 
 async function getSubscription(subID) {

backend/utils.js

@@ -347,6 +347,26 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
+function createEdgeNGrams(str) {
+    if (str && str.length > 3) {
+        const minGram = 3
+        const maxGram = str.length
+        
+        return str.split(" ").reduce((ngrams, token) => {
+            if (token.length > minGram) {   
+                for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
+                    ngrams = [...ngrams, token.substr(0, i)]
+                }
+            } else {
+                ngrams = [...ngrams, token]
+            }
+            return ngrams
+        }, []).join(" ")
+    }
+    
+    return str
+}
+
 // ffmpeg helper functions
 
 async function cropFile(file_path, start, end, ext) {
@@ -425,6 +445,7 @@ module.exports = {
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
     cropFile: cropFile,
+    createEdgeNGrams: createEdgeNGrams,
     wait: wait,
     File: File
 }

src/app/components/custom-playlists/custom-playlists.component.ts

@@ -35,7 +35,7 @@ export class CustomPlaylistsComponent implements OnInit {
   getAllPlaylists() {
     this.playlists_received = false;
     // must call getAllFiles as we need to get category playlists as well
-    this.postsService.getAllFiles().subscribe(res => {
+    this.postsService.getPlaylists().subscribe(res => {
       this.playlists = res['playlists'];
       this.playlists_received = true;
     });

src/app/components/recent-videos/recent-videos.component.html

@@ -34,7 +34,7 @@
                 <div *ngFor="let file of paged_data; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
                     <app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [availablePlaylists]="playlists" (addFileToPlaylist)="addFileToPlaylist($event)" [loading]="false" (deleteFile)="deleteFile($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? '?jwt=' + this.postsService.token : ''"></app-unified-file-card>
                 </div>
-                <div *ngIf="filtered_files.length === 0">
+                <div *ngIf="paged_data.length === 0">
                     <ng-container i18n="No videos found">No videos found.</ng-container>
                 </div>
             </ng-container>
@@ -46,7 +46,7 @@
         </div>
     </div>
 
-    <mat-paginator class="paginator" #paginator *ngIf="filtered_files && filtered_files.length > 0" (page)="pageChangeEvent($event)" [length]="filtered_files.length"
+    <mat-paginator class="paginator" #paginator *ngIf="paged_data && paged_data.length > 0" (page)="pageChangeEvent($event)" [length]="file_count"
               [pageSize]="pageSize"
               [pageSizeOptions]="[5, 10, 25, 100, this.paged_data && this.paged_data.length > 100 ? this.paged_data.length : 250]">
     </mat-paginator>

src/app/components/recent-videos/recent-videos.component.ts

@@ -2,6 +2,8 @@ import { Component, OnInit, ViewChild } from '@angular/core';
 import { PostsService } from 'app/posts.services';
 import { Router } from '@angular/router';
 import { MatPaginator } from '@angular/material/paginator';
+import { Subject } from 'rxjs';
+import { distinctUntilChanged } from 'rxjs/operators';
 
 @Component({
   selector: 'app-recent-videos',
@@ -15,8 +17,8 @@ export class RecentVideosComponent implements OnInit {
 
   normal_files_received = false;
   subscription_files_received = false;
-  files: any[] = null;
+  file_count = 10;
-  filtered_files: any[] = null;
+  searchChangedSubject: Subject<string> = new Subject<string>();
   downloading_content = {'video': {}, 'audio': {}};
   search_mode = false;
   search_text = '';
@@ -97,6 +99,18 @@ export class RecentVideosComponent implements OnInit {
     if (cached_filter_property && this.filterProperties[cached_filter_property]) {
       this.filterProperty = this.filterProperties[cached_filter_property];
     }
+
+    this.searchChangedSubject
+      .debounceTime(500)
+      .pipe(distinctUntilChanged()
+      ).subscribe(model => {
+        if (model.length > 0) {
+          this.search_mode = true;
+        } else {
+          this.search_mode = false;
+        }
+        this.getAllFiles();
+      });
   }
 
   getAllPlaylists() {
@@ -108,64 +122,40 @@ export class RecentVideosComponent implements OnInit {
   // search
 
   onSearchInputChanged(newvalue) {
-    if (newvalue.length > 0) {
+    this.normal_files_received = false;
-      this.search_mode = true;
+    this.searchChangedSubject.next(newvalue);
-      this.filterFiles(newvalue);
-    } else {
-      this.search_mode = false;
-      this.filtered_files = this.files;
-    }
-  }
-
-  private filterFiles(value: string) {
-    const filterValue = value.toLowerCase();
-    this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.name?.toLowerCase().includes(filterValue));
-    this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex});
-  }
-
-  filterByProperty(prop) {
-    if (this.descendingMode) {
-      this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? -1 : 1));
-    } else {
-      this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? 1 : -1));
-    }
-    if (this.paginator) { this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}) };
   }
 
   filterOptionChanged(value) {
-    this.filterByProperty(value['property']);
     localStorage.setItem('filter_property', value['key']);
+    this.getAllFiles();
   }
 
   toggleModeChange() {
     this.descendingMode = !this.descendingMode;
-    this.filterByProperty(this.filterProperty['property']);
+    this.getAllFiles();
   }
 
   // get files
 
-  getAllFiles() {
+  getAllFiles(cache_mode = false) {
-    this.normal_files_received = false;
+    this.normal_files_received = cache_mode;
-    this.postsService.getAllFiles().subscribe(res => {
+    const current_file_index = (this.paginator?.pageIndex ? this.paginator.pageIndex : 0)*this.pageSize;
-      this.files = res['files'];
+    const sort = {by: this.filterProperty['property'], order: this.descendingMode ? -1 : 1};
-      this.files.sort(this.sortFiles);
+    const range = [current_file_index, current_file_index + this.pageSize];
-      for (let i = 0; i < this.files.length; i++) {
+    this.postsService.getAllFiles(sort, range, this.search_mode ? this.search_text : null).subscribe(res => {
-        const file = this.files[i];
+      this.file_count = res['file_count'];
+      this.paged_data = res['files'];
+      for (let i = 0; i < this.paged_data.length; i++) {
+        const file = this.paged_data[i];
         file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration);
       }
-      if (this.search_mode) {
-        this.filterFiles(this.search_text);
-      } else {
-        this.filtered_files = this.files;
-      }
-      this.filterByProperty(this.filterProperty['property']);
 
       // set cached file count for future use, note that we convert the amount of files to a string
-      localStorage.setItem('cached_file_count', '' + this.files.length);
+      localStorage.setItem('cached_file_count', '' + this.file_count);
 
       this.normal_files_received = true;
 
-      this.paged_data = this.filtered_files.slice(0, 10);
     });
   }
 
@@ -301,12 +291,9 @@ export class RecentVideosComponent implements OnInit {
   }
 
   removeFileCard(file_to_remove) {
-    const index = this.files.map(e => e.uid).indexOf(file_to_remove.uid);
+    const index = this.paged_data.map(e => e.uid).indexOf(file_to_remove.uid);
-    this.files.splice(index, 1);
+    this.paged_data.splice(index, 1);
-    if (this.search_mode) {
+    this.getAllFiles(true);
-      this.filterFiles(this.search_text);
-    }
-    this.filterByProperty(this.filterProperty['property']);
   }
 
   addFileToPlaylist(info_obj) {
@@ -344,7 +331,8 @@ export class RecentVideosComponent implements OnInit {
   }
 
   pageChangeEvent(event) {
-    const offset = ((event.pageIndex + 1) - 1) * event.pageSize;
+    this.pageSize = event.pageSize;
-    this.paged_data = this.filtered_files.slice(offset).slice(0, event.pageSize);
+    this.loading_files = Array(this.pageSize).fill(0);
+    this.getAllFiles();
   }
 }

src/app/posts.services.ts

@@ -238,8 +238,8 @@ export class PostsService implements CanActivate {
         return this.http.post(this.path + 'getFile', {uid: uid, type: type, uuid: uuid}, this.httpOptions);
     }
 
-    getAllFiles() {
+    getAllFiles(sort, range, text_search) {
-        return this.http.post(this.path + 'getAllFiles', {}, this.httpOptions);
+        return this.http.post(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search}, this.httpOptions);
     }
 
     getFullTwitchChat(id, type, uuid = null, sub = null) {