Pagination and filtering of files is now server-side

Importing unregistered files does not block server start anymore

backend/app.js

@@ -631,7 +631,7 @@ async function loadConfig() {
         watchSubscriptionsInterval();
     }
 
-    await db_api.importUnregisteredFiles();
+    db_api.importUnregisteredFiles();
 
     // load in previous downloads
     downloads = await db_api.getRecords('downloads');
@@ -1632,9 +1632,23 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
     // these are returned
     let files = null;
     let playlists = null;
+    let sort = req.body.sort;
+    let range = req.body.range;
+    let text_search = req.body.text_search;
     const uuid = req.isAuthenticated() ? req.user.uid : null;
 
-    files = await db_api.getRecords('files', {user_uid: uuid});
+    const filter_obj = {user_uid: uuid};
+    const regex = true;
+    if (text_search) {
+        if (regex) {
+            filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
+        } else {
+            filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
+        }
+    }
+    
+    files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
+    let file_count = await db_api.getRecords('files', filter_obj, true);
     playlists = await db_api.getRecords('playlists', {user_uid: uuid});
 
     const categories = await categories_api.getCategoriesAsPlaylists(files);
@@ -1646,6 +1660,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
 
     res.send({
         files: files,
+        file_count: file_count,
         playlists: playlists
     });
 });
@@ -2116,8 +2131,15 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
 
 app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
     const uuid = req.isAuthenticated() ? req.user.uid : null;
+    const include_categories = req.body.include_categories;
 
     const playlists = await db_api.getRecords('playlists', {user_uid: uuid});
+    if (include_categories) {
+        const categories = await categories_api.getCategoriesAsPlaylists(files);
+        if (categories) {
+            playlists = playlists.concat(categories);
+        }
+    }
 
     res.send({
         playlists: playlists

backend/db.js

@@ -18,7 +18,12 @@ var database = null;
 const tables = {
     files: {
         name: 'files',
-        primary_key: 'uid'
+        primary_key: 'uid',
+        text_search: {
+            title: 'text',
+            uploader: 'text',
+            uid: 'text'
+        }
     },
     playlists: {
         name: 'playlists',
@@ -131,8 +136,13 @@ exports._connectToDB = async (custom_connection_string = null) => {
 
         tables_list.forEach(async table => {
             const primary_key = tables[table]['primary_key'];
-            if (!primary_key) return;
-            await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
+            if (primary_key) {
+                await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
+            }
+            const text_search = tables[table]['text_search'];
+            if (text_search) {
+                await database.collection(table).createIndex(text_search);
+            }
         });
         return true;
     } catch(err) {
@@ -695,13 +705,28 @@ exports.getRecord = async (table, filter_obj) => {
     return await database.collection(table).findOne(filter_obj);
 }
 
-exports.getRecords = async (table, filter_obj = null) => {
+exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
     // local db override
     if (using_local_db) {
-        return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
+        if (sort) {
+            cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
+        }
+        if (range) {
+            cursor = cursor.slice(range[0], range[1]);
+        }
+        return !return_count ? cursor : cursor.length;
     }
 
-    return filter_obj ? await database.collection(table).find(filter_obj).toArray()  : await database.collection(table).find().toArray();
+    const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
+    if (sort) {
+        cursor.sort({[sort['by']]: sort['order']});
+    }
+    if (range) {
+        cursor.skip(range[0]).limit(range[1] - range[0]);
+    }
+
+    return !return_count ? await cursor.toArray() : await cursor.count();
 }
 
 // Update
@@ -1027,7 +1052,13 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
             if (filter_prop_value === undefined || filter_prop_value === null) {
                 filtered &= record[filter_prop] === undefined || record[filter_prop] === null
             } else {
-                filtered &= record[filter_prop] === filter_prop_value;
+                if (typeof filter_prop_value === 'object') {
+                    if (filter_prop_value['$regex']) {
+                        filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
+                    }
+                } else {
+                    filtered &= record[filter_prop] === filter_prop_value;
+                }
             }
         }
         return filtered;

backend/utils.js

@@ -349,6 +349,26 @@ function removeFileExtension(filename) {
     return filename_parts.join('.');
 }
 
+function createEdgeNGrams(str) {
+    if (str && str.length > 3) {
+        const minGram = 3
+        const maxGram = str.length
+        
+        return str.split(" ").reduce((ngrams, token) => {
+            if (token.length > minGram) {   
+                for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
+                    ngrams = [...ngrams, token.substr(0, i)]
+                }
+            } else {
+                ngrams = [...ngrams, token]
+            }
+            return ngrams
+        }, []).join(" ")
+    } 
+    
+    return str
+}
+
 /**
  * setTimeout, but its a promise.
  * @param {number} ms
@@ -399,6 +419,7 @@ module.exports = {
     getCurrentDownloader: getCurrentDownloader,
     recFindByExt: recFindByExt,
     removeFileExtension: removeFileExtension,
+    createEdgeNGrams: createEdgeNGrams,
     wait: wait,
     File: File
 }