Added scheduler for tasks

2026-03-23 13:10:59 +03:00 · 2022-04-19 22:29:41 -04:00
parent 2b1771d30d
commit 5b4d4d5f81
5 changed files with 181 additions and 5 deletions
--- a/backend/db.js
+++ b/backend/db.js
@@ -54,6 +54,10 @@ const tables = {
        name: 'download_queue',
        primary_key: 'uid'
    },
+    tasks: {
+        name: 'tasks',
+        primary_key: 'key'
+    },
    test: {
        name: 'test'
    }
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -599,6 +599,15 @@
        }
      }
    },
+    "call-bind": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
+      "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+      "requires": {
+        "function-bind": "^1.1.1",
+        "get-intrinsic": "^1.0.2"
+      }
+    },
    "camelcase": {
      "version": "5.3.1",
      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@@ -909,6 +918,15 @@
        "readable-stream": "^3.4.0"
      }
    },
+    "cron-parser": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-3.5.0.tgz",
+      "integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==",
+      "requires": {
+        "is-nan": "^1.3.2",
+        "luxon": "^1.26.0"
+      }
+    },
    "cross-spawn": {
      "version": "7.0.1",
      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz",
@@ -968,6 +986,15 @@
      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
      "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ=="
    },
+    "define-properties": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz",
+      "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==",
+      "requires": {
+        "has-property-descriptors": "^1.0.0",
+        "object-keys": "^1.1.1"
+      }
+    },
    "delayed-stream": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -1358,11 +1385,26 @@
        "rimraf": "2"
      }
    },
+    "function-bind": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+    },
    "get-caller-file": {
      "version": "2.0.5",
      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
    },
+    "get-intrinsic": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
+      "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
+      "requires": {
+        "function-bind": "^1.1.1",
+        "has": "^1.0.3",
+        "has-symbols": "^1.0.1"
+      }
+    },
    "get-stream": {
      "version": "5.1.0",
      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
@@ -1460,11 +1502,32 @@
        "har-schema": "^2.0.0"
      }
    },
+    "has": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
+      "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
+      "requires": {
+        "function-bind": "^1.1.1"
+      }
+    },
    "has-flag": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
      "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
    },
+    "has-property-descriptors": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
+      "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
+      "requires": {
+        "get-intrinsic": "^1.1.1"
+      }
+    },
+    "has-symbols": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+      "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
+    },
    "has-yarn": {
      "version": "2.1.0",
      "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz",
@@ -1635,6 +1698,15 @@
        "is-path-inside": "^3.0.1"
      }
    },
+    "is-nan": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz",
+      "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==",
+      "requires": {
+        "call-bind": "^1.0.0",
+        "define-properties": "^1.1.3"
+      }
+    },
    "is-npm": {
      "version": "4.0.0",
      "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz",
@@ -2034,6 +2106,11 @@
        }
      }
    },
+    "long-timeout": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
+      "integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ="
+    },
    "lowdb": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz",
@@ -2051,6 +2128,11 @@
      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
      "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA=="
    },
+    "luxon": {
+      "version": "1.28.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
+      "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
+    },
    "make-dir": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@@ -2407,6 +2489,16 @@
        "iconv-lite": "^0.4.15"
      }
    },
+    "node-schedule": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz",
+      "integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==",
+      "requires": {
+        "cron-parser": "^3.5.0",
+        "long-timeout": "0.1.1",
+        "sorted-array-functions": "^1.3.0"
+      }
+    },
    "nodemon": {
      "version": "2.0.7",
      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz",
@@ -2475,6 +2567,11 @@
      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
    },
+    "object-keys": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
+      "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
+    },
    "on-finished": {
      "version": "2.3.0",
      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -3053,6 +3150,11 @@
        "is-arrayish": "^0.3.1"
      }
    },
+    "sorted-array-functions": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
+      "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA=="
+    },
    "sparse-bitfield": {
      "version": "3.0.3",
      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
--- a/backend/package.json
+++ b/backend/package.json
@@ -52,6 +52,7 @@
    "multer": "^1.4.2",
    "node-fetch": "^2.6.7",
    "node-id3": "^0.1.14",
+    "node-schedule": "^2.1.0",
    "nodemon": "^2.0.7",
    "passport": "^0.4.1",
    "passport-http": "^0.3.0",
--- a/backend/tasks.js
+++ b/backend/tasks.js
@@ -3,34 +3,53 @@ const db_api = require('./db');

 const fs = require('fs-extra');
 const logger = require('./logger');
+const scheduler = require('node-schedule');

 const TASKS = {
    backup_local_db: {
        run: utils.backupLocalDB,
        title: 'Backup Local DB',
+        job: null
    },
    missing_files_check: {
        run: checkForMissingFiles,
        confirm: deleteMissingFiles,
-        title: 'Missing files check'
+        title: 'Missing files check',
+        job: null
    },
    missing_db_records: {
        run: db_api.importUnregisteredFiles,
-        title: 'Import missing DB records'
+        title: 'Import missing DB records',
+        job: null
    },
    duplicate_files_check: {
        run: checkForDuplicateFiles,
        confirm: removeDuplicates,
-        title: 'Find duplicate files in DB'
+        title: 'Find duplicate files in DB',
+        job: null
    }
 }

+function scheduleJob(task_key, schedule) {
+    return scheduler.scheduleJob(schedule, async () => {
+        const task_state = await db_api.getRecord('tasks', {key: task_key});
+        if (task_state['running'] || task_state['confirming']) {
+            logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
+            return;
+        }
+        
+        // we're just "running" the task, any confirmation should be user-initiated
+        exports.executeRun(task_key);
+    });
+}
+
 exports.initialize = async () => {
    const tasks_keys = Object.keys(TASKS);
    for (let i = 0; i < tasks_keys.length; i++) {
        const task_key = tasks_keys[i];
        const task_in_db = await db_api.getRecord('tasks', {key: task_key});
        if (!task_in_db) {
+            // insert task into table if missing
            await db_api.insertRecordIntoTable('tasks', {
                key: task_key,
                last_ran: null,
@@ -38,8 +57,17 @@ exports.initialize = async () => {
                running: false,
                confirming: false,
                data: null,
-                error: null
+                error: null,
+                schedule: null
            });
+        } else {
+            // reset task if necessary
+            await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
+
+            // schedule task and save job
+            if (task_in_db['schedule']) {
+                TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
+            }
        }
    }
 }
@@ -73,6 +101,16 @@ exports.executeConfirm = async (task_key) => {
    await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000});
 }

+exports.updateTaskSchedule = async (task_key, schedule) => {
+    await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
+    if (TASKS[task_key]['job']) {
+        TASKS[task_key]['job'].cancel();
+    }
+    if (schedule) {
+        TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
+    }
+}
+
 // missing files check

 async function checkForMissingFiles() {
@@ -109,4 +147,6 @@ async function removeDuplicates(data) {
    for (let i = 0; i < data['uids'].length; i++) {
        await db_api.removeRecord('files', {uid: data['uids'][i]});
    }
-}
+}
+
+exports.TASKS = TASKS;
--- a/backend/test/tests.js
+++ b/backend/test/tests.js
@@ -382,6 +382,15 @@ describe('Tasks', function() {
    beforeEach(async function() {
        await db_api.connectToDB();
        await db_api.removeAllRecords('tasks');
+
+        const dummy_task = {
+            run: async () => { await utils.wait(500); return true; },
+            confirm: async () => { await utils.wait(500); return true; },
+            title: 'Dummy task',
+            job: null
+        };
+        tasks_api.TASKS['dummy_task'] = dummy_task;
+
        await tasks_api.initialize();
    });
    it('Backup local db', async function() {
@@ -438,4 +447,24 @@ describe('Tasks', function() {
        if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
        if (fs.existsSync('video/sample.mp4'))       fs.unlinkSync('video/sample.mp4');
    });
+
+    it('Schedule and cancel task', async function() {
+        const today_4_hours = new Date();
+        today_4_hours.setHours(today_4_hours.getHours() + 4);
+        await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
+        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+        await tasks_api.updateTaskSchedule('dummy_task', null);
+        assert(!!tasks_api.TASKS['dummy_task']['job'], false);
+    });
+
+    it('Schedule and run task', async function() {
+        this.timeout(5000);
+        const today_1_second = new Date();
+        today_1_second.setSeconds(today_1_second.getSeconds() + 1);
+        await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
+        assert(!!tasks_api.TASKS['dummy_task']['job'], true);
+        await utils.wait(2000);
+        const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
+        assert(dummy_task_obj['data'], true);
+    });
 });