From d04a17bf694a83d5f9b28c2b02037b4f21b29d31 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Fri, 6 Dec 2019 15:53:34 +0100 Subject: [PATCH] implementing task scheduling at backend --- backend/Logger.ts | 11 +- backend/middlewares/AdminMWs.ts | 1 + backend/model/ObjectManagers.ts | 15 ++- backend/model/interfaces/ITaskManager.ts | 5 +- backend/model/tasks/TaskManager.ts | 103 +++++++++++++----- backend/server.ts | 6 +- common/config/private/IPrivateConfig.ts | 2 +- common/entities/task/TaskScheduleDTO.ts | 2 +- .../unit/model/tasks/TaskManager.spec.ts | 77 +++++++++++++ 9 files changed, 186 insertions(+), 36 deletions(-) create mode 100644 test/backend/unit/model/tasks/TaskManager.spec.ts diff --git a/backend/Logger.ts b/backend/Logger.ts index 75418fba..38d80cf4 100644 --- a/backend/Logger.ts +++ b/backend/Logger.ts @@ -25,4 +25,13 @@ export const winstonSettings = { exitOnError: false }; -export const Logger = new (winston).Logger(winstonSettings); +type logFN = (...args: (string | number)[]) => {}; + +export const Logger: { + error: logFN, + warn: logFN, + info: logFN, + verbose: logFN, + debug: logFN, + silly: logFN +} = new (winston).Logger(winstonSettings); diff --git a/backend/middlewares/AdminMWs.ts b/backend/middlewares/AdminMWs.ts index 502e63c6..602067ee 100644 --- a/backend/middlewares/AdminMWs.ts +++ b/backend/middlewares/AdminMWs.ts @@ -457,6 +457,7 @@ export class AdminMWs { original.save(); await ConfigDiagnostics.runDiagnostics(); + ObjectManagers.getInstance().TaskManager.runSchedules(); Logger.info(LOG_TAG, 'new config:'); Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t')); return next(); diff --git a/backend/model/ObjectManagers.ts b/backend/model/ObjectManagers.ts index 7034dd01..24f648cf 100644 --- a/backend/model/ObjectManagers.ts +++ b/backend/model/ObjectManagers.ts @@ -105,10 +105,19 @@ export class ObjectManagers { } public static async reset() { + if (ObjectManagers.getInstance().TaskManager) { + ObjectManagers.getInstance().TaskManager.stopSchedules(); + } await SQLConnection.close(); this._instance = null; } + + public static async InitCommonManagers() { + const TaskManager = require('./tasks/TaskManager').TaskManager; + ObjectManagers.getInstance().TaskManager = new TaskManager(); + } + public static async InitMemoryManagers() { await ObjectManagers.reset(); const GalleryManager = require('./memory/GalleryManager').GalleryManager; @@ -119,7 +128,6 @@ export class ObjectManagers { const IndexingManager = require('./memory/IndexingManager').IndexingManager; const PersonManager = require('./memory/PersonManager').PersonManager; const VersionManager = require('./memory/VersionManager').VersionManager; - const TaskManager = require('./tasks/TaskManager').TaskManager; ObjectManagers.getInstance().GalleryManager = new GalleryManager(); ObjectManagers.getInstance().UserManager = new UserManager(); ObjectManagers.getInstance().SearchManager = new SearchManager(); @@ -128,7 +136,7 @@ export class ObjectManagers { ObjectManagers.getInstance().IndexingManager = new IndexingManager(); ObjectManagers.getInstance().PersonManager = new PersonManager(); ObjectManagers.getInstance().VersionManager = new VersionManager(); - ObjectManagers.getInstance().TaskManager = new TaskManager(); + this.InitCommonManagers(); } public static async InitSQLManagers() { @@ -142,7 +150,6 @@ export class ObjectManagers { const IndexingManager = require('./sql/IndexingManager').IndexingManager; const PersonManager = require('./sql/PersonManager').PersonManager; const VersionManager = require('./sql/VersionManager').VersionManager; - const TaskManager = require('./tasks/TaskManager').TaskManager; ObjectManagers.getInstance().GalleryManager = new GalleryManager(); ObjectManagers.getInstance().UserManager = new UserManager(); ObjectManagers.getInstance().SearchManager = new SearchManager(); @@ -151,7 +158,7 @@ export class ObjectManagers { ObjectManagers.getInstance().IndexingManager = new IndexingManager(); ObjectManagers.getInstance().PersonManager = new PersonManager(); ObjectManagers.getInstance().VersionManager = new VersionManager(); - ObjectManagers.getInstance().TaskManager = new TaskManager(); + this.InitCommonManagers(); Logger.debug('SQL DB inited'); } diff --git a/backend/model/interfaces/ITaskManager.ts b/backend/model/interfaces/ITaskManager.ts index d8dd667a..6423b68e 100644 --- a/backend/model/interfaces/ITaskManager.ts +++ b/backend/model/interfaces/ITaskManager.ts @@ -1,5 +1,4 @@ import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO'; -import {TaskScheduleDTO} from '../../../common/entities/task/TaskScheduleDTO'; import {TaskDTO} from '../../../common/entities/task/TaskDTO'; export interface ITaskManager { @@ -12,4 +11,8 @@ export interface ITaskManager { getAvailableTasks(): TaskDTO[]; + + stopSchedules(): void; + + runSchedules(): void; } diff --git a/backend/model/tasks/TaskManager.ts b/backend/model/tasks/TaskManager.ts index 2bdfe6f2..64b63c27 100644 --- a/backend/model/tasks/TaskManager.ts +++ b/backend/model/tasks/TaskManager.ts @@ -3,10 +3,18 @@ import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO import {ITask} from './ITask'; import {TaskRepository} from './TaskRepository'; import {Config} from '../../../common/config/private/Config'; -import {TaskTriggerType} from '../../../common/entities/task/TaskScheduleDTO'; +import {TaskScheduleDTO, TaskTriggerType} from '../../../common/entities/task/TaskScheduleDTO'; +import {Logger} from '../../Logger'; + +const LOG_TAG = '[TaskManager]'; export class TaskManager implements ITaskManager { + protected timers: NodeJS.Timeout[] = []; + + constructor() { + this.runSchedules(); + } getProgresses(): { [id: string]: TaskProgressDTO } { const m: { [id: string]: TaskProgressDTO } = {}; @@ -14,10 +22,12 @@ export class TaskManager implements ITaskManager { return m; } - start(taskName: string, config: any): void { + start(taskName: string, config: T): void { const t = this.findTask(taskName); if (t) { t.start(config); + } else { + Logger.warn(LOG_TAG, 'cannot find task to start:' + taskName); } } @@ -25,6 +35,8 @@ export class TaskManager implements ITaskManager { const t = this.findTask(taskName); if (t) { t.stop(); + } else { + Logger.warn(LOG_TAG, 'cannot find task to stop:' + taskName); } } @@ -32,38 +44,79 @@ export class TaskManager implements ITaskManager { return TaskRepository.Instance.getAvailableTasks(); } + public stopSchedules(): void { + this.timers.forEach(timer => clearTimeout(timer)); + this.timers = []; + } + public runSchedules(): void { + this.stopSchedules(); + Logger.info(LOG_TAG, 'Running task schedules'); Config.Server.tasks.scheduled.forEach(schedule => { - let nextRun = null; - switch (schedule.trigger.type) { - case TaskTriggerType.scheduled: - nextRun = Date.now() - schedule.trigger.time; - break; - /*case TaskTriggerType.periodic: + const nextDate = this.getDateFromSchedule(new Date(), schedule); + if (nextDate && nextDate.getTime() > Date.now()) { + Logger.debug(LOG_TAG, 'running schedule: [' + schedule.id + '] ' + schedule.name + + ' at ' + nextDate.toLocaleString(undefined, {hour12: false})); - //TODo finish it - const getNextDayOfTheWeek = (dayOfWeek: number) => { - const refDate = new Date(); - refDate.setHours(0, 0, 0, 0); - refDate.setDate(refDate.getDate() + (dayOfWeek + 7 - refDate.getDay()) % 7); - return refDate; - }; - - nextRun = Date.now() - schedule.trigger.periodicity; - break;*/ - } - - if (nextRun != null) { - setTimeout(() => { + const timer: NodeJS.Timeout = setTimeout(() => { this.start(schedule.taskName, schedule.config); - }, nextRun); + this.timers = this.timers.filter(t => t !== timer); + }, nextDate.getTime() - Date.now()); + this.timers.push(timer); + + } else { + Logger.debug(LOG_TAG, 'skipping schedule: [' + schedule.id + '] ' + schedule.name); } + }); } - protected findTask(taskName: string): ITask { - return this.getAvailableTasks().find(t => t.Name === taskName); + protected getDateFromSchedule(refDate: Date, schedule: TaskScheduleDTO): Date { + switch (schedule.trigger.type) { + case TaskTriggerType.scheduled: + return new Date(schedule.trigger.time); + case TaskTriggerType.periodic: + const nextValidHM = (date: Date, h: number, m: number, dayDiff: number): Date => { + + if (date.getHours() < h || (date.getHours() === h && date.getMinutes() <= m)) { + date.setHours(h); + date.setMinutes(m); + } else { + date.setTime(date.getTime() + dayDiff); + date.setHours(h); + date.setMinutes(m); + } + return date; + }; + + const getNextDayOfTheWeek = (dayOfWeek: number, h: number, m: number): Date => { + const date = new Date(refDate); + date.setDate(refDate.getDate() + (dayOfWeek + 1 + 7 - refDate.getDay()) % 7); + if (date.getDay() === refDate.getDay()) { + return new Date(refDate); + } + date.setHours(0, 0, 0, 0); + return date; + }; + + + const hour = Math.floor(schedule.trigger.atTime / 1000 / (60 * 60)); + const minute = (schedule.trigger.atTime / 1000 / 60) % 60; + + if (schedule.trigger.periodicity <= 6) { // Between Monday and Sunday + const nextRunDate = getNextDayOfTheWeek(schedule.trigger.periodicity, hour, minute); + return nextValidHM(nextRunDate, hour, minute, 7 * 24 * 60 * 60 * 1000); + } + + // every day + return nextValidHM(new Date(refDate), hour, minute, 24 * 60 * 60 * 1000); + } + return null; + } + + protected findTask(taskName: string): ITask { + return this.getAvailableTasks().find(t => t.Name === taskName); } } diff --git a/backend/server.ts b/backend/server.ts index a1916e29..ff9bfc05 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -35,7 +35,7 @@ export class Server { constructor() { if (!(process.env.NODE_ENV === 'production')) { - Logger.debug(LOG_TAG, 'Running in DEBUG mode, set env variable NODE_ENV=production to disable '); + Logger.info(LOG_TAG, 'Running in DEBUG mode, set env variable NODE_ENV=production to disable '); } this.init(); } @@ -43,10 +43,10 @@ export class Server { async init() { Logger.info(LOG_TAG, 'running diagnostics...'); await ConfigDiagnostics.runDiagnostics(); - Logger.info(LOG_TAG, 'using config:'); + Logger.verbose(LOG_TAG, 'using config:'); const appVer = require('../package.json').version; Config.Client.appVersion = appVer; - Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t')); + Logger.verbose(LOG_TAG, JSON.stringify(Config, null, '\t')); this.app = _express(); diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index 3fc7d256..16b95e14 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -6,7 +6,7 @@ export enum DatabaseType { } export enum LogLevel { - error = 1, warn = 2, info = 3, debug = 4, verbose = 5, silly = 6 + error = 1, warn = 2, info = 3, verbose = 4, debug = 5, silly = 6 } export enum SQLLogLevel { diff --git a/common/entities/task/TaskScheduleDTO.ts b/common/entities/task/TaskScheduleDTO.ts index b5277ad7..92487b6b 100644 --- a/common/entities/task/TaskScheduleDTO.ts +++ b/common/entities/task/TaskScheduleDTO.ts @@ -17,7 +17,7 @@ export interface ScheduledTaskTrigger extends TaskTrigger { export interface PeriodicTaskTrigger extends TaskTrigger { type: TaskTriggerType.periodic; - periodicity: number; // 1-7: week days 8+ every x days + periodicity: number; // 0-6: week days 7 every day atTime: number; // day time } diff --git a/test/backend/unit/model/tasks/TaskManager.spec.ts b/test/backend/unit/model/tasks/TaskManager.spec.ts new file mode 100644 index 00000000..46822cf4 --- /dev/null +++ b/test/backend/unit/model/tasks/TaskManager.spec.ts @@ -0,0 +1,77 @@ +import {expect} from 'chai'; +import {TaskManager} from '../../../../../backend/model/tasks/TaskManager'; +import {TaskScheduleDTO, TaskTriggerType} from '../../../../../common/entities/task/TaskScheduleDTO'; + +class TaskManagerSpec extends TaskManager { + + public getDateFromSchedule(refDate: Date, schedule: TaskScheduleDTO): Date { + return super.getDateFromSchedule(refDate, schedule); + } +} + +describe('TaskManager', () => { + + it('should get date from schedule', async () => { + const tm = new TaskManagerSpec(); + + const refDate = new Date(2019, 7, 18, 5, 10); // its a sunday + + + expect(tm.getDateFromSchedule(refDate, { + trigger: { + type: TaskTriggerType.scheduled, + time: (new Date(2019, 7, 18, 5, 10)).getTime() + } + })).to.be.deep.equal((new Date(2019, 7, 18, 5, 10))); + + + for (let dayOfWeek = 0; dayOfWeek < 7; ++dayOfWeek) { + let nextDay = dayOfWeek < 6 ? (18 + dayOfWeek + 1) : 18; + + let h = 10; + let m = 5; + expect(tm.getDateFromSchedule(refDate, { + trigger: { + type: TaskTriggerType.periodic, + atTime: (h * 60 + m) * 60 * 1000, + periodicity: dayOfWeek + } + })).to.be.deep.equal((new Date(2019, 7, nextDay, h, m)), 'for day: ' + dayOfWeek); + + h = 2; + m = 5; + nextDay = 18 + dayOfWeek + 1; + expect(tm.getDateFromSchedule(refDate, { + trigger: { + type: TaskTriggerType.periodic, + atTime: (h * 60 + m) * 60 * 1000, + periodicity: dayOfWeek + } + })).to.be.deep.equal((new Date(2019, 7, nextDay, h, m)), 'for day: ' + dayOfWeek); + } + + { + const h = 10; + const m = 5; + expect(tm.getDateFromSchedule(refDate, { + trigger: { + type: TaskTriggerType.periodic, + atTime: (h * 60 + m) * 60 * 1000, + periodicity: 7 + } + })).to.be.deep.equal((new Date(2019, 7, 18, h, m))); + } + { + const h = 2; + const m = 5; + expect(tm.getDateFromSchedule(refDate, { + trigger: { + type: TaskTriggerType.periodic, + atTime: (h * 60 + m) * 60 * 1000, + periodicity: 7 + } + })).to.be.deep.equal((new Date(2019, 7, 19, h, m))); + } + }); + +});