diff --git a/backend/middlewares/AdminMWs.ts b/backend/middlewares/AdminMWs.ts index 602067ee..fbc45d17 100644 --- a/backend/middlewares/AdminMWs.ts +++ b/backend/middlewares/AdminMWs.ts @@ -11,7 +11,6 @@ import {BasicConfigDTO} from '../../common/entities/settings/BasicConfigDTO'; import {OtherConfigDTO} from '../../common/entities/settings/OtherConfigDTO'; import {ProjectPath} from '../ProjectPath'; import {PrivateConfigClass} from '../../common/config/private/PrivateConfigClass'; -import {IndexingDTO} from '../../common/entities/settings/IndexingDTO'; import {ISQLGalleryManager} from '../model/sql/IGalleryManager'; const LOG_TAG = '[AdminMWs]'; @@ -523,57 +522,4 @@ export class AdminMWs { return next(new ErrorDTO(ErrorCodes.TASK_ERROR, 'Task error: ' + JSON.stringify(err, null, ' '), err)); } } - - public static startIndexing(req: Request, res: Response, next: NextFunction) { - try { - const createThumbnails: boolean = (req.body).createThumbnails || false; - ObjectManagers.getInstance().IndexingTaskManager.startIndexing(createThumbnails); - req.resultPipe = 'ok'; - return next(); - } catch (err) { - if (err instanceof Error) { - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + err.toString(), err)); - } - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + JSON.stringify(err, null, ' '), err)); - } - } - - - public static getIndexingProgress(req: Request, res: Response, next: NextFunction) { - try { - req.resultPipe = ObjectManagers.getInstance().IndexingTaskManager.getProgress(); - return next(); - } catch (err) { - if (err instanceof Error) { - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + err.toString(), err)); - } - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + JSON.stringify(err, null, ' '), err)); - } - } - - public static cancelIndexing(req: Request, res: Response, next: NextFunction) { - try { - ObjectManagers.getInstance().IndexingTaskManager.cancelIndexing(); - req.resultPipe = 'ok'; - return next(); - } catch (err) { - if (err instanceof Error) { - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + err.toString(), err)); - } - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + JSON.stringify(err, null, ' '), err)); - } - } - - public static async resetIndexes(req: Express.Request, res: Response, next: NextFunction) { - try { - await ObjectManagers.getInstance().IndexingTaskManager.reset(); - req.resultPipe = 'ok'; - return next(); - } catch (err) { - if (err instanceof Error) { - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + err.toString(), err)); - } - return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, 'Settings error: ' + JSON.stringify(err, null, ' '), err)); - } - } } diff --git a/backend/model/ObjectManagers.ts b/backend/model/ObjectManagers.ts index 24f648cf..57b6e74a 100644 --- a/backend/model/ObjectManagers.ts +++ b/backend/model/ObjectManagers.ts @@ -4,7 +4,6 @@ import {ISearchManager} from './interfaces/ISearchManager'; import {SQLConnection} from './sql/SQLConnection'; import {ISharingManager} from './interfaces/ISharingManager'; import {Logger} from '../Logger'; -import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager'; import {IIndexingManager} from './interfaces/IIndexingManager'; import {IPersonManager} from './interfaces/IPersonManager'; import {IVersionManager} from './interfaces/IVersionManager'; @@ -19,7 +18,6 @@ export class ObjectManagers { private _searchManager: ISearchManager; private _sharingManager: ISharingManager; private _indexingManager: IIndexingManager; - private _indexingTaskManager: IIndexingTaskManager; private _personManager: IPersonManager; private _versionManager: IVersionManager; private _taskManager: ITaskManager; @@ -49,13 +47,6 @@ export class ObjectManagers { this._indexingManager = value; } - get IndexingTaskManager(): IIndexingTaskManager { - return this._indexingTaskManager; - } - - set IndexingTaskManager(value: IIndexingTaskManager) { - this._indexingTaskManager = value; - } get GalleryManager(): IGalleryManager { return this._galleryManager; @@ -124,7 +115,6 @@ export class ObjectManagers { const UserManager = require('./memory/UserManager').UserManager; const SearchManager = require('./memory/SearchManager').SearchManager; const SharingManager = require('./memory/SharingManager').SharingManager; - const IndexingTaskManager = require('./memory/IndexingTaskManager').IndexingTaskManager; const IndexingManager = require('./memory/IndexingManager').IndexingManager; const PersonManager = require('./memory/PersonManager').PersonManager; const VersionManager = require('./memory/VersionManager').VersionManager; @@ -132,7 +122,6 @@ export class ObjectManagers { ObjectManagers.getInstance().UserManager = new UserManager(); ObjectManagers.getInstance().SearchManager = new SearchManager(); ObjectManagers.getInstance().SharingManager = new SharingManager(); - ObjectManagers.getInstance().IndexingTaskManager = new IndexingTaskManager(); ObjectManagers.getInstance().IndexingManager = new IndexingManager(); ObjectManagers.getInstance().PersonManager = new PersonManager(); ObjectManagers.getInstance().VersionManager = new VersionManager(); @@ -146,7 +135,6 @@ export class ObjectManagers { const UserManager = require('./sql/UserManager').UserManager; const SearchManager = require('./sql/SearchManager').SearchManager; const SharingManager = require('./sql/SharingManager').SharingManager; - const IndexingTaskManager = require('./sql/IndexingTaskManager').IndexingTaskManager; const IndexingManager = require('./sql/IndexingManager').IndexingManager; const PersonManager = require('./sql/PersonManager').PersonManager; const VersionManager = require('./sql/VersionManager').VersionManager; @@ -154,7 +142,6 @@ export class ObjectManagers { ObjectManagers.getInstance().UserManager = new UserManager(); ObjectManagers.getInstance().SearchManager = new SearchManager(); ObjectManagers.getInstance().SharingManager = new SharingManager(); - ObjectManagers.getInstance().IndexingTaskManager = new IndexingTaskManager(); ObjectManagers.getInstance().IndexingManager = new IndexingManager(); ObjectManagers.getInstance().PersonManager = new PersonManager(); ObjectManagers.getInstance().VersionManager = new VersionManager(); diff --git a/backend/model/interfaces/IIndexingTaskManager.ts b/backend/model/interfaces/IIndexingTaskManager.ts deleted file mode 100644 index 217e6948..00000000 --- a/backend/model/interfaces/IIndexingTaskManager.ts +++ /dev/null @@ -1,11 +0,0 @@ -import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO'; - -export interface IIndexingTaskManager { - startIndexing(createThumbnails?: boolean): void; - - getProgress(): TaskProgressDTO; - - cancelIndexing(): void; - - reset(): Promise; -} diff --git a/backend/model/memory/IndexingTaskManager.ts b/backend/model/memory/IndexingTaskManager.ts deleted file mode 100644 index dfbd7f4c..00000000 --- a/backend/model/memory/IndexingTaskManager.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager'; -import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO'; - -export class IndexingTaskManager implements IIndexingTaskManager { - - startIndexing(): void { - throw new Error('not supported by memory DB'); - } - - getProgress(): TaskProgressDTO { - throw new Error('not supported by memory DB'); - } - - cancelIndexing(): void { - throw new Error('not supported by memory DB'); - } - - reset(): Promise { - throw new Error('Method not implemented.'); - } -} diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index cfc69a52..d21f511e 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -153,7 +153,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { .createQueryBuilder('media') .select('SUM(media.metadata.fileSize)', 'sum') .getRawOne(); - return sum; + return sum || 0; } async countPhotos(): Promise { diff --git a/backend/model/sql/IndexingTaskManager.ts b/backend/model/sql/IndexingTaskManager.ts deleted file mode 100644 index b605fc9b..00000000 --- a/backend/model/sql/IndexingTaskManager.ts +++ /dev/null @@ -1,118 +0,0 @@ -import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager'; -import {TaskProgressDTO} from '../../../common/entities/settings/TaskProgressDTO'; -import {ObjectManagers} from '../ObjectManagers'; -import * as path from 'path'; -import * as fs from 'fs'; -import {SQLConnection} from './SQLConnection'; -import {DirectoryEntity} from './enitites/DirectoryEntity'; -import {Logger} from '../../Logger'; -import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker'; -import {Config} from '../../../common/config/private/Config'; -import {MediaDTO} from '../../../common/entities/MediaDTO'; -import {ProjectPath} from '../../ProjectPath'; -import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs'; - -const LOG_TAG = '[IndexingTaskManager]'; - -export class IndexingTaskManager implements IIndexingTaskManager { - directoriesToIndex: string[] = []; - indexingProgress: TaskProgressDTO = null; - enabled = false; - private indexNewDirectory = async (createThumbnails: boolean = false) => { - if (this.directoriesToIndex.length === 0) { - this.indexingProgress = null; - if (global.gc) { - global.gc(); - } - return; - } - const directory = this.directoriesToIndex.shift(); - this.indexingProgress.comment = directory; - this.indexingProgress.left = this.directoriesToIndex.length; - const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory); - if (this.enabled === false) { - return; - } - this.indexingProgress.progress++; - this.indexingProgress.time.current = Date.now(); - for (let i = 0; i < scanned.directories.length; i++) { - this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name)); - } - if (createThumbnails) { - for (let i = 0; i < scanned.media.length; i++) { - try { - const media = scanned.media[i]; - const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name); - const thPath = path.join(ProjectPath.ThumbnailFolder, - ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0])); - if (fs.existsSync(thPath)) { // skip existing thumbnails - continue; - } - await ThumbnailWorker.render({ - type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image, - mediaPath: mPath, - size: Config.Client.Thumbnail.thumbnailSizes[0], - thPath: thPath, - makeSquare: false, - qualityPriority: Config.Server.thumbnail.qualityPriority - }, Config.Server.thumbnail.processingLibrary); - } catch (e) { - console.error(e); - Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString()); - } - } - - } - process.nextTick(() => { - this.indexNewDirectory(createThumbnails).catch(console.error); - }); - }; - - startIndexing(createThumbnails: boolean = false): void { - if (this.directoriesToIndex.length === 0 && this.enabled === false) { - Logger.info(LOG_TAG, 'Starting indexing'); - this.indexingProgress = { - progress: 0, - left: 0, - comment: '', - time: { - start: Date.now(), - current: Date.now() - } - }; - this.directoriesToIndex.push('/'); - this.enabled = true; - this.indexNewDirectory(createThumbnails).catch(console.error); - } else { - Logger.info(LOG_TAG, 'Already indexing..'); - } - } - - getProgress(): TaskProgressDTO { - return this.indexingProgress; - } - - cancelIndexing(): void { - Logger.info(LOG_TAG, 'Canceling indexing'); - this.directoriesToIndex = []; - this.indexingProgress = null; - this.enabled = false; - if (global.gc) { - global.gc(); - } - } - - async reset(): Promise { - Logger.info(LOG_TAG, 'Resetting DB'); - this.directoriesToIndex = []; - this.indexingProgress = null; - this.enabled = false; - const connection = await SQLConnection.getConnection(); - return connection - .getRepository(DirectoryEntity) - .createQueryBuilder('directory') - .delete() - .execute().then(() => { - }); - } -} diff --git a/backend/model/tasks/Task.ts b/backend/model/tasks/Task.ts index d86f9c85..a81b2e44 100644 --- a/backend/model/tasks/Task.ts +++ b/backend/model/tasks/Task.ts @@ -7,7 +7,7 @@ declare const process: any; export abstract class Task implements ITask { - protected progress: TaskProgressDTO; + protected progress: TaskProgressDTO = null; protected running = false; protected config: T; @@ -25,7 +25,7 @@ export abstract class Task implements ITask { public start(config: T): void { if (this.running === false && this.Supported) { - Logger.info('[Task]', 'running task:' + this.Name); + Logger.info('[Task]', 'Running task: ' + this.Name); this.config = config; this.progress = { progress: 0, @@ -45,15 +45,20 @@ export abstract class Task implements ITask { } public stop(): void { - Logger.info('[Task]', 'stopping task' + this.Name); - this.progress = null; + Logger.info('[Task]', 'Stopping task: ' + this.Name); this.running = false; + this.onFinish(); } protected abstract async step(): Promise; protected abstract async init(): Promise; + private onFinish(): void { + this.progress = null; + Logger.info('[Task]', 'Task finished: ' + this.Name); + } + private run() { process.nextTick(async () => { try { @@ -64,6 +69,7 @@ export abstract class Task implements ITask { this.progress = await this.step(); if (this.progress == null) { // finished this.running = false; + this.onFinish(); return; } this.run(); diff --git a/backend/model/tasks/TaskManager.ts b/backend/model/tasks/TaskManager.ts index edf3bfcc..5f1f88cc 100644 --- a/backend/model/tasks/TaskManager.ts +++ b/backend/model/tasks/TaskManager.ts @@ -18,7 +18,9 @@ export class TaskManager implements ITaskManager { getProgresses(): { [id: string]: TaskProgressDTO } { const m: { [id: string]: TaskProgressDTO } = {}; - TaskRepository.Instance.getAvailableTasks().forEach(t => m[t.Name] = t.Progress); + TaskRepository.Instance.getAvailableTasks() + .filter(t => t.Progress) + .forEach(t => m[t.Name] = t.Progress); return m; } @@ -35,6 +37,9 @@ export class TaskManager implements ITaskManager { const t = this.findTask(taskName); if (t) { t.stop(); + if (global.gc) { + global.gc(); + } } else { Logger.warn(LOG_TAG, 'cannot find task to stop:' + taskName); } diff --git a/backend/routes/AdminRouter.ts b/backend/routes/AdminRouter.ts index 42e2dc98..72c17abf 100644 --- a/backend/routes/AdminRouter.ts +++ b/backend/routes/AdminRouter.ts @@ -10,7 +10,6 @@ export class AdminRouter { this.addGetStatistic(app); this.addGetDuplicates(app); this.addTasks(app); - this.addIndexGallery(app); this.addSettings(app); } @@ -32,33 +31,6 @@ export class AdminRouter { ); } - private static addIndexGallery(app: Express) { - app.get('/api/admin/indexes/job/progress', - AuthenticationMWs.authenticate, - AuthenticationMWs.authorise(UserRoles.Admin), - AdminMWs.getIndexingProgress, - RenderingMWs.renderResult - ); - app.post('/api/admin/indexes/job', - AuthenticationMWs.authenticate, - AuthenticationMWs.authorise(UserRoles.Admin), - AdminMWs.startIndexing, - RenderingMWs.renderResult - ); - app.delete('/api/admin/indexes/job', - AuthenticationMWs.authenticate, - AuthenticationMWs.authorise(UserRoles.Admin), - AdminMWs.cancelIndexing, - RenderingMWs.renderResult - ); - app.delete('/api/admin/indexes', - AuthenticationMWs.authenticate, - AuthenticationMWs.authorise(UserRoles.Admin), - AdminMWs.resetIndexes, - RenderingMWs.renderResult - ); - } - private static addTasks(app: Express) { app.get('/api/admin/tasks/available', AuthenticationMWs.authenticate, diff --git a/common/entities/task/TaskDTO.ts b/common/entities/task/TaskDTO.ts index b5720c94..f9fbe138 100644 --- a/common/entities/task/TaskDTO.ts +++ b/common/entities/task/TaskDTO.ts @@ -2,7 +2,7 @@ export type fieldType = 'string' | 'number' | 'boolean'; export enum DefaultsTasks { - Indexing, 'Database Reset', Dummy + Indexing = 1, 'Database Reset' = 2 } export interface ConfigTemplateEntry { @@ -12,11 +12,6 @@ export interface ConfigTemplateEntry { defaultValue: any; } -/* -export interface NestedFieldType { - [key: string]: fieldType | NestedFieldType; -}*/ - export interface TaskDTO { Name: string; ConfigTemplate: ConfigTemplateEntry[]; diff --git a/frontend/app/ui/settings/indexing/indexing.settings.component.html b/frontend/app/ui/settings/indexing/indexing.settings.component.html index d0020080..0909c654 100644 --- a/frontend/app/ui/settings/indexing/indexing.settings.component.html +++ b/frontend/app/ui/settings/indexing/indexing.settings.component.html @@ -52,7 +52,7 @@
- @@ -67,7 +67,7 @@
- @@ -150,20 +150,20 @@
- {{statistic ? statistic.directories : '...'}} + {{_settingsService.statistic.value ? _settingsService.statistic.value.directories : '...'}}
- {{statistic ? statistic.photos : '...'}} + {{_settingsService.statistic.value ? _settingsService.statistic.value.photos : '...'}}
- {{statistic ? statistic.videos : '...'}} + {{_settingsService.statistic.value ? _settingsService.statistic.value.videos : '...'}}
- {{statistic ? (statistic.diskUsage | fileSize) : '...'}} + {{_settingsService.statistic.value ? (_settingsService.statistic.value.diskUsage | fileSize) : '...'}}
diff --git a/frontend/app/ui/settings/indexing/indexing.settings.component.ts b/frontend/app/ui/settings/indexing/indexing.settings.component.ts index 675cf192..673a8e7f 100644 --- a/frontend/app/ui/settings/indexing/indexing.settings.component.ts +++ b/frontend/app/ui/settings/indexing/indexing.settings.component.ts @@ -8,7 +8,6 @@ import {IndexingConfig, ReIndexingSensitivity} from '../../../../../common/confi import {SettingsComponent} from '../_abstract/abstract.settings.component'; import {Utils} from '../../../../../common/Utils'; import {I18n} from '@ngx-translate/i18n-polyfill'; -import {StatisticDTO} from '../../../../../common/entities/settings/StatisticDTO'; import {ScheduledTasksService} from '../scheduled-tasks.service'; import {DefaultsTasks} from '../../../../../common/entities/task/TaskDTO'; @@ -24,7 +23,6 @@ export class IndexingSettingsComponent extends SettingsComponent { + public statistic: BehaviorSubject; constructor(private _networkService: NetworkService, + private _tasksService: ScheduledTasksService, _settingsService: SettingsService) { super(_settingsService); + this.statistic = new BehaviorSubject(null); + const sub = _settingsService.settings.subscribe(() => { + if (this.isSupported()) { + this.loadStatistic(); + sub.unsubscribe(); + } + }); + this._tasksService.onTaskFinish.subscribe((taskName: string) => { + if (taskName === DefaultsTasks[DefaultsTasks.Indexing] || + taskName === DefaultsTasks[DefaultsTasks['Database Reset']]) { + if (this.isSupported()) { + this.loadStatistic(); + } + } + }); } public updateSettings(settings: IndexingConfig): Promise { @@ -28,7 +45,7 @@ export class IndexingSettingsService extends AbstractSettingsService('/admin/statistic'); + async loadStatistic() { + this.statistic.next(await this._networkService.getJson('/admin/statistic')); } } diff --git a/frontend/app/ui/settings/scheduled-tasks.service.ts b/frontend/app/ui/settings/scheduled-tasks.service.ts index 01310c87..07f4d2ea 100644 --- a/frontend/app/ui/settings/scheduled-tasks.service.ts +++ b/frontend/app/ui/settings/scheduled-tasks.service.ts @@ -1,4 +1,4 @@ -import {Injectable} from '@angular/core'; +import {EventEmitter, Injectable} from '@angular/core'; import {BehaviorSubject} from 'rxjs'; import {TaskProgressDTO} from '../../../../common/entities/settings/TaskProgressDTO'; import {NetworkService} from '../../model/network/network.service'; @@ -8,6 +8,7 @@ export class ScheduledTasksService { public progress: BehaviorSubject<{ [key: string]: TaskProgressDTO }>; + public onTaskFinish: EventEmitter = new EventEmitter(); timer: number = null; private subscribers = 0; @@ -28,16 +29,24 @@ export class ScheduledTasksService { return this.getProgress(); } - public async start(id: string, config?: any) { - return await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/start', {config: config}); + public async start(id: string, config?: any): Promise { + await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/start', {config: config}); + this.forceUpdate(); } - public async stop(id: string) { - return await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/stop'); + public async stop(id: string): Promise { + await this._networkService.postJson('/admin/tasks/scheduled/' + id + '/stop'); + this.forceUpdate(); } protected async getProgress() { - return this.progress.next(await this._networkService.getJson<{ [key: string]: TaskProgressDTO }>('/admin/tasks/scheduled/progress')); + const prevPrg = this.progress.value; + this.progress.next(await this._networkService.getJson<{ [key: string]: TaskProgressDTO }>('/admin/tasks/scheduled/progress')); + for (const prg in prevPrg) { + if (!this.progress.value.hasOwnProperty(prg)) { + this.onTaskFinish.emit(prg); + } + } } protected getProgressPeriodically() { diff --git a/frontend/app/ui/settings/tasks/tasks.settings.component.ts b/frontend/app/ui/settings/tasks/tasks.settings.component.ts index e088f428..652e95a7 100644 --- a/frontend/app/ui/settings/tasks/tasks.settings.component.ts +++ b/frontend/app/ui/settings/tasks/tasks.settings.component.ts @@ -111,7 +111,6 @@ export class TasksSettingsComponent extends SettingsComponent