From 1d7bc3c48913830e87870e5b39d9a3e26cd5ef74 Mon Sep 17 00:00:00 2001 From: Patrik Braun Date: Thu, 27 Jul 2017 23:10:16 +0200 Subject: [PATCH] improving indexing --- backend/middlewares/AdminMWs.ts | 29 +++++++- backend/model/memory/GalleryManager.ts | 5 +- backend/model/sql/GalleryManager.ts | 20 ++++-- backend/model/threading/DiskMangerWorker.ts | 2 +- backend/routes/AdminRouter.ts | 6 ++ common/config/private/IPrivateConfig.ts | 13 +++- common/config/private/PrivateConfigClass.ts | 17 +++-- .../_abstract/abstract.settings.component.ts | 16 ++--- .../indexing/indexing.settings.component.html | 54 ++++++++++++++- .../indexing/indexing.settings.component.ts | 66 +++++++++---------- .../indexing/indexing.settings.service.ts | 12 ++-- frontend/app/settings/settings.service.ts | 22 +++++-- 12 files changed, 193 insertions(+), 69 deletions(-) diff --git a/backend/middlewares/AdminMWs.ts b/backend/middlewares/AdminMWs.ts index 46476246..e53f3fc3 100644 --- a/backend/middlewares/AdminMWs.ts +++ b/backend/middlewares/AdminMWs.ts @@ -3,7 +3,12 @@ import {ErrorCodes, ErrorDTO} from "../../common/entities/Error"; import {ObjectManagerRepository} from "../model/ObjectManagerRepository"; import {Logger} from "../Logger"; import {SQLConnection} from "../model/sql/SQLConnection"; -import {DataBaseConfig, DatabaseType, ThumbnailConfig} from "../../common/config/private/IPrivateConfig"; +import { + DataBaseConfig, + DatabaseType, + IndexingConfig, + ThumbnailConfig +} from "../../common/config/private/IPrivateConfig"; import {Config} from "../../common/config/private/Config"; import {ConfigDiagnostics} from "../model/ConfigDiagnostics"; import {ClientConfig} from "../../common/config/public/ConfigClass"; @@ -237,6 +242,28 @@ export class AdminMWs { } } + public static async updateIndexingSettings(req: Request, res: Response, next: NextFunction) { + if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) { + return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed")); + } + + try { + const settings: IndexingConfig = req.body.settings; + Config.Server.indexing = settings; + + //only updating explicitly set config (not saving config set by the diagnostics) + const original = Config.original(); + original.Server.indexing = settings; + original.save(); + await ConfigDiagnostics.runDiagnostics(); + Logger.info(LOG_TAG, "new config:"); + Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t')); + return next(); + } catch (err) { + return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Settings error: " + JSON.stringify(err, null, ' '), err)); + } + } + public static startIndexing(req: Request, res: Response, next: NextFunction) { try { diff --git a/backend/model/memory/GalleryManager.ts b/backend/model/memory/GalleryManager.ts index cff5b963..0f51a596 100644 --- a/backend/model/memory/GalleryManager.ts +++ b/backend/model/memory/GalleryManager.ts @@ -5,6 +5,7 @@ import * as fs from "fs"; import {DiskManager} from "../DiskManger"; import {ProjectPath} from "../../ProjectPath"; import {Config} from "../../../common/config/private/Config"; +import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig"; export class GalleryManager implements IGalleryManager { @@ -13,7 +14,9 @@ export class GalleryManager implements IGalleryManager { if (knownLastModified && knownLastScanned) { const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime()); - if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && lastModified == knownLastModified) { + if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout && + lastModified == knownLastModified && + Config.Server.indexing.reIndexingSensitivity < ReIndexingSensitivity.high) { return Promise.resolve(null); } } diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 0919f210..013d564a 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -10,6 +10,7 @@ import {Utils} from "../../../common/Utils"; import {ProjectPath} from "../../ProjectPath"; import {Config} from "../../../common/config/private/Config"; import {ISQLGalleryManager} from "./IGalleryManager"; +import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig"; export class GalleryManager implements IGalleryManager, ISQLGalleryManager { @@ -37,10 +38,15 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { if (dir && dir.scanned == true) { //iF it seems that the content did not changed, do not work on it - if (knownLastModified && knownLastScanned) { - if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && - lastModified == knownLastModified && - dir.lastScanned == knownLastScanned) { + if (knownLastModified && knownLastScanned + && lastModified == knownLastModified && + dir.lastScanned == knownLastScanned) { + + if (Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.low) { + return null; + } + if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout && + Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.medium) { return null; } } @@ -61,7 +67,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { dir: dir.directories[i].id }) .orderBy("photo.metadata.creationDate", "ASC") - .setLimit(Config.Server.folderPreviewSize) + .setLimit(Config.Server.indexing.folderPreviewSize) .getMany(); dir.directories[i].isPartial = true; @@ -79,7 +85,9 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { return this.indexDirectory(relativeDirectoryName); } - if (Date.now() - dir.lastScanned > Config.Server.cachedFolderTimeout) { + if ((Date.now() - dir.lastScanned > Config.Server.indexing.cachedFolderTimeout && + Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.medium) || + Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.high) { //on the fly reindexing this.indexDirectory(relativeDirectoryName).catch((err) => { console.error(err); diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 5d514467..6fe68ec3 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -55,7 +55,7 @@ export class DiskMangerWorker { const fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), - Config.Server.folderPreviewSize, true + Config.Server.indexing.folderPreviewSize, true ); d.lastScanned = 0; //it was not a fully scan d.isPartial = true; diff --git a/backend/routes/AdminRouter.ts b/backend/routes/AdminRouter.ts index 0b9482d4..33c88bcd 100644 --- a/backend/routes/AdminRouter.ts +++ b/backend/routes/AdminRouter.ts @@ -96,6 +96,12 @@ export class AdminRouter { AdminMWs.updateOtherSettings, RenderingMWs.renderOK ); + app.put("/api/settings/indexing", + AuthenticationMWs.authenticate, + AuthenticationMWs.authorise(UserRoles.Admin), + AdminMWs.updateIndexingSettings, + RenderingMWs.renderOK + ); }; diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index 738c3e01..595603e0 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -41,6 +41,16 @@ export interface SharingConfig { updateTimeout: number; } +export enum ReIndexingSensitivity { + low, medium, high +} + +export interface IndexingConfig { + folderPreviewSize: number; + cachedFolderTimeout: number;//Do not rescans the folder if seems ok + reIndexingSensitivity: ReIndexingSensitivity; +} + export interface ServerConfig { port: number; imagesFolder: string; @@ -49,8 +59,7 @@ export interface ServerConfig { enableThreading: boolean; sharing: SharingConfig; sessionTimeout: number - folderPreviewSize: number; - cachedFolderTimeout: number;//Do not rescans the folder if seems ok + indexing: IndexingConfig; } export interface IPrivateConfig { diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index 0371ddae..21aa500f 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -1,5 +1,11 @@ import {PublicConfigClass} from "../public/ConfigClass"; -import {DatabaseType, IPrivateConfig, ServerConfig, ThumbnailProcessingLib} from "./IPrivateConfig"; +import { + DatabaseType, + IPrivateConfig, + ReIndexingSensitivity, + ServerConfig, + ThumbnailProcessingLib +} from "./IPrivateConfig"; import * as path from "path"; import {ConfigLoader} from "typeconfig"; @@ -33,9 +39,12 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon sharing: { updateTimeout: 1000 * 60 * 5 }, - enableThreading: true, - folderPreviewSize: 2, - cachedFolderTimeout: 1000 * 60 * 60 + indexing: { + folderPreviewSize: 2, + cachedFolderTimeout: 1000 * 60 * 60, + reIndexingSensitivity: ReIndexingSensitivity.high + }, + enableThreading: true }; private ConfigLoader: any; diff --git a/frontend/app/settings/_abstract/abstract.settings.component.ts b/frontend/app/settings/_abstract/abstract.settings.component.ts index 227e1b85..776087f1 100644 --- a/frontend/app/settings/_abstract/abstract.settings.component.ts +++ b/frontend/app/settings/_abstract/abstract.settings.component.ts @@ -23,8 +23,8 @@ export abstract class SettingsComponent implements OnInit, OnDestroy, OnChang public inProgress = false; public error: string = null; public changed: boolean = false; - private subscription = null; - private settingsSubscription = null; + private _subscription = null; + private _settingsSubscription = null; public settings: T = {}; public original: T = {}; @@ -36,7 +36,7 @@ export abstract class SettingsComponent implements OnInit, OnDestroy, OnChang protected notification: NotificationService, private sliceFN?: (s: IPrivateConfig) => T) { if (this.sliceFN) { - this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings); + this._settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings); this.onNewSettings(this._settingsService._settingsService.settings.value); } } @@ -55,7 +55,7 @@ export abstract class SettingsComponent implements OnInit, OnDestroy, OnChang } this.getSettings(); - this.subscription = this.form.valueChanges.subscribe((data) => { + this._subscription = this.form.valueChanges.subscribe((data) => { this.changed = !Utils.equalsFilter(this.settings, this.original); }); @@ -67,11 +67,11 @@ export abstract class SettingsComponent implements OnInit, OnDestroy, OnChang ngOnDestroy() { - if (this.subscription != null) { - this.subscription.unsubscribe(); + if (this._subscription != null) { + this._subscription.unsubscribe(); } - if (this.settingsSubscription != null) { - this.settingsSubscription.unsubscribe(); + if (this._settingsSubscription != null) { + this._settingsSubscription.unsubscribe(); } } diff --git a/frontend/app/settings/indexing/indexing.settings.component.html b/frontend/app/settings/indexing/indexing.settings.component.html index 48127fed..dcd5c647 100644 --- a/frontend/app/settings/indexing/indexing.settings.component.html +++ b/frontend/app/settings/indexing/indexing.settings.component.html @@ -6,6 +6,56 @@
+ +
+ +
+ + If there was no indexing in this time, it reindexes. (skipped if indexen in DB and sensitivity is low) +
+
+
+ +
+ + Reads this many photos from sub folders +
+
+ +
+ +
+ + Set the reindexing sensitivity. High value check the folders for change more often +
+
+ + + + +
+
+ If you add a new folder to your gallery, the site indexes it automatically. If you would like to trigger indexing manually, click index button.
(Note: search ony searched among the indexed directories)
@@ -34,11 +84,11 @@
diff --git a/frontend/app/settings/indexing/indexing.settings.component.ts b/frontend/app/settings/indexing/indexing.settings.component.ts index 7cdd885b..c7bb5f03 100644 --- a/frontend/app/settings/indexing/indexing.settings.component.ts +++ b/frontend/app/settings/indexing/indexing.settings.component.ts @@ -1,11 +1,13 @@ -import {Component, Input, OnDestroy, OnInit, Output} from "@angular/core"; +import {Component} from "@angular/core"; import {IndexingSettingsService} from "./indexing.settings.service"; import {AuthenticationService} from "../../model/network/authentication.service"; import {NavigationService} from "../../model/navigation.service"; import {NotificationService} from "../../model/notification.service"; import {ErrorDTO} from "../../../../common/entities/Error"; -import {UserRoles} from "../../../../common/entities/UserDTO"; import {Observable} from "rxjs/Rx"; +import {IndexingConfig, ReIndexingSensitivity} from "../../../../common/config/private/IPrivateConfig"; +import {SettingsComponent} from "../_abstract/abstract.settings.component"; +import {Utils} from "../../../../common/Utils"; @Component({ selector: 'settings-indexing', @@ -14,61 +16,55 @@ import {Observable} from "rxjs/Rx"; './../_abstract/abstract.settings.component.css'], providers: [IndexingSettingsService], }) -export class IndexingSettingsComponent implements OnInit, OnDestroy { +export class IndexingSettingsComponent extends SettingsComponent { - @Input() - public simplifiedMode: boolean = true; - @Output('hasAvailableSettings') - hasAvailableSettings: boolean = true; - public inProgress = false; - public error: string = null; + + types: Array = []; + private subscription: { timer: any, settings: any } = { + timer: null, + settings: null + }; + private $counter: Observable = null; updateProgress = async () => { try { - await this._settingsService.getProgress(); + await (this._settingsService).getProgress(); } catch (err) { if (this.subscription.timer != null) { this.subscription.timer.unsubscribe(); this.subscription.timer = null; } } - if (this._settingsService.progress.value != null && this.subscription.timer == null) { + if ((this._settingsService).progress.value != null && this.subscription.timer == null) { if (!this.$counter) { this.$counter = Observable.interval(5000); } this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress()); } - if (this._settingsService.progress.value == null && this.subscription.timer != null) { + if ((this._settingsService).progress.value == null && this.subscription.timer != null) { this.subscription.timer.unsubscribe(); this.subscription.timer = null; } }; - private subscription: { timer: any, settings: any } = { - timer: null, - settings: null - }; - private $counter: Observable = null; - constructor(private _authService: AuthenticationService, - private _navigation: NavigationService, - public _settingsService: IndexingSettingsService, - private notification: NotificationService) { + constructor(_authService: AuthenticationService, + _navigation: NavigationService, + _settingsService: IndexingSettingsService, + notification: NotificationService) { + + super("Indexing", _authService, _navigation, _settingsService, notification, s => s.Server.indexing); + } async ngOnInit() { - if (!this._authService.isAuthenticated() || - this._authService.user.value.role < UserRoles.Admin) { - this._navigation.toLogin(); - return; - } - this.subscription.settings = this._settingsService.Settings.subscribe(() => { - this.hasAvailableSettings = (this._settingsService.isSupported() || !this.simplifiedMode); - }); - + super.ngOnInit(); + this.types = Utils + .enumToArray(ReIndexingSensitivity); this.updateProgress(); } ngOnDestroy() { + super.ngOnDestroy(); if (this.subscription.timer != null) { this.subscription.timer.unsubscribe(); this.subscription.timer = null; @@ -83,7 +79,7 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy { this.inProgress = true; this.error = ""; try { - await this._settingsService.index(); + await (this._settingsService).index(); this.updateProgress(); this.notification.success("Folder indexed", "Success"); this.inProgress = false; @@ -99,11 +95,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy { return false; } - async cancel() { + async cancelIndexing() { this.inProgress = true; this.error = ""; try { - await this._settingsService.cancel(); + await (this._settingsService).cancel(); this.notification.success("Folder indexed", "Success"); this.inProgress = false; return true; @@ -118,11 +114,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy { return false; } - async reset() { + async resetDatabase() { this.inProgress = true; this.error = ""; try { - await this._settingsService.reset(); + await (this._settingsService).reset(); this.notification.success('Database reset', "Success"); this.inProgress = false; return true; diff --git a/frontend/app/settings/indexing/indexing.settings.service.ts b/frontend/app/settings/indexing/indexing.settings.service.ts index 556c9edb..4666942f 100644 --- a/frontend/app/settings/indexing/indexing.settings.service.ts +++ b/frontend/app/settings/indexing/indexing.settings.service.ts @@ -2,12 +2,12 @@ import {Injectable} from "@angular/core"; import {NetworkService} from "../../model/network/network.service"; import {SettingsService} from "../settings.service"; import {AbstractSettingsService} from "../_abstract/abstract.settings.service"; -import {DatabaseType} from "../../../../common/config/private/IPrivateConfig"; +import {DatabaseType, IndexingConfig} from "../../../../common/config/private/IPrivateConfig"; import {IndexingProgressDTO} from "../../../../common/entities/settings/IndexingProgressDTO"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; @Injectable() -export class IndexingSettingsService extends AbstractSettingsService { +export class IndexingSettingsService extends AbstractSettingsService { public progress: BehaviorSubject; @@ -18,6 +18,11 @@ export class IndexingSettingsService extends AbstractSettingsService { this.progress = new BehaviorSubject(null); } + public updateSettings(settings: IndexingConfig): Promise { + return this._networkService.putJson("/settings/indexing", {settings: settings}); + } + + public isSupported(): boolean { return this._settingsService.settings.value.Server.database.type != DatabaseType.memory; } @@ -39,7 +44,4 @@ export class IndexingSettingsService extends AbstractSettingsService { } - updateSettings(settings: void): Promise { - throw new Error("Method not implemented."); - } } diff --git a/frontend/app/settings/settings.service.ts b/frontend/app/settings/settings.service.ts index 2a185db0..030e5a11 100644 --- a/frontend/app/settings/settings.service.ts +++ b/frontend/app/settings/settings.service.ts @@ -1,19 +1,27 @@ import {Injectable} from "@angular/core"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; -import {DatabaseType, IPrivateConfig, ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig"; +import { + DatabaseType, + IPrivateConfig, + ReIndexingSensitivity, + ThumbnailProcessingLib +} from "../../../common/config/private/IPrivateConfig"; import {NetworkService} from "../model/network/network.service"; + @Injectable() export class SettingsService { public settings: BehaviorSubject; constructor(private _networkService: NetworkService) { - this.settings = new BehaviorSubject({ + this.settings = new BehaviorSubject({ Client: { Search: { enabled: true, autocompleteEnabled: true, - instantSearchEnabled: true + instantSearchEnabled: true, + InstantSearchTimeout: 0 }, + concurrentThumbnailGenerations: null, Thumbnail: { iconSize: 30, thumbnailSizes: [] @@ -47,12 +55,18 @@ export class SettingsService { folder: "", qualityPriority: true, processingLibrary: ThumbnailProcessingLib.sharp + }, + sessionTimeout: 0, + indexing: { + cachedFolderTimeout: 0, + folderPreviewSize: 0, + reIndexingSensitivity: ReIndexingSensitivity.medium } } }); } - public async getSettings(): Promise { + public async getSettings(): Promise { this.settings.next(await >this._networkService.getJson("/settings")); }