improving indexing

backend/middlewares/AdminMWs.ts

@@ -3,7 +3,12 @@ import {ErrorCodes, ErrorDTO} from "../../common/entities/Error";
 import {ObjectManagerRepository} from "../model/ObjectManagerRepository";
 import {Logger} from "../Logger";
 import {SQLConnection} from "../model/sql/SQLConnection";
-import {DataBaseConfig, DatabaseType, ThumbnailConfig} from "../../common/config/private/IPrivateConfig";
+import {
+  DataBaseConfig,
+  DatabaseType,
+  IndexingConfig,
+  ThumbnailConfig
+} from "../../common/config/private/IPrivateConfig";
 import {Config} from "../../common/config/private/Config";
 import {ConfigDiagnostics} from "../model/ConfigDiagnostics";
 import {ClientConfig} from "../../common/config/public/ConfigClass";
@@ -237,6 +242,28 @@ export class AdminMWs {
     }
   }
 
+  public static async updateIndexingSettings(req: Request, res: Response, next: NextFunction) {
+    if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
+      return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
+    }
+
+    try {
+      const settings: IndexingConfig = req.body.settings;
+      Config.Server.indexing = settings;
+
+      //only updating explicitly set config (not saving config set by the diagnostics)
+      const original = Config.original();
+      original.Server.indexing = settings;
+      original.save();
+      await ConfigDiagnostics.runDiagnostics();
+      Logger.info(LOG_TAG, "new config:");
+      Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t'));
+      return next();
+    } catch (err) {
+      return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Settings error: " + JSON.stringify(err, null, '  '), err));
+    }
+  }
+
 
   public static startIndexing(req: Request, res: Response, next: NextFunction) {
     try {

backend/model/memory/GalleryManager.ts

@@ -5,6 +5,7 @@ import * as fs from "fs";
 import {DiskManager} from "../DiskManger";
 import {ProjectPath} from "../../ProjectPath";
 import {Config} from "../../../common/config/private/Config";
+import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig";
 
 export class GalleryManager implements IGalleryManager {
 
@@ -13,7 +14,9 @@ export class GalleryManager implements IGalleryManager {
     if (knownLastModified && knownLastScanned) {
       const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
       const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
-      if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && lastModified == knownLastModified) {
+      if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout &&
+        lastModified == knownLastModified &&
+        Config.Server.indexing.reIndexingSensitivity < ReIndexingSensitivity.high) {
         return Promise.resolve(null);
       }
     }

backend/model/sql/GalleryManager.ts

@@ -10,6 +10,7 @@ import {Utils} from "../../../common/Utils";
 import {ProjectPath} from "../../ProjectPath";
 import {Config} from "../../../common/config/private/Config";
 import {ISQLGalleryManager} from "./IGalleryManager";
+import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig";
 
 export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
@@ -37,10 +38,15 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
     if (dir && dir.scanned == true) {
       //iF it seems that the content did not changed, do not work on it
-      if (knownLastModified && knownLastScanned) {
+      if (knownLastModified && knownLastScanned
-        if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout &&
+        && lastModified == knownLastModified &&
-          lastModified == knownLastModified &&
+        dir.lastScanned == knownLastScanned) {
-          dir.lastScanned == knownLastScanned) {
+
+        if (Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.low) {
+          return null;
+        }
+        if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout &&
+          Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.medium) {
           return null;
         }
       }
@@ -61,7 +67,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
               dir: dir.directories[i].id
             })
             .orderBy("photo.metadata.creationDate", "ASC")
-            .setLimit(Config.Server.folderPreviewSize)
+            .setLimit(Config.Server.indexing.folderPreviewSize)
             .getMany();
           dir.directories[i].isPartial = true;
 
@@ -79,7 +85,9 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
         return this.indexDirectory(relativeDirectoryName);
       }
 
-      if (Date.now() - dir.lastScanned > Config.Server.cachedFolderTimeout) {
+      if ((Date.now() - dir.lastScanned > Config.Server.indexing.cachedFolderTimeout &&
+          Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.medium) ||
+        Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.high) {
         //on the fly reindexing
         this.indexDirectory(relativeDirectoryName).catch((err) => {
           console.error(err);

backend/model/threading/DiskMangerWorker.ts

@@ -55,7 +55,7 @@ export class DiskMangerWorker {
             const fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
             if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
               const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
-                Config.Server.folderPreviewSize, true
+                Config.Server.indexing.folderPreviewSize, true
               );
               d.lastScanned = 0; //it was not a fully scan
               d.isPartial = true;

backend/routes/AdminRouter.ts

@@ -96,6 +96,12 @@ export class AdminRouter {
       AdminMWs.updateOtherSettings,
       RenderingMWs.renderOK
     );
+    app.put("/api/settings/indexing",
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.updateIndexingSettings,
+      RenderingMWs.renderOK
+    );
 
   };
 

common/config/private/IPrivateConfig.ts

@@ -41,6 +41,16 @@ export interface SharingConfig {
   updateTimeout: number;
 }
 
+export enum ReIndexingSensitivity {
+  low, medium, high
+}
+
+export interface IndexingConfig {
+  folderPreviewSize: number;
+  cachedFolderTimeout: number;//Do not rescans the folder if seems ok
+  reIndexingSensitivity: ReIndexingSensitivity;
+}
+
 export interface ServerConfig {
   port: number;
   imagesFolder: string;
@@ -49,8 +59,7 @@ export interface ServerConfig {
   enableThreading: boolean;
   sharing: SharingConfig;
   sessionTimeout: number
-  folderPreviewSize: number;
+  indexing: IndexingConfig;
-  cachedFolderTimeout: number;//Do not rescans the folder if seems ok
 }
 
 export interface IPrivateConfig {

common/config/private/PrivateConfigClass.ts

@@ -1,5 +1,11 @@
 import {PublicConfigClass} from "../public/ConfigClass";
-import {DatabaseType, IPrivateConfig, ServerConfig, ThumbnailProcessingLib} from "./IPrivateConfig";
+import {
+  DatabaseType,
+  IPrivateConfig,
+  ReIndexingSensitivity,
+  ServerConfig,
+  ThumbnailProcessingLib
+} from "./IPrivateConfig";
 import * as path from "path";
 import {ConfigLoader} from "typeconfig";
 
@@ -33,9 +39,12 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
     sharing: {
       updateTimeout: 1000 * 60 * 5
     },
-    enableThreading: true,
+    indexing: {
-    folderPreviewSize: 2,
+      folderPreviewSize: 2,
-    cachedFolderTimeout: 1000 * 60 * 60
+      cachedFolderTimeout: 1000 * 60 * 60,
+      reIndexingSensitivity: ReIndexingSensitivity.high
+    },
+    enableThreading: true
   };
   private ConfigLoader: any;
 

frontend/app/settings/_abstract/abstract.settings.component.ts

@@ -23,8 +23,8 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
   public inProgress = false;
   public error: string = null;
   public changed: boolean = false;
-  private subscription = null;
+  private _subscription = null;
-  private settingsSubscription = null;
+  private _settingsSubscription = null;
 
   public settings: T = <any>{};
   public original: T = <any>{};
@@ -36,7 +36,7 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
               protected notification: NotificationService,
               private sliceFN?: (s: IPrivateConfig) => T) {
     if (this.sliceFN) {
-      this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
+      this._settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
       this.onNewSettings(this._settingsService._settingsService.settings.value);
     }
   }
@@ -55,7 +55,7 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
     }
     this.getSettings();
 
-    this.subscription = this.form.valueChanges.subscribe((data) => {
+    this._subscription = this.form.valueChanges.subscribe((data) => {
       this.changed = !Utils.equalsFilter(this.settings, this.original);
     });
 
@@ -67,11 +67,11 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
 
 
   ngOnDestroy() {
-    if (this.subscription != null) {
+    if (this._subscription != null) {
-      this.subscription.unsubscribe();
+      this._subscription.unsubscribe();
     }
-    if (this.settingsSubscription != null) {
+    if (this._settingsSubscription != null) {
-      this.settingsSubscription.unsubscribe();
+      this._settingsSubscription.unsubscribe();
     }
   }
 

frontend/app/settings/indexing/indexing.settings.component.html

@@ -6,6 +6,56 @@
     <div class="panel-body">
       <div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
 
+      <ng-container *ngIf="!simplifiedMode">
+        <div class="form-group">
+          <label class="col-sm-2 control-label" for="cachedFolderTimeout">Index cache timeout [ms]</label>
+          <div class="col-sm-10">
+            <input type="number" class="form-control" placeholder="36000"
+                   id="cachedFolderTimeout"
+                   min="0"
+                   step="1"
+                   [(ngModel)]="settings.cachedFolderTimeout"
+                   name="cachedFolderTimeout" required>
+            <span class="help-block">If there was no indexing in this time, it reindexes. (skipped if indexen in DB and sensitivity is low)</span>
+          </div>
+        </div>
+        <div class="form-group">
+          <label class="col-sm-2 control-label" for="folderPreviewSize">Sub folder preview size</label>
+          <div class="col-sm-10">
+            <input type="number" class="form-control" placeholder="1"
+                   id="folderPreviewSize"
+                   min="0"
+                   step="1"
+                   [(ngModel)]="settings.folderPreviewSize"
+                   name="folderPreviewSize" required>
+            <span class="help-block">Reads this many photos from sub folders</span>
+          </div>
+        </div>
+
+        <div class="form-group">
+          <label class="col-sm-2 control-label" for="reIndexingSensitivity">Folder reindexing sensitivity</label>
+          <div class="col-sm-10">
+            <select id="reIndexingSensitivity" class="form-control" [(ngModel)]="settings.reIndexingSensitivity"
+                    name="reIndexingSensitivity" required>
+              <option *ngFor="let type of types" [ngValue]="type.key">{{type.value}}
+              </option>
+            </select>
+            <span
+              class="help-block">Set the reindexing sensitivity. High value check the folders for change more often</span>
+          </div>
+        </div>
+
+
+        <button class="btn btn-success pull-right"
+                [disabled]="!settingsForm.form.valid || !changed || inProgress"
+                (click)="save()">Save
+        </button>
+        <button class="btn btn-default pull-right"
+                (click)="reset()">Reset
+        </button>
+        <br/>
+        <hr/>
+      </ng-container>
       If you add a new folder to your gallery, the site indexes it automatically.
       If you would like to trigger indexing manually, click index button.<br/>
       (Note: search ony searched among the indexed directories)<br/>
@@ -34,11 +84,11 @@
         <button class="btn btn-default"
                 *ngIf="_settingsService.progress.value != null"
                 [disabled]="inProgress"
-                (click)="cancel()">Cancel
+                (click)="cancelIndexing()">Cancel
         </button>
         <button class="btn btn-danger"
                 [disabled]="inProgress"
-                (click)="reset()">Reset Indexes
+                (click)="resetDatabase()">Reset Indexes
         </button>
       </div>
     </div>

frontend/app/settings/indexing/indexing.settings.component.ts

@@ -1,11 +1,13 @@
-import {Component, Input, OnDestroy, OnInit, Output} from "@angular/core";
+import {Component} from "@angular/core";
 import {IndexingSettingsService} from "./indexing.settings.service";
 import {AuthenticationService} from "../../model/network/authentication.service";
 import {NavigationService} from "../../model/navigation.service";
 import {NotificationService} from "../../model/notification.service";
 import {ErrorDTO} from "../../../../common/entities/Error";
-import {UserRoles} from "../../../../common/entities/UserDTO";
 import {Observable} from "rxjs/Rx";
+import {IndexingConfig, ReIndexingSensitivity} from "../../../../common/config/private/IPrivateConfig";
+import {SettingsComponent} from "../_abstract/abstract.settings.component";
+import {Utils} from "../../../../common/Utils";
 
 @Component({
   selector: 'settings-indexing',
@@ -14,61 +16,55 @@ import {Observable} from "rxjs/Rx";
     './../_abstract/abstract.settings.component.css'],
   providers: [IndexingSettingsService],
 })
-export class IndexingSettingsComponent implements OnInit, OnDestroy {
+export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig> {
 
-  @Input()
+
-  public simplifiedMode: boolean = true;
+  types: Array<any> = [];
-  @Output('hasAvailableSettings')
+  private subscription: { timer: any, settings: any } = {
-  hasAvailableSettings: boolean = true;
+    timer: null,
-  public inProgress = false;
+    settings: null
-  public error: string = null;
+  };
+  private $counter: Observable<number> = null;
   updateProgress = async () => {
     try {
-      await this._settingsService.getProgress();
+      await (<IndexingSettingsService>this._settingsService).getProgress();
     } catch (err) {
       if (this.subscription.timer != null) {
         this.subscription.timer.unsubscribe();
         this.subscription.timer = null;
       }
     }
-    if (this._settingsService.progress.value != null && this.subscription.timer == null) {
+    if ((<IndexingSettingsService>this._settingsService).progress.value != null && this.subscription.timer == null) {
       if (!this.$counter) {
         this.$counter = Observable.interval(5000);
       }
       this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress());
     }
-    if (this._settingsService.progress.value == null && this.subscription.timer != null) {
+    if ((<IndexingSettingsService>this._settingsService).progress.value == null && this.subscription.timer != null) {
       this.subscription.timer.unsubscribe();
       this.subscription.timer = null;
     }
 
   };
-  private subscription: { timer: any, settings: any } = {
-    timer: null,
-    settings: null
-  };
-  private $counter: Observable<number> = null;
 
-  constructor(private _authService: AuthenticationService,
+  constructor(_authService: AuthenticationService,
-              private _navigation: NavigationService,
+              _navigation: NavigationService,
-              public _settingsService: IndexingSettingsService,
+              _settingsService: IndexingSettingsService,
-              private notification: NotificationService) {
+              notification: NotificationService) {
+
+    super("Indexing", _authService, _navigation, <any>_settingsService, notification, s => s.Server.indexing);
+
   }
 
   async ngOnInit() {
-    if (!this._authService.isAuthenticated() ||
+    super.ngOnInit();
-      this._authService.user.value.role < UserRoles.Admin) {
+    this.types = Utils
-      this._navigation.toLogin();
+      .enumToArray(ReIndexingSensitivity);
-      return;
-    }
-    this.subscription.settings = this._settingsService.Settings.subscribe(() => {
-      this.hasAvailableSettings = (this._settingsService.isSupported() || !this.simplifiedMode);
-    });
-
     this.updateProgress();
   }
 
   ngOnDestroy() {
+    super.ngOnDestroy();
     if (this.subscription.timer != null) {
       this.subscription.timer.unsubscribe();
       this.subscription.timer = null;
@@ -83,7 +79,7 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
     this.inProgress = true;
     this.error = "";
     try {
-      await this._settingsService.index();
+      await (<IndexingSettingsService>this._settingsService).index();
       this.updateProgress();
       this.notification.success("Folder indexed", "Success");
       this.inProgress = false;
@@ -99,11 +95,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
     return false;
   }
 
-  async cancel() {
+  async cancelIndexing() {
     this.inProgress = true;
     this.error = "";
     try {
-      await this._settingsService.cancel();
+      await (<IndexingSettingsService>this._settingsService).cancel();
       this.notification.success("Folder indexed", "Success");
       this.inProgress = false;
       return true;
@@ -118,11 +114,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
     return false;
   }
 
-  async reset() {
+  async resetDatabase() {
     this.inProgress = true;
     this.error = "";
     try {
-      await this._settingsService.reset();
+      await (<IndexingSettingsService>this._settingsService).reset();
       this.notification.success('Database reset', "Success");
       this.inProgress = false;
       return true;

frontend/app/settings/indexing/indexing.settings.service.ts

@@ -2,12 +2,12 @@ import {Injectable} from "@angular/core";
 import {NetworkService} from "../../model/network/network.service";
 import {SettingsService} from "../settings.service";
 import {AbstractSettingsService} from "../_abstract/abstract.settings.service";
-import {DatabaseType} from "../../../../common/config/private/IPrivateConfig";
+import {DatabaseType, IndexingConfig} from "../../../../common/config/private/IPrivateConfig";
 import {IndexingProgressDTO} from "../../../../common/entities/settings/IndexingProgressDTO";
 import {BehaviorSubject} from "rxjs/BehaviorSubject";
 
 @Injectable()
-export class IndexingSettingsService extends AbstractSettingsService<void> {
+export class IndexingSettingsService extends AbstractSettingsService<IndexingConfig> {
 
 
   public progress: BehaviorSubject<IndexingProgressDTO>;
@@ -18,6 +18,11 @@ export class IndexingSettingsService extends AbstractSettingsService<void> {
     this.progress = new BehaviorSubject(null);
   }
 
+  public updateSettings(settings: IndexingConfig): Promise<void> {
+    return this._networkService.putJson("/settings/indexing", {settings: settings});
+  }
+
+
   public isSupported(): boolean {
     return this._settingsService.settings.value.Server.database.type != DatabaseType.memory;
   }
@@ -39,7 +44,4 @@ export class IndexingSettingsService extends AbstractSettingsService<void> {
   }
 
 
-  updateSettings(settings: void): Promise<void> {
-    throw new Error("Method not implemented.");
-  }
 }

frontend/app/settings/settings.service.ts

@@ -1,19 +1,27 @@
 import {Injectable} from "@angular/core";
 import {BehaviorSubject} from "rxjs/BehaviorSubject";
-import {DatabaseType, IPrivateConfig, ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
+import {
+  DatabaseType,
+  IPrivateConfig,
+  ReIndexingSensitivity,
+  ThumbnailProcessingLib
+} from "../../../common/config/private/IPrivateConfig";
 import {NetworkService} from "../model/network/network.service";
+
 @Injectable()
 export class SettingsService {
   public settings: BehaviorSubject<IPrivateConfig>;
 
   constructor(private _networkService: NetworkService) {
-    this.settings = new BehaviorSubject<IPrivateConfig>(<any>{
+    this.settings = new BehaviorSubject<IPrivateConfig>({
       Client: {
         Search: {
           enabled: true,
           autocompleteEnabled: true,
-          instantSearchEnabled: true
+          instantSearchEnabled: true,
+          InstantSearchTimeout: 0
         },
+        concurrentThumbnailGenerations: null,
         Thumbnail: {
           iconSize: 30,
           thumbnailSizes: []
@@ -47,12 +55,18 @@ export class SettingsService {
           folder: "",
           qualityPriority: true,
           processingLibrary: ThumbnailProcessingLib.sharp
+        },
+        sessionTimeout: 0,
+        indexing: {
+          cachedFolderTimeout: 0,
+          folderPreviewSize: 0,
+          reIndexingSensitivity: ReIndexingSensitivity.medium
         }
       }
     });
   }
 
-  public async  getSettings(): Promise<void> {
+  public async getSettings(): Promise<void> {
     this.settings.next(await <Promise<IPrivateConfig>>this._networkService.getJson("/settings"));
   }