implementing manual directory indexing

backend/middlewares/AdminMWs.ts

@@ -13,6 +13,7 @@ import {ProjectPath} from "../ProjectPath";
 
 
 const LOG_TAG = "[AdminMWs]";
+
 export class AdminMWs {
 
 
@@ -237,4 +238,42 @@ export class AdminMWs {
   }
 
 
+  public static startIndexing(req: Request, res: Response, next: NextFunction) {
+    try {
+      ObjectManagerRepository.getInstance().IndexingManager.startIndexing();
+      req.resultPipe = "ok";
+      return next();
+    } catch (err) {
+      return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, '  '), err));
+    }
+  }
+
+  public static getIndexingProgress(req: Request, res: Response, next: NextFunction) {
+    try {
+      req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress();
+      return next();
+    } catch (err) {
+      return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, '  '), err));
+    }
+  }
+
+  public static cancelIndexing(req: Request, res: Response, next: NextFunction) {
+    try {
+      ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing();
+      req.resultPipe = "ok";
+      return next();
+    } catch (err) {
+      return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, '  '), err));
+    }
+  }
+
+  public static async resetIndexes(req: Request, res: Response, next: NextFunction) {
+    try {
+      await ObjectManagerRepository.getInstance().IndexingManager.reset();
+      req.resultPipe = "ok";
+      return next();
+    } catch (err) {
+      return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, '  '), err));
+    }
+  }
 }

backend/middlewares/RenderingMWs.ts

@@ -12,7 +12,7 @@ import {Logger} from "../Logger";
 export class RenderingMWs {
 
   public static renderResult(req: Request, res: Response, next: NextFunction) {
-    if (!req.resultPipe)
+    if (typeof req.resultPipe == "undefined")
       return next();
 
     return RenderingMWs.renderMessage(res, req.resultPipe);

backend/model/DiskManger.ts

@@ -7,8 +7,6 @@ import {DiskMangerWorker} from "./threading/DiskMangerWorker";
 
 
 const LOG_TAG = "[DiskManager]";
-
-
 export class DiskManager {
   static threadPool: DiskManagerTH = null;
 

backend/model/ObjectManagerRepository.ts

@@ -4,6 +4,7 @@ import {ISearchManager} from "./interfaces/ISearchManager";
 import {SQLConnection} from "./sql/SQLConnection";
 import {ISharingManager} from "./interfaces/ISharingManager";
 import {Logger} from "../Logger";
+import {IIndexingManager} from "./interfaces/IIndexingManager";
 
 export class ObjectManagerRepository {
 
@@ -11,33 +12,15 @@ export class ObjectManagerRepository {
   private _userManager: IUserManager;
   private _searchManager: ISearchManager;
   private _sharingManager: ISharingManager;
+  private _indexingManager: IIndexingManager;
   private static _instance: ObjectManagerRepository = null;
 
-
-  public static async InitMemoryManagers() {
-    await ObjectManagerRepository.reset();
-    const GalleryManager = require("./memory/GalleryManager").GalleryManager;
-    const UserManager = require("./memory/UserManager").UserManager;
-    const SearchManager = require("./memory/SearchManager").SearchManager;
-    const SharingManager = require("./memory/SharingManager").SharingManager;
-    ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
-    ObjectManagerRepository.getInstance().UserManager = new UserManager();
-    ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
-    ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+  get IndexingManager(): IIndexingManager {
+    return this._indexingManager;
   }
 
-  public static async InitSQLManagers() {
-    await ObjectManagerRepository.reset();
-    await SQLConnection.init();
-    const GalleryManager = require("./sql/GalleryManager").GalleryManager;
-    const UserManager = require("./sql/UserManager").UserManager;
-    const SearchManager = require("./sql/SearchManager").SearchManager;
-    const SharingManager = require("./sql/SharingManager").SharingManager;
-    ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
-    ObjectManagerRepository.getInstance().UserManager = new UserManager();
-    ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
-    ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
-    Logger.debug("SQL DB inited");
+  set IndexingManager(value: IIndexingManager) {
+    this._indexingManager = value;
   }
 
   public static getInstance() {
@@ -85,4 +68,34 @@ export class ObjectManagerRepository {
     this._sharingManager = value;
   }
 
+  public static async InitMemoryManagers() {
+    await ObjectManagerRepository.reset();
+    const GalleryManager = require("./memory/GalleryManager").GalleryManager;
+    const UserManager = require("./memory/UserManager").UserManager;
+    const SearchManager = require("./memory/SearchManager").SearchManager;
+    const SharingManager = require("./memory/SharingManager").SharingManager;
+    const IndexingManager = require("./memory/IndexingManager").IndexingManager;
+    ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
+    ObjectManagerRepository.getInstance().UserManager = new UserManager();
+    ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
+    ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+  }
+
+  public static async InitSQLManagers() {
+    await ObjectManagerRepository.reset();
+    await SQLConnection.init();
+    const GalleryManager = require("./sql/GalleryManager").GalleryManager;
+    const UserManager = require("./sql/UserManager").UserManager;
+    const SearchManager = require("./sql/SearchManager").SearchManager;
+    const SharingManager = require("./sql/SharingManager").SharingManager;
+    const IndexingManager = require("./sql/IndexingManager").IndexingManager;
+    ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
+    ObjectManagerRepository.getInstance().UserManager = new UserManager();
+    ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
+    ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
+    ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
+    Logger.debug("SQL DB inited");
+  }
+
 }

backend/model/interfaces/IGalleryManager.ts

@@ -4,4 +4,5 @@ export interface IGalleryManager {
   listDirectory(relativeDirectoryName: string,
                 knownLastModified?: number,
                 knownLastScanned?: number): Promise<DirectoryDTO>;
+
 }

backend/model/interfaces/IIndexingManager.ts

@@ -0,0 +1,11 @@
+import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
+
+export interface IIndexingManager {
+  startIndexing(): void;
+
+  getProgress(): IndexingProgressDTO;
+
+  cancelIndexing(): void;
+
+  reset(): Promise<void> ;
+}

backend/model/memory/IndexingManager.ts

@@ -0,0 +1,21 @@
+import {IIndexingManager} from "../interfaces/IIndexingManager";
+import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
+
+export class IndexingManager implements IIndexingManager {
+
+  startIndexing(): void {
+    throw new Error("not supported by memory DB");
+  }
+
+  getProgress(): IndexingProgressDTO {
+    throw new Error("not supported by memory DB");
+  }
+
+  cancelIndexing(): void {
+    throw new Error("not supported by memory DB");
+  }
+
+  reset(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+}

backend/model/sql/GalleryManager.ts

@@ -9,8 +9,9 @@ import {PhotoEntity, PhotoMetadataEntity} from "./enitites/PhotoEntity";
 import {Utils} from "../../../common/Utils";
 import {ProjectPath} from "../../ProjectPath";
 import {Config} from "../../../common/config/private/Config";
+import {ISQLGalleryManager} from "./IGalleryManager";
 
-export class GalleryManager implements IGalleryManager {
+export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
 
   public async listDirectory(relativeDirectoryName: string,

backend/model/sql/IGalleryManager.ts

@@ -0,0 +1,10 @@
+import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+
+export interface ISQLGalleryManager {
+  listDirectory(relativeDirectoryName: string,
+                knownLastModified?: number,
+                knownLastScanned?: number): Promise<DirectoryDTO>;
+
+  indexDirectory(relativeDirectoryName): Promise<DirectoryDTO>;
+
+}

backend/model/sql/IndexingManager.ts

@@ -0,0 +1,80 @@
+import {IIndexingManager} from "../interfaces/IIndexingManager";
+import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
+import {ObjectManagerRepository} from "../ObjectManagerRepository";
+import {ISQLGalleryManager} from "./IGalleryManager";
+import * as path from "path";
+import {SQLConnection} from "./SQLConnection";
+import {DirectoryEntity} from "./enitites/DirectoryEntity";
+import {Logger} from "../../Logger";
+
+const LOG_TAG = "[IndexingManager]";
+
+export class IndexingManager implements IIndexingManager {
+  directoriesToIndex: string[] = [];
+  indexingProgress = null;
+  enabled = false;
+  private indexNewDirectory = async () => {
+    if (this.directoriesToIndex.length == 0) {
+      this.indexingProgress = null;
+      if (global.gc) {
+        global.gc();
+      }
+      return;
+    }
+    const directory = this.directoriesToIndex.shift();
+    this.indexingProgress.current = directory;
+    this.indexingProgress.left = this.directoriesToIndex.length;
+    const scanned = await (<ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory);
+    if (this.enabled == false) {
+      return;
+    }
+    this.indexingProgress.indexed++;
+    for (let i = 0; i < scanned.directories.length; i++) {
+      this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name))
+    }
+    process.nextTick(this.indexNewDirectory);
+  };
+
+  startIndexing(): void {
+    if (this.directoriesToIndex.length == 0 && this.enabled == false) {
+      Logger.info(LOG_TAG, "Starting indexing");
+      this.indexingProgress = <IndexingProgressDTO>{
+        indexed: 0,
+        left: 0,
+        current: ""
+      };
+      this.directoriesToIndex.push("/");
+      this.enabled = true;
+      this.indexNewDirectory();
+    } else {
+      Logger.info(LOG_TAG, "Already indexing..");
+    }
+  }
+
+  getProgress(): IndexingProgressDTO {
+    return this.indexingProgress;
+  }
+
+  cancelIndexing(): void {
+    Logger.info(LOG_TAG, "Canceling indexing");
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    if (global.gc) {
+      global.gc();
+    }
+  }
+
+  async reset(): Promise<void> {
+    Logger.info(LOG_TAG, "Resetting DB");
+    this.directoriesToIndex = [];
+    this.indexingProgress = null;
+    this.enabled = false;
+    const connection = await SQLConnection.getConnection();
+    return connection
+      .getRepository(DirectoryEntity)
+      .createQueryBuilder("directory")
+      .delete()
+      .execute();
+  }
+}

backend/model/threading/DiskMangerWorker.ts

@@ -155,7 +155,7 @@ export class DiskMangerWorker {
               metadata.creationDate = <number> (iptcData.date_time ? iptcData.date_time.getTime() : metadata.creationDate);
 
             } catch (err) {
-              Logger.debug(LOG_TAG, "Error parsing iptc data", fullPath, err);
+              // Logger.debug(LOG_TAG, "Error parsing iptc data", fullPath, err);
             }
 
             metadata.creationDate = metadata.creationDate || 0;

backend/routes/AdminRouter.ts

@@ -6,24 +6,35 @@ import {AdminMWs} from "../middlewares/AdminMWs";
 export class AdminRouter {
   public static route(app: any) {
 
-    this.addResetDB(app);
     this.addIndexGallery(app);
     this.addSettings(app);
   }
 
-  private static addResetDB(app) {
-    app.post("/api/admin/db/reset",
-      AuthenticationMWs.authenticate,
-      AuthenticationMWs.authorise(UserRoles.Admin)
-      //TODO: implement
-    );
-  };
 
   private static addIndexGallery(app) {
-    app.post("/api/admin/gallery/index",
+    app.get("/api/admin/indexes/job/progress",
       AuthenticationMWs.authenticate,
-      AuthenticationMWs.authorise(UserRoles.Admin)
-      //TODO: implement
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.getIndexingProgress,
+      RenderingMWs.renderResult
+    );
+    app.put("/api/admin/indexes/job",
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.startIndexing,
+      RenderingMWs.renderResult
+    );
+    app.delete("/api/admin/indexes/job",
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.cancelIndexing,
+      RenderingMWs.renderResult
+    );
+    app.delete("/api/admin/indexes",
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.authorise(UserRoles.Admin),
+      AdminMWs.resetIndexes,
+      RenderingMWs.renderResult
     );
   };
 

common/entities/settings/IndexingProgressDTO.ts

@@ -0,0 +1,5 @@
+export interface IndexingProgressDTO {
+  indexed: number;
+  left: number;
+  current: string;
+}

frontend/app/admin/admin.component.html

@@ -50,5 +50,7 @@
     <settings-share #share [hidden]="!share.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-share>
     <settings-map #map [hidden]="!map.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-map>
     <settings-other #other [hidden]="!other.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-other>
+    <settings-indexing #other [hidden]="!other.hasAvailableSettings"
+                       [simplifiedMode]="simplifiedMode"></settings-indexing>
   </div>
 </app-frame>

frontend/app/app.module.ts

@@ -54,6 +54,7 @@ import {ShareSettingsComponent} from "./settings/share/share.settings.component"
 import {BasicSettingsComponent} from "./settings/basic/basic.settings.component";
 import {OtherSettingsComponent} from "./settings/other/other.settings.component";
 import {DefaultUrlSerializer, UrlSerializer, UrlTree} from '@angular/router';
+import {IndexingSettingsComponent} from "./settings/indexing/indexing.settings.component";
 
 @Injectable()
 export class GoogleMapsConfig {
@@ -129,6 +130,7 @@ export class CustomUrlSerializer implements UrlSerializer {
     ShareSettingsComponent,
     BasicSettingsComponent,
     OtherSettingsComponent,
+    IndexingSettingsComponent,
     StringifyRole],
   providers: [
     {provide: UrlSerializer, useClass: CustomUrlSerializer},

frontend/app/settings/_abstract/abstract.settings.component.ts

@@ -34,10 +34,12 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
               private _navigation: NavigationService,
               public _settingsService: AbstractSettingsService<T>,
               protected notification: NotificationService,
-              private sliceFN: (s: IPrivateConfig) => T) {
+              private sliceFN?: (s: IPrivateConfig) => T) {
+    if (this.sliceFN) {
       this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
       this.onNewSettings(this._settingsService._settingsService.settings.value);
     }
+  }
 
   onNewSettings = (s) => {
     this.settings = Utils.clone(this.sliceFN(s));

frontend/app/settings/indexing/indexing.settings.component.css

@@ -0,0 +1,3 @@
+.buttons {
+  text-align: center;
+}

frontend/app/settings/indexing/indexing.settings.component.html

@@ -0,0 +1,47 @@
+<form #settingsForm="ngForm" class="form-horizontal">
+  <div class="panel panel-default">
+    <div class="panel-heading">
+      <h3 class="panel-title col-xs-4">Folder indexing</h3>
+    </div>
+    <div class="panel-body">
+      <div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
+
+      If you add a new folder to your gallery, the site indexes it automatically.
+      If you would like to trigger indexing manually, click index button.<br/>
+      (Note: search ony searched among the indexed directories)<br/>
+
+      <div *ngIf="_settingsService.progress.value != null">
+        indexing: {{_settingsService.progress.value.current}}
+        <div class="progress">
+          <div class="progress-bar  progress-bar-success"
+               role="progressbar"
+               aria-valuenow="2"
+               aria-valuemin="0"
+               aria-valuemax="100"
+               style="min-width: 2em;"
+               [style.width.%]="(_settingsService.progress.value.indexed/(_settingsService.progress.value.left+_settingsService.progress.value.indexed))*100">
+            {{_settingsService.progress.value.indexed}}/{{_settingsService.progress.value.indexed+_settingsService.progress.value.left}}
+          </div>
+        </div>
+      </div>
+
+      <div class="row buttons">
+        <button class="btn btn-success"
+                *ngIf="_settingsService.progress.value == null"
+                [disabled]="inProgress"
+                (click)="index()">Index
+        </button>
+        <button class="btn btn-default"
+                *ngIf="_settingsService.progress.value != null"
+                [disabled]="inProgress"
+                (click)="cancel()">Cancel
+        </button>
+        <button class="btn btn-danger"
+                [disabled]="inProgress"
+                (click)="reset()">Reset Indexes
+        </button>
+      </div>
+    </div>
+  </div>
+
+</form>

frontend/app/settings/indexing/indexing.settings.component.ts

@@ -0,0 +1,144 @@
+import {Component, Input, OnDestroy, OnInit, Output} from "@angular/core";
+import {IndexingSettingsService} from "./indexing.settings.service";
+import {AuthenticationService} from "../../model/network/authentication.service";
+import {NavigationService} from "../../model/navigation.service";
+import {NotificationService} from "../../model/notification.service";
+import {ErrorDTO} from "../../../../common/entities/Error";
+import {UserRoles} from "../../../../common/entities/UserDTO";
+import {Observable} from "rxjs/Rx";
+
+@Component({
+  selector: 'settings-indexing',
+  templateUrl: './indexing.settings.component.html',
+  styleUrls: ['./indexing.settings.component.css',
+    './../_abstract/abstract.settings.component.css'],
+  providers: [IndexingSettingsService],
+})
+export class IndexingSettingsComponent implements OnInit, OnDestroy {
+
+  @Input()
+  public simplifiedMode: boolean = true;
+  @Output('hasAvailableSettings')
+  hasAvailableSettings: boolean = true;
+  public inProgress = false;
+  public error: string = null;
+  updateProgress = async () => {
+    try {
+      await this._settingsService.getProgress();
+    } catch (err) {
+      if (this.subscription.timer != null) {
+        this.subscription.timer.unsubscribe();
+        this.subscription.timer = null;
+      }
+    }
+    if (this._settingsService.progress.value != null && this.subscription.timer == null) {
+      if (!this.$counter) {
+        this.$counter = Observable.interval(5000);
+      }
+      this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress());
+    }
+    if (this._settingsService.progress.value == null && this.subscription.timer != null) {
+      this.subscription.timer.unsubscribe();
+      this.subscription.timer = null;
+    }
+
+  };
+  private subscription: { timer: any, settings: any } = {
+    timer: null,
+    settings: null
+  };
+  private $counter: Observable<number> = null;
+
+  constructor(private _authService: AuthenticationService,
+              private _navigation: NavigationService,
+              public _settingsService: IndexingSettingsService,
+              private notification: NotificationService) {
+  }
+
+  async ngOnInit() {
+    if (!this._authService.isAuthenticated() ||
+      this._authService.user.value.role < UserRoles.Admin) {
+      this._navigation.toLogin();
+      return;
+    }
+    this.subscription.settings = this._settingsService.Settings.subscribe(() => {
+      this.hasAvailableSettings = (this._settingsService.isSupported() || !this.simplifiedMode);
+    });
+
+    this.updateProgress();
+  }
+
+  ngOnDestroy() {
+    if (this.subscription.timer != null) {
+      this.subscription.timer.unsubscribe();
+      this.subscription.timer = null;
+    }
+    if (this.subscription.settings != null) {
+      this.subscription.settings.unsubscribe();
+      this.subscription.settings = null;
+    }
+  }
+
+  async index() {
+    this.inProgress = true;
+    this.error = "";
+    try {
+      await this._settingsService.index();
+      this.updateProgress();
+      this.notification.success("Folder indexed", "Success");
+      this.inProgress = false;
+      return true;
+    } catch (err) {
+      console.log(err);
+      if (err.message) {
+        this.error = (<ErrorDTO>err).message;
+      }
+    }
+
+    this.inProgress = false;
+    return false;
+  }
+
+  async cancel() {
+    this.inProgress = true;
+    this.error = "";
+    try {
+      await this._settingsService.cancel();
+      this.notification.success("Folder indexed", "Success");
+      this.inProgress = false;
+      return true;
+    } catch (err) {
+      console.log(err);
+      if (err.message) {
+        this.error = (<ErrorDTO>err).message;
+      }
+    }
+
+    this.inProgress = false;
+    return false;
+  }
+
+  async reset() {
+    this.inProgress = true;
+    this.error = "";
+    try {
+      await this._settingsService.reset();
+      this.notification.success('Database reset', "Success");
+      this.inProgress = false;
+      return true;
+    } catch (err) {
+      console.log(err);
+      if (err.message) {
+        this.error = (<ErrorDTO>err).message;
+      }
+    }
+
+    this.inProgress = false;
+    return false;
+  }
+
+
+}
+
+
+

frontend/app/settings/indexing/indexing.settings.service.ts

@@ -0,0 +1,45 @@
+import {Injectable} from "@angular/core";
+import {NetworkService} from "../../model/network/network.service";
+import {SettingsService} from "../settings.service";
+import {AbstractSettingsService} from "../_abstract/abstract.settings.service";
+import {DatabaseType} from "../../../../common/config/private/IPrivateConfig";
+import {IndexingProgressDTO} from "../../../../common/entities/settings/IndexingProgressDTO";
+import {BehaviorSubject} from "rxjs/BehaviorSubject";
+
+@Injectable()
+export class IndexingSettingsService extends AbstractSettingsService<void> {
+
+
+  public progress: BehaviorSubject<IndexingProgressDTO>;
+
+  constructor(private _networkService: NetworkService,
+              _settingsService: SettingsService) {
+    super(_settingsService);
+    this.progress = new BehaviorSubject(null);
+  }
+
+  public isSupported(): boolean {
+    return this._settingsService.settings.value.Server.database.type != DatabaseType.memory;
+  }
+
+  public index() {
+    return this._networkService.putJson("/admin/indexes/job");
+  }
+
+  public cancel() {
+    return this._networkService.deleteJson("/admin/indexes/job");
+  }
+
+  public async getProgress() {
+    this.progress.next(await this._networkService.getJson<IndexingProgressDTO>("/admin/indexes/job/progress"));
+  }
+
+  public reset() {
+    return this._networkService.deleteJson("/admin/indexes");
+  }
+
+
+  updateSettings(settings: void): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+}