improving lazy loading

backend/middlewares/GalleryMWs.ts

@@ -2,7 +2,7 @@ import * as path from "path";
 import * as fs from "fs";
 import {NextFunction, Request, Response} from "express";
 import {ErrorCodes, ErrorDTO} from "../../common/entities/Error";
-import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../common/entities/DirectoryDTO";
 import {ObjectManagerRepository} from "../model/ObjectManagerRepository";
 import {SearchTypes} from "../../common/entities/AutoCompleteItem";
 import {ContentWrapper} from "../../common/entities/ConentWrapper";
@@ -13,6 +13,7 @@ import {UserDTO} from "../../common/entities/UserDTO";
 
 
 const LOG_TAG = "[GalleryMWs]";
+
 export class GalleryMWs {
 
 
@@ -26,11 +27,15 @@ export class GalleryMWs {
 
     try {
 
-      const directory = await ObjectManagerRepository.getInstance().GalleryManager.listDirectory(directoryName);
+      const directory = await ObjectManagerRepository.getInstance().GalleryManager.listDirectory(directoryName, req.query.knownLastModified, req.query.knownLastScanned);
+      if ((<NotModifiedDirectoryDTO>directory).notModified == true) {
+        req.resultPipe = new ContentWrapper(directory, null);
+        return next();
+      }
       if (req.session.user.permissions &&
         req.session.user.permissions.length > 0 &&
         req.session.user.permissions[0] != "/") {
-        directory.directories = directory.directories.filter(d =>
+        (<DirectoryDTO>directory).directories = (<DirectoryDTO>directory).directories.filter(d =>
           UserDTO.isDirectoryAvailable(d, req.session.user.permissions));
       }
       req.resultPipe = new ContentWrapper(directory, null);
@@ -47,6 +52,9 @@ export class GalleryMWs {
       return next();
 
     let cw: ContentWrapper = req.resultPipe;
+    if ((<NotModifiedDirectoryDTO>cw.directory).notModified == true) {
+      return next();
+    }
     let removeDirs = (dir) => {
       dir.photos.forEach((photo: PhotoDTO) => {
         photo.directory = null;
@@ -84,7 +92,6 @@ export class GalleryMWs {
     }
 
 
-
     req.resultPipe = fullImagePath;
     return next();
   }

backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts

@@ -6,7 +6,7 @@ import * as os from "os";
 import {NextFunction, Request, Response} from "express";
 import {ErrorCodes, ErrorDTO} from "../../../common/entities/Error";
 import {ContentWrapper} from "../../../common/entities/ConentWrapper";
-import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import {ProjectPath} from "../../ProjectPath";
 import {PhotoDTO} from "../../../common/entities/PhotoDTO";
 import {Config} from "../../../common/config/private/Config";
@@ -85,8 +85,11 @@ export class ThumbnailGeneratorMWs {
       return next();
 
     let cw: ContentWrapper = req.resultPipe;
+    if ((<NotModifiedDirectoryDTO>cw.directory).notModified == true) {
+      return next();
+    }
     if (cw.directory) {
-      ThumbnailGeneratorMWs.addThInfoTODir(cw.directory);
+      ThumbnailGeneratorMWs.addThInfoTODir(<DirectoryDTO>cw.directory);
     }
     if (cw.searchResult) {
       ThumbnailGeneratorMWs.addThInfoToPhotos(cw.searchResult.photos);

backend/model/interfaces/IGalleryManager.ts

@@ -1,4 +1,7 @@
-import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO";
+
 export interface IGalleryManager {
-  listDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
+  listDirectory(relativeDirectoryName: string,
+                knownLastModified?: number,
+                knownLastScanned?: number): Promise<DirectoryDTO | NotModifiedDirectoryDTO>;
 }

backend/model/memory/GalleryManager.ts

@@ -1,10 +1,22 @@
-import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import {IGalleryManager} from "../interfaces/IGalleryManager";
+import * as path from "path";
+import * as fs from "fs";
 import {DiskManager} from "../DiskManger";
+import {ProjectPath} from "../../ProjectPath";
+import {Config} from "../../../common/config/private/Config";
 
 export class GalleryManager implements IGalleryManager {
 
-  public listDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
+  public listDirectory(relativeDirectoryName: string, knownLastModified?: number, knownLastScanned?: number): Promise<DirectoryDTO | NotModifiedDirectoryDTO> {
+    //If it seems that the content did not changed, do not work on it
+    if (knownLastModified && knownLastScanned) {
+      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
+      const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
+      if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && lastModified == knownLastModified) {
+        return Promise.resolve(<NotModifiedDirectoryDTO>{notModified: true});
+      }
+    }
     return DiskManager.scanDirectory(relativeDirectoryName);
   }
 

backend/model/mysql/GalleryManager.ts

@@ -1,5 +1,5 @@
 import {IGalleryManager} from "../interfaces/IGalleryManager";
-import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import * as path from "path";
 import * as fs from "fs";
 import {DirectoryEntity} from "./enitites/DirectoryEntity";
@@ -13,11 +13,15 @@ import {Config} from "../../../common/config/private/Config";
 export class GalleryManager implements IGalleryManager {
 
 
-  public async listDirectory(relativeDirectoryName): Promise<DirectoryDTO> {
+  public async listDirectory(relativeDirectoryName: string,
+                             knownLastModified?: number,
+                             knownLastScanned?: number): Promise<DirectoryDTO | NotModifiedDirectoryDTO> {
     relativeDirectoryName = path.normalize(path.join("." + path.sep, relativeDirectoryName));
     const directoryName = path.basename(relativeDirectoryName);
     const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
     const connection = await MySQLConnection.getConnection();
+    const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
+    const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
     let dir = await connection
       .getRepository(DirectoryEntity)
       .createQueryBuilder("directory")
@@ -29,7 +33,16 @@ export class GalleryManager implements IGalleryManager {
       .leftJoinAndSelect("directory.photos", "photos")
       .getOne();
 
+
     if (dir && dir.scanned == true) {
+      //iF it seems that the content did not changed, do not work on it
+      if (knownLastModified && knownLastScanned) {
+        if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout &&
+          lastModified == knownLastModified &&
+          dir.lastScanned == knownLastScanned) {
+          return Promise.resolve(<NotModifiedDirectoryDTO>{notModified: true});
+        }
+      }
       if (dir.photos) {
         for (let i = 0; i < dir.photos.length; i++) {
           dir.photos[i].directory = dir;
@@ -58,19 +71,17 @@ export class GalleryManager implements IGalleryManager {
         }
       }
 
-      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
-      const lastUpdate = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
 
-      if (dir.lastUpdate != lastUpdate) {
+      if (dir.lastModified != lastModified) {
         return this.indexDirectory(relativeDirectoryName);
       }
 
-      console.log("lazy");
+      if (Date.now() - dir.lastScanned > Config.Server.cachedFolderTimeout) {
-      //on the fly updating
+        //on the fly reindexing
-      this.indexDirectory(relativeDirectoryName).catch((err) => {
+        this.indexDirectory(relativeDirectoryName).catch((err) => {
-        console.error(err);
+          console.error(err);
-      });
+        });
-
+      }
       return dir;
 
 
@@ -103,7 +114,8 @@ export class GalleryManager implements IGalleryManager {
 
         if (!!parentDir) {
           parentDir.scanned = true;
-          parentDir.lastUpdate = scannedDirectory.lastUpdate;
+          parentDir.lastModified = scannedDirectory.lastModified;
+          parentDir.lastScanned = scannedDirectory.lastScanned;
           parentDir = await directoryRepository.persist(parentDir);
         } else {
           (<DirectoryEntity>scannedDirectory).scanned = true;

backend/model/mysql/enitites/DirectoryEntity.ts

@@ -20,7 +20,9 @@ export class DirectoryEntity implements DirectoryDTO {
 
 
   @Column('number')
-  public lastUpdate: number;
+  public lastModified: number;
+  @Column('number')
+  public lastScanned: number;
 
   @Column({type: 'smallint', length: 1})
   public scanned: boolean;

backend/model/threading/DiskMangerWorker.ts

@@ -9,6 +9,7 @@ import {ProjectPath} from "../../ProjectPath";
 import {Config} from "../../../common/config/private/Config";
 
 const LOG_TAG = "[DiskManagerTask]";
+
 export class DiskMangerWorker {
   private static isImage(fullPath: string) {
     const extensions = [
@@ -26,6 +27,60 @@ export class DiskMangerWorker {
     return extensions.indexOf(extension) !== -1;
   }
 
+  public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> {
+    return new Promise<DirectoryDTO>((resolve, reject) => {
+
+      const directoryName = path.basename(relativeDirectoryName);
+      const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
+      const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
+
+      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
+      let directory = <DirectoryDTO>{
+        name: directoryName,
+        path: directoryParent,
+        lastModified: Math.max(stat.ctime.getTime(), stat.mtime.getTime()),
+        lastScanned: Date.now(),
+        directories: [],
+        photos: []
+      };
+      fs.readdir(absoluteDirectoryName, async (err, list) => {
+        if (err) {
+          return reject(err);
+        }
+
+        try {
+          for (let i = 0; i < list.length; i++) {
+            let file = list[i];
+            let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
+            if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
+              const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
+                Config.Server.folderPreviewSize, true
+              );
+              d.lastScanned = 0; //it was not a fully scan
+              directory.directories.push(d);
+            } else if (DiskMangerWorker.isImage(fullFilePath)) {
+              directory.photos.push(<PhotoDTO>{
+                name: file,
+                directory: null,
+                metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath)
+              });
+
+              if (maxPhotos != null && directory.photos.length > maxPhotos) {
+                break;
+              }
+            }
+          }
+
+          return resolve(directory);
+        } catch (err) {
+          return reject({error: err});
+        }
+
+      });
+    });
+
+  }
+
   private static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
     return new Promise<PhotoMetadata>((resolve, reject) => {
       fs.readFile(fullPath, (err, data) => {
@@ -111,55 +166,4 @@ export class DiskMangerWorker {
       }
     );
   }
-
-  public static  scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> {
-    return new Promise<DirectoryDTO>((resolve, reject) => {
-
-      const directoryName = path.basename(relativeDirectoryName);
-      const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
-      const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
-
-      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
-      let directory = <DirectoryDTO>{
-        name: directoryName,
-        path: directoryParent,
-        lastUpdate: Math.max(stat.ctime.getTime(), stat.mtime.getTime()),
-        directories: [],
-        photos: []
-      };
-      fs.readdir(absoluteDirectoryName, async (err, list) => {
-        if (err) {
-          return reject(err);
-        }
-
-        try {
-          for (let i = 0; i < list.length; i++) {
-            let file = list[i];
-            let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
-            if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
-              directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
-                Config.Server.folderPreviewSize, true
-              ));
-            } else if (DiskMangerWorker.isImage(fullFilePath)) {
-              directory.photos.push(<PhotoDTO>{
-                name: file,
-                directory: null,
-                metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath)
-              });
-
-              if (maxPhotos != null && directory.photos.length > maxPhotos) {
-                break;
-              }
-            }
-          }
-
-          return resolve(directory);
-        } catch (err) {
-          return reject({error: err});
-        }
-
-      });
-    });
-
-  }
 }

common/Utils.ts

@@ -26,8 +26,7 @@ export class Utils {
   }
 
 
-  static
+  static concatUrls(...args: Array<string>) {
-  concatUrls(...args: Array<string>) {
     let url = "";
     for (let i = 0; i < args.length; i++) {
       if (args[i] === "" || typeof args[i] === "undefined") continue;
@@ -39,11 +38,14 @@ export class Utils {
     }
     url = url.replace("//", "/");
 
+    if (url.trim() == "") {
+      url = "./";
+    }
+
     return url.substring(0, url.length - 1);
   }
 
-  public static
+  public static updateKeys(targetObject: any, sourceObject: any) {
-  updateKeys(targetObject: any, sourceObject: any) {
     Object.keys(sourceObject).forEach((key) => {
       if (typeof targetObject[key] === "undefined") {
         return;
@@ -56,8 +58,7 @@ export class Utils {
     });
   }
 
-  public static
+  public static setKeys(targetObject: any, sourceObject: any) {
-  setKeys(targetObject: any, sourceObject: any) {
     Object.keys(sourceObject).forEach((key) => {
       if (typeof targetObject[key] === "object") {
         Utils.setKeys(targetObject[key], sourceObject[key]);
@@ -67,8 +68,7 @@ export class Utils {
     });
   }
 
-  public static
+  public static setKeysForced(targetObject: any, sourceObject: any) {
-  setKeysForced(targetObject: any, sourceObject: any) {
     Object.keys(sourceObject).forEach((key) => {
       if (typeof sourceObject[key] === "object") {
         if (typeof targetObject[key] === "undefined") {
@@ -81,8 +81,7 @@ export class Utils {
     });
   }
 
-  public static
+  public static enumToArray(EnumType: any): Array<{
-  enumToArray(EnumType: any): Array<{
     key: number;
     value: string;
   }> {
@@ -100,8 +99,7 @@ export class Utils {
   }
 
 
-  public static
+  public static findClosest(number: number, arr: Array<number>) {
-  findClosest(number: number, arr: Array<number>) {
 
     let curr = arr[0];
     let diff = Math.abs(number - curr);

common/config/private/IPrivateConfig.ts

@@ -1,4 +1,5 @@
 import {ClientConfig} from "../public/ConfigClass";
+
 export enum DatabaseType{
   memory = 0, mysql = 1
 }
@@ -39,6 +40,7 @@ export interface ServerConfig {
   sharing: SharingConfig;
   sessionTimeout: number
   folderPreviewSize: number;
+  cachedFolderTimeout: number;//Do not rescans the folder if seems ok
 }
 export interface IPrivateConfig {
   Server: ServerConfig;

common/config/private/PrivateConfigClass.ts

@@ -31,7 +31,8 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
       updateTimeout: 1000 * 60 * 5
     },
     enableThreading: true,
-    folderPreviewSize: 2
+    folderPreviewSize: 2,
+    cachedFolderTimeout: 1000 * 60 * 60
   };
   private ConfigLoader: any;
 

common/entities/ConentWrapper.ts

@@ -1,11 +1,12 @@
-import {DirectoryDTO} from "./DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "./DirectoryDTO";
 import {SearchResultDTO} from "./SearchResult";
+
 export class ContentWrapper {
 
-  public directory: DirectoryDTO;
+  public directory: DirectoryDTO | NotModifiedDirectoryDTO;
   public searchResult: SearchResultDTO;
 
-  constructor(directory: DirectoryDTO = null, searchResult: SearchResultDTO = null) {
+  constructor(directory: DirectoryDTO | NotModifiedDirectoryDTO = null, searchResult: SearchResultDTO = null) {
     this.directory = directory;
     this.searchResult = searchResult;
   }

common/entities/DirectoryDTO.ts

@@ -4,12 +4,17 @@ export interface DirectoryDTO {
   id: number;
   name: string;
   path: string;
-  lastUpdate: number;
+  lastModified: number;
+  lastScanned: number;
   parent: DirectoryDTO;
   directories: Array<DirectoryDTO>;
   photos: Array<PhotoDTO>;
 }
 
+export interface NotModifiedDirectoryDTO {
+  notModified: boolean;
+}
+
 export module DirectoryDTO {
   export const addReferences = (dir: DirectoryDTO): void => {
     dir.photos.forEach((photo: PhotoDTO) => {

frontend/app/gallery/cache.gallery.service.ts

@@ -12,7 +12,7 @@ export class GalleryCacheService {
     if (Config.Client.enableCache == false) {
       return null;
     }
-    let value = localStorage.getItem(directoryName);
+    let value = localStorage.getItem(Utils.concatUrls(directoryName));
     if (value != null) {
       let directory: DirectoryDTO = JSON.parse(value);
 

frontend/app/gallery/gallery.service.ts

@@ -1,7 +1,7 @@
 import {Injectable} from "@angular/core";
 import {NetworkService} from "../model/network/network.service";
 import {ContentWrapper} from "../../../common/entities/ConentWrapper";
-import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import {SearchTypes} from "../../../common/entities/AutoCompleteItem";
 import {GalleryCacheService} from "./cache.gallery.service";
 import {BehaviorSubject} from "rxjs/BehaviorSubject";
@@ -35,32 +35,37 @@ export class GalleryService {
     this.content.next(content);
     this.lastRequest.directory = directoryName;
 
-    let cw: ContentWrapper = null;
+    const params = {};
     if (Config.Client.Sharing.enabled == true) {
       if (this._shareService.isSharing()) {
-        cw = await this.networkService.getJson<ContentWrapper>("/gallery/content/" + directoryName + "?sk=" + this._shareService.getSharingKey());
+        params['sk'] = this._shareService.getSharingKey();
       }
     }
 
-    if (cw == null) {
+    if (content.directory && content.directory.lastModified && content.directory.lastScanned) {
-      cw = await this.networkService.getJson<ContentWrapper>("/gallery/content/" + directoryName);
+      params['knownLastModified'] = content.directory.lastModified;
+      params['knownLastScanned'] = content.directory.lastScanned;
     }
 
-    if (!cw) {
+
+    const cw = await this.networkService.getJson<ContentWrapper>("/gallery/content/" + directoryName, params);
+
+
+    if (!cw || (<NotModifiedDirectoryDTO>cw.directory).notModified == true) {
       return;
     }
 
-    this.galleryCacheService.setDirectory(cw.directory); //save it before adding references
+    this.galleryCacheService.setDirectory(<DirectoryDTO>cw.directory); //save it before adding references
 
     if (this.lastRequest.directory != directoryName) {
       return;
     }
 
 
-    DirectoryDTO.addReferences(cw.directory);
+    DirectoryDTO.addReferences(<DirectoryDTO>cw.directory);
 
 
-    this.lastDirectory = cw.directory;
+    this.lastDirectory = <DirectoryDTO>cw.directory;
     this.content.next(cw);
 
 
@@ -75,11 +80,7 @@ export class GalleryService {
       return null
     }
 
-    let queryString = "/search/" + text;
+    const cw: ContentWrapper = await this.networkService.getJson<ContentWrapper>("/search/" + text, {type: type});
-    if (type) {
-      queryString += "?type=" + type;
-    }
-    const cw: ContentWrapper = await this.networkService.getJson<ContentWrapper>(queryString);
     this.content.next(cw);
     return cw;
   }

frontend/app/gallery/share/share.gallery.component.ts

@@ -7,6 +7,7 @@ import {SharingDTO} from "../../../../common/entities/SharingDTO";
 import {ModalDirective} from "ngx-bootstrap/modal";
 import {Config} from "../../../../common/config/public/Config";
 import {NotificationService} from "../../model/notification.service";
+import {DirectoryDTO} from "../../../../common/entities/DirectoryDTO";
 
 
 @Component({
@@ -49,7 +50,7 @@ export class GalleryShareComponent implements OnInit, OnDestroy {
       if (!this.enabled) {
         return;
       }
-      this.currentDir = Utils.concatUrls(content.directory.path, content.directory.name);
+      this.currentDir = Utils.concatUrls((<DirectoryDTO>content.directory).path, (<DirectoryDTO>content.directory).name);
     });
     this.passwordProtection = Config.Client.Sharing.passwordProtected;
   }

frontend/app/model/network/network.service.ts

@@ -63,7 +63,19 @@ export class NetworkService {
     return this.callJson("put", url, data);
   }
 
-  public getJson<T>(url: string): Promise<T> {
+  public getJson<T>(url: string, data?: { [key: string]: any }): Promise<T> {
+    if (data) {
+      const keys = Object.getOwnPropertyNames(data);
+      if (keys.length > 0) {
+        url += "?";
+        for (let i = 0; i < keys.length; i++) {
+          url += keys[i] + "=" + data[keys[i]];
+          if (i < keys.length - 1) {
+            url += "&";
+          }
+        }
+      }
+    }
     return this.callJson("get", url);
   }