From 6c1325de499f47599b9f496e34b5cec14a580719 Mon Sep 17 00:00:00 2001 From: Patrik Braun Date: Wed, 19 Jul 2017 20:47:09 +0200 Subject: [PATCH] improving lazy loading --- backend/middlewares/GalleryMWs.ts | 15 ++- .../thumbnail/ThumbnailGeneratorMWs.ts | 7 +- backend/model/interfaces/IGalleryManager.ts | 7 +- backend/model/memory/GalleryManager.ts | 16 ++- backend/model/mysql/GalleryManager.ts | 36 ++++-- .../model/mysql/enitites/DirectoryEntity.ts | 4 +- backend/model/threading/DiskMangerWorker.ts | 106 +++++++++--------- common/Utils.ts | 22 ++-- common/config/private/IPrivateConfig.ts | 2 + common/config/private/PrivateConfigClass.ts | 3 +- common/entities/ConentWrapper.ts | 7 +- common/entities/DirectoryDTO.ts | 7 +- frontend/app/gallery/cache.gallery.service.ts | 2 +- frontend/app/gallery/gallery.service.ts | 29 ++--- .../gallery/share/share.gallery.component.ts | 3 +- frontend/app/model/network/network.service.ts | 14 ++- 16 files changed, 172 insertions(+), 108 deletions(-) diff --git a/backend/middlewares/GalleryMWs.ts b/backend/middlewares/GalleryMWs.ts index 96d3ebd4..19681c9e 100644 --- a/backend/middlewares/GalleryMWs.ts +++ b/backend/middlewares/GalleryMWs.ts @@ -2,7 +2,7 @@ import * as path from "path"; import * as fs from "fs"; import {NextFunction, Request, Response} from "express"; import {ErrorCodes, ErrorDTO} from "../../common/entities/Error"; -import {DirectoryDTO} from "../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../common/entities/DirectoryDTO"; import {ObjectManagerRepository} from "../model/ObjectManagerRepository"; import {SearchTypes} from "../../common/entities/AutoCompleteItem"; import {ContentWrapper} from "../../common/entities/ConentWrapper"; @@ -13,6 +13,7 @@ import {UserDTO} from "../../common/entities/UserDTO"; const LOG_TAG = "[GalleryMWs]"; + export class GalleryMWs { @@ -26,11 +27,15 @@ export class GalleryMWs { try { - const directory = await ObjectManagerRepository.getInstance().GalleryManager.listDirectory(directoryName); + const directory = await ObjectManagerRepository.getInstance().GalleryManager.listDirectory(directoryName, req.query.knownLastModified, req.query.knownLastScanned); + if ((directory).notModified == true) { + req.resultPipe = new ContentWrapper(directory, null); + return next(); + } if (req.session.user.permissions && req.session.user.permissions.length > 0 && req.session.user.permissions[0] != "/") { - directory.directories = directory.directories.filter(d => + (directory).directories = (directory).directories.filter(d => UserDTO.isDirectoryAvailable(d, req.session.user.permissions)); } req.resultPipe = new ContentWrapper(directory, null); @@ -47,6 +52,9 @@ export class GalleryMWs { return next(); let cw: ContentWrapper = req.resultPipe; + if ((cw.directory).notModified == true) { + return next(); + } let removeDirs = (dir) => { dir.photos.forEach((photo: PhotoDTO) => { photo.directory = null; @@ -84,7 +92,6 @@ export class GalleryMWs { } - req.resultPipe = fullImagePath; return next(); } diff --git a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 4de320ea..1abeffbe 100644 --- a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -6,7 +6,7 @@ import * as os from "os"; import {NextFunction, Request, Response} from "express"; import {ErrorCodes, ErrorDTO} from "../../../common/entities/Error"; import {ContentWrapper} from "../../../common/entities/ConentWrapper"; -import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO"; import {ProjectPath} from "../../ProjectPath"; import {PhotoDTO} from "../../../common/entities/PhotoDTO"; import {Config} from "../../../common/config/private/Config"; @@ -85,8 +85,11 @@ export class ThumbnailGeneratorMWs { return next(); let cw: ContentWrapper = req.resultPipe; + if ((cw.directory).notModified == true) { + return next(); + } if (cw.directory) { - ThumbnailGeneratorMWs.addThInfoTODir(cw.directory); + ThumbnailGeneratorMWs.addThInfoTODir(cw.directory); } if (cw.searchResult) { ThumbnailGeneratorMWs.addThInfoToPhotos(cw.searchResult.photos); diff --git a/backend/model/interfaces/IGalleryManager.ts b/backend/model/interfaces/IGalleryManager.ts index cfaf5c26..b5100fc3 100644 --- a/backend/model/interfaces/IGalleryManager.ts +++ b/backend/model/interfaces/IGalleryManager.ts @@ -1,4 +1,7 @@ -import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO"; + export interface IGalleryManager { - listDirectory(relativeDirectoryName: string): Promise; + listDirectory(relativeDirectoryName: string, + knownLastModified?: number, + knownLastScanned?: number): Promise; } diff --git a/backend/model/memory/GalleryManager.ts b/backend/model/memory/GalleryManager.ts index 100fa77c..6a3c499b 100644 --- a/backend/model/memory/GalleryManager.ts +++ b/backend/model/memory/GalleryManager.ts @@ -1,10 +1,22 @@ -import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO"; import {IGalleryManager} from "../interfaces/IGalleryManager"; +import * as path from "path"; +import * as fs from "fs"; import {DiskManager} from "../DiskManger"; +import {ProjectPath} from "../../ProjectPath"; +import {Config} from "../../../common/config/private/Config"; export class GalleryManager implements IGalleryManager { - public listDirectory(relativeDirectoryName: string): Promise { + public listDirectory(relativeDirectoryName: string, knownLastModified?: number, knownLastScanned?: number): Promise { + //If it seems that the content did not changed, do not work on it + if (knownLastModified && knownLastScanned) { + const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); + const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime()); + if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && lastModified == knownLastModified) { + return Promise.resolve({notModified: true}); + } + } return DiskManager.scanDirectory(relativeDirectoryName); } diff --git a/backend/model/mysql/GalleryManager.ts b/backend/model/mysql/GalleryManager.ts index b992130c..7f3032a6 100644 --- a/backend/model/mysql/GalleryManager.ts +++ b/backend/model/mysql/GalleryManager.ts @@ -1,5 +1,5 @@ import {IGalleryManager} from "../interfaces/IGalleryManager"; -import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO"; import * as path from "path"; import * as fs from "fs"; import {DirectoryEntity} from "./enitites/DirectoryEntity"; @@ -13,11 +13,15 @@ import {Config} from "../../../common/config/private/Config"; export class GalleryManager implements IGalleryManager { - public async listDirectory(relativeDirectoryName): Promise { + public async listDirectory(relativeDirectoryName: string, + knownLastModified?: number, + knownLastScanned?: number): Promise { relativeDirectoryName = path.normalize(path.join("." + path.sep, relativeDirectoryName)); const directoryName = path.basename(relativeDirectoryName); const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); const connection = await MySQLConnection.getConnection(); + const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); + const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime()); let dir = await connection .getRepository(DirectoryEntity) .createQueryBuilder("directory") @@ -29,7 +33,16 @@ export class GalleryManager implements IGalleryManager { .leftJoinAndSelect("directory.photos", "photos") .getOne(); + if (dir && dir.scanned == true) { + //iF it seems that the content did not changed, do not work on it + if (knownLastModified && knownLastScanned) { + if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && + lastModified == knownLastModified && + dir.lastScanned == knownLastScanned) { + return Promise.resolve({notModified: true}); + } + } if (dir.photos) { for (let i = 0; i < dir.photos.length; i++) { dir.photos[i].directory = dir; @@ -58,19 +71,17 @@ export class GalleryManager implements IGalleryManager { } } - const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); - const lastUpdate = Math.max(stat.ctime.getTime(), stat.mtime.getTime()); - if (dir.lastUpdate != lastUpdate) { + if (dir.lastModified != lastModified) { return this.indexDirectory(relativeDirectoryName); } - console.log("lazy"); - //on the fly updating - this.indexDirectory(relativeDirectoryName).catch((err) => { - console.error(err); - }); - + if (Date.now() - dir.lastScanned > Config.Server.cachedFolderTimeout) { + //on the fly reindexing + this.indexDirectory(relativeDirectoryName).catch((err) => { + console.error(err); + }); + } return dir; @@ -103,7 +114,8 @@ export class GalleryManager implements IGalleryManager { if (!!parentDir) { parentDir.scanned = true; - parentDir.lastUpdate = scannedDirectory.lastUpdate; + parentDir.lastModified = scannedDirectory.lastModified; + parentDir.lastScanned = scannedDirectory.lastScanned; parentDir = await directoryRepository.persist(parentDir); } else { (scannedDirectory).scanned = true; diff --git a/backend/model/mysql/enitites/DirectoryEntity.ts b/backend/model/mysql/enitites/DirectoryEntity.ts index 85967e3d..b95d1562 100644 --- a/backend/model/mysql/enitites/DirectoryEntity.ts +++ b/backend/model/mysql/enitites/DirectoryEntity.ts @@ -20,7 +20,9 @@ export class DirectoryEntity implements DirectoryDTO { @Column('number') - public lastUpdate: number; + public lastModified: number; + @Column('number') + public lastScanned: number; @Column({type: 'smallint', length: 1}) public scanned: boolean; diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 12d550ec..b31037a5 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -9,6 +9,7 @@ import {ProjectPath} from "../../ProjectPath"; import {Config} from "../../../common/config/private/Config"; const LOG_TAG = "[DiskManagerTask]"; + export class DiskMangerWorker { private static isImage(fullPath: string) { const extensions = [ @@ -26,6 +27,60 @@ export class DiskMangerWorker { return extensions.indexOf(extension) !== -1; } + public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise { + return new Promise((resolve, reject) => { + + const directoryName = path.basename(relativeDirectoryName); + const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); + const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); + + const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); + let directory = { + name: directoryName, + path: directoryParent, + lastModified: Math.max(stat.ctime.getTime(), stat.mtime.getTime()), + lastScanned: Date.now(), + directories: [], + photos: [] + }; + fs.readdir(absoluteDirectoryName, async (err, list) => { + if (err) { + return reject(err); + } + + try { + for (let i = 0; i < list.length; i++) { + let file = list[i]; + let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); + if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { + const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), + Config.Server.folderPreviewSize, true + ); + d.lastScanned = 0; //it was not a fully scan + directory.directories.push(d); + } else if (DiskMangerWorker.isImage(fullFilePath)) { + directory.photos.push({ + name: file, + directory: null, + metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath) + }); + + if (maxPhotos != null && directory.photos.length > maxPhotos) { + break; + } + } + } + + return resolve(directory); + } catch (err) { + return reject({error: err}); + } + + }); + }); + + } + private static loadPhotoMetadata(fullPath: string): Promise { return new Promise((resolve, reject) => { fs.readFile(fullPath, (err, data) => { @@ -111,55 +166,4 @@ export class DiskMangerWorker { } ); } - - public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise { - return new Promise((resolve, reject) => { - - const directoryName = path.basename(relativeDirectoryName); - const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); - const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); - - const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); - let directory = { - name: directoryName, - path: directoryParent, - lastUpdate: Math.max(stat.ctime.getTime(), stat.mtime.getTime()), - directories: [], - photos: [] - }; - fs.readdir(absoluteDirectoryName, async (err, list) => { - if (err) { - return reject(err); - } - - try { - for (let i = 0; i < list.length; i++) { - let file = list[i]; - let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); - if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { - directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), - Config.Server.folderPreviewSize, true - )); - } else if (DiskMangerWorker.isImage(fullFilePath)) { - directory.photos.push({ - name: file, - directory: null, - metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath) - }); - - if (maxPhotos != null && directory.photos.length > maxPhotos) { - break; - } - } - } - - return resolve(directory); - } catch (err) { - return reject({error: err}); - } - - }); - }); - - } } diff --git a/common/Utils.ts b/common/Utils.ts index 36ee8ae3..25da7ca6 100644 --- a/common/Utils.ts +++ b/common/Utils.ts @@ -26,8 +26,7 @@ export class Utils { } - static - concatUrls(...args: Array) { + static concatUrls(...args: Array) { let url = ""; for (let i = 0; i < args.length; i++) { if (args[i] === "" || typeof args[i] === "undefined") continue; @@ -39,11 +38,14 @@ export class Utils { } url = url.replace("//", "/"); + if (url.trim() == "") { + url = "./"; + } + return url.substring(0, url.length - 1); } - public static - updateKeys(targetObject: any, sourceObject: any) { + public static updateKeys(targetObject: any, sourceObject: any) { Object.keys(sourceObject).forEach((key) => { if (typeof targetObject[key] === "undefined") { return; @@ -56,8 +58,7 @@ export class Utils { }); } - public static - setKeys(targetObject: any, sourceObject: any) { + public static setKeys(targetObject: any, sourceObject: any) { Object.keys(sourceObject).forEach((key) => { if (typeof targetObject[key] === "object") { Utils.setKeys(targetObject[key], sourceObject[key]); @@ -67,8 +68,7 @@ export class Utils { }); } - public static - setKeysForced(targetObject: any, sourceObject: any) { + public static setKeysForced(targetObject: any, sourceObject: any) { Object.keys(sourceObject).forEach((key) => { if (typeof sourceObject[key] === "object") { if (typeof targetObject[key] === "undefined") { @@ -81,8 +81,7 @@ export class Utils { }); } - public static - enumToArray(EnumType: any): Array<{ + public static enumToArray(EnumType: any): Array<{ key: number; value: string; }> { @@ -100,8 +99,7 @@ export class Utils { } - public static - findClosest(number: number, arr: Array) { + public static findClosest(number: number, arr: Array) { let curr = arr[0]; let diff = Math.abs(number - curr); diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index 202124b9..ff198ddf 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -1,4 +1,5 @@ import {ClientConfig} from "../public/ConfigClass"; + export enum DatabaseType{ memory = 0, mysql = 1 } @@ -39,6 +40,7 @@ export interface ServerConfig { sharing: SharingConfig; sessionTimeout: number folderPreviewSize: number; + cachedFolderTimeout: number;//Do not rescans the folder if seems ok } export interface IPrivateConfig { Server: ServerConfig; diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index c44df739..c0dd7d34 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -31,7 +31,8 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon updateTimeout: 1000 * 60 * 5 }, enableThreading: true, - folderPreviewSize: 2 + folderPreviewSize: 2, + cachedFolderTimeout: 1000 * 60 * 60 }; private ConfigLoader: any; diff --git a/common/entities/ConentWrapper.ts b/common/entities/ConentWrapper.ts index 3c598837..dc9984af 100644 --- a/common/entities/ConentWrapper.ts +++ b/common/entities/ConentWrapper.ts @@ -1,11 +1,12 @@ -import {DirectoryDTO} from "./DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "./DirectoryDTO"; import {SearchResultDTO} from "./SearchResult"; + export class ContentWrapper { - public directory: DirectoryDTO; + public directory: DirectoryDTO | NotModifiedDirectoryDTO; public searchResult: SearchResultDTO; - constructor(directory: DirectoryDTO = null, searchResult: SearchResultDTO = null) { + constructor(directory: DirectoryDTO | NotModifiedDirectoryDTO = null, searchResult: SearchResultDTO = null) { this.directory = directory; this.searchResult = searchResult; } diff --git a/common/entities/DirectoryDTO.ts b/common/entities/DirectoryDTO.ts index fc58f095..7c2ef132 100644 --- a/common/entities/DirectoryDTO.ts +++ b/common/entities/DirectoryDTO.ts @@ -4,12 +4,17 @@ export interface DirectoryDTO { id: number; name: string; path: string; - lastUpdate: number; + lastModified: number; + lastScanned: number; parent: DirectoryDTO; directories: Array; photos: Array; } +export interface NotModifiedDirectoryDTO { + notModified: boolean; +} + export module DirectoryDTO { export const addReferences = (dir: DirectoryDTO): void => { dir.photos.forEach((photo: PhotoDTO) => { diff --git a/frontend/app/gallery/cache.gallery.service.ts b/frontend/app/gallery/cache.gallery.service.ts index fb4705fd..d13bd8ef 100644 --- a/frontend/app/gallery/cache.gallery.service.ts +++ b/frontend/app/gallery/cache.gallery.service.ts @@ -12,7 +12,7 @@ export class GalleryCacheService { if (Config.Client.enableCache == false) { return null; } - let value = localStorage.getItem(directoryName); + let value = localStorage.getItem(Utils.concatUrls(directoryName)); if (value != null) { let directory: DirectoryDTO = JSON.parse(value); diff --git a/frontend/app/gallery/gallery.service.ts b/frontend/app/gallery/gallery.service.ts index 8ffa2bae..7e6b5621 100644 --- a/frontend/app/gallery/gallery.service.ts +++ b/frontend/app/gallery/gallery.service.ts @@ -1,7 +1,7 @@ import {Injectable} from "@angular/core"; import {NetworkService} from "../model/network/network.service"; import {ContentWrapper} from "../../../common/entities/ConentWrapper"; -import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; +import {DirectoryDTO, NotModifiedDirectoryDTO} from "../../../common/entities/DirectoryDTO"; import {SearchTypes} from "../../../common/entities/AutoCompleteItem"; import {GalleryCacheService} from "./cache.gallery.service"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; @@ -35,32 +35,37 @@ export class GalleryService { this.content.next(content); this.lastRequest.directory = directoryName; - let cw: ContentWrapper = null; + const params = {}; if (Config.Client.Sharing.enabled == true) { if (this._shareService.isSharing()) { - cw = await this.networkService.getJson("/gallery/content/" + directoryName + "?sk=" + this._shareService.getSharingKey()); + params['sk'] = this._shareService.getSharingKey(); } } - if (cw == null) { - cw = await this.networkService.getJson("/gallery/content/" + directoryName); + if (content.directory && content.directory.lastModified && content.directory.lastScanned) { + params['knownLastModified'] = content.directory.lastModified; + params['knownLastScanned'] = content.directory.lastScanned; } - if (!cw) { + + const cw = await this.networkService.getJson("/gallery/content/" + directoryName, params); + + + if (!cw || (cw.directory).notModified == true) { return; } - this.galleryCacheService.setDirectory(cw.directory); //save it before adding references + this.galleryCacheService.setDirectory(cw.directory); //save it before adding references if (this.lastRequest.directory != directoryName) { return; } - DirectoryDTO.addReferences(cw.directory); + DirectoryDTO.addReferences(cw.directory); - this.lastDirectory = cw.directory; + this.lastDirectory = cw.directory; this.content.next(cw); @@ -75,11 +80,7 @@ export class GalleryService { return null } - let queryString = "/search/" + text; - if (type) { - queryString += "?type=" + type; - } - const cw: ContentWrapper = await this.networkService.getJson(queryString); + const cw: ContentWrapper = await this.networkService.getJson("/search/" + text, {type: type}); this.content.next(cw); return cw; } diff --git a/frontend/app/gallery/share/share.gallery.component.ts b/frontend/app/gallery/share/share.gallery.component.ts index 0c9c331d..9343a2d6 100644 --- a/frontend/app/gallery/share/share.gallery.component.ts +++ b/frontend/app/gallery/share/share.gallery.component.ts @@ -7,6 +7,7 @@ import {SharingDTO} from "../../../../common/entities/SharingDTO"; import {ModalDirective} from "ngx-bootstrap/modal"; import {Config} from "../../../../common/config/public/Config"; import {NotificationService} from "../../model/notification.service"; +import {DirectoryDTO} from "../../../../common/entities/DirectoryDTO"; @Component({ @@ -49,7 +50,7 @@ export class GalleryShareComponent implements OnInit, OnDestroy { if (!this.enabled) { return; } - this.currentDir = Utils.concatUrls(content.directory.path, content.directory.name); + this.currentDir = Utils.concatUrls((content.directory).path, (content.directory).name); }); this.passwordProtection = Config.Client.Sharing.passwordProtected; } diff --git a/frontend/app/model/network/network.service.ts b/frontend/app/model/network/network.service.ts index 3a0e7491..e00ff055 100644 --- a/frontend/app/model/network/network.service.ts +++ b/frontend/app/model/network/network.service.ts @@ -63,7 +63,19 @@ export class NetworkService { return this.callJson("put", url, data); } - public getJson(url: string): Promise { + public getJson(url: string, data?: { [key: string]: any }): Promise { + if (data) { + const keys = Object.getOwnPropertyNames(data); + if (keys.length > 0) { + url += "?"; + for (let i = 0; i < keys.length; i++) { + url += keys[i] + "=" + data[keys[i]]; + if (i < keys.length - 1) { + url += "&"; + } + } + } + } return this.callJson("get", url); }