From 849ebbec9ef7118042d23158da54a7f86a201b6d Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Mon, 27 Jun 2022 23:46:17 +0200 Subject: [PATCH] Smaller performance tweeks to thumbnail reading #437 --- .../thumbnail/ThumbnailGeneratorMWs.ts | 43 ++++++++-------- src/common/config/public/ClientConfig.ts | 51 +++++++++++-------- src/common/entities/ConentWrapper.ts | 4 ++ test/common/unit/ContentWrapper.spec.ts | 3 ++ 4 files changed, 58 insertions(+), 43 deletions(-) diff --git a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 1c284423..0ae2eb69 100644 --- a/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -1,23 +1,23 @@ import * as path from 'path'; import * as fs from 'fs'; -import { NextFunction, Request, Response } from 'express'; -import { ErrorCodes, ErrorDTO } from '../../../common/entities/Error'; -import { ContentWrapper } from '../../../common/entities/ConentWrapper'; +import {NextFunction, Request, Response} from 'express'; +import {ErrorCodes, ErrorDTO} from '../../../common/entities/Error'; +import {ContentWrapper} from '../../../common/entities/ConentWrapper'; import { ParentDirectoryDTO, SubDirectoryDTO, } from '../../../common/entities/DirectoryDTO'; -import { ProjectPath } from '../../ProjectPath'; -import { Config } from '../../../common/config/private/Config'; -import { ThumbnailSourceType } from '../../model/threading/PhotoWorker'; -import { MediaDTO } from '../../../common/entities/MediaDTO'; -import { PhotoProcessing } from '../../model/fileprocessing/PhotoProcessing'; -import { PersonWithSampleRegion } from '../../../common/entities/PersonDTO'; -import { ServerTime } from '../ServerTimingMWs'; +import {ProjectPath} from '../../ProjectPath'; +import {Config} from '../../../common/config/private/Config'; +import {ThumbnailSourceType} from '../../model/threading/PhotoWorker'; +import {MediaDTO} from '../../../common/entities/MediaDTO'; +import {PhotoProcessing} from '../../model/fileprocessing/PhotoProcessing'; +import {PersonWithSampleRegion} from '../../../common/entities/PersonDTO'; +import {ServerTime} from '../ServerTimingMWs'; export class ThumbnailGeneratorMWs { - private static ThumbnailMap: { [key: number]: number } = - Config.Client.Media.Thumbnail.generateThumbnailMap(); + private static ThumbnailMapEntries = + Config.Client.Media.Thumbnail.generateThumbnailMapEntries(); @ServerTime('2.th', 'Thumbnail decoration') public static async addThumbnailInformation( @@ -34,6 +34,10 @@ export class ThumbnailGeneratorMWs { if (cw.notModified === true) { return next(); } + + // regenerate in case the list change since startup + ThumbnailGeneratorMWs.ThumbnailMapEntries = + Config.Client.Media.Thumbnail.generateThumbnailMapEntries(); if (cw.directory) { ThumbnailGeneratorMWs.addThInfoTODir(cw.directory); } @@ -209,8 +213,6 @@ export class ThumbnailGeneratorMWs { private static addThInfoTODir( directory: ParentDirectoryDTO | SubDirectoryDTO ): void { - ThumbnailGeneratorMWs.ThumbnailMap = - Config.Client.Media.Thumbnail.generateThumbnailMap(); if (typeof directory.media !== 'undefined') { ThumbnailGeneratorMWs.addThInfoToPhotos(directory.media); } @@ -220,8 +222,8 @@ export class ThumbnailGeneratorMWs { } private static addThInfoToPhotos(photos: MediaDTO[]): void { - for (const item of photos) { - this.addThInfoToAPhoto(item); + for (let i = 0; i < photos.length; ++i) { + this.addThInfoToAPhoto(photos[i]); } } @@ -232,19 +234,18 @@ export class ThumbnailGeneratorMWs { photo.directory.name, photo.name ); - for (const _s of Object.keys(ThumbnailGeneratorMWs.ThumbnailMap)) { - const size = parseInt(_s) + for (let i = 0; i < ThumbnailGeneratorMWs.ThumbnailMapEntries.length; ++i) { + const entry = ThumbnailGeneratorMWs.ThumbnailMapEntries[i]; const thPath = PhotoProcessing.generateConvertedPath( fullMediaPath, - size + entry.size ); if (fs.existsSync(thPath) !== true) { if (typeof photo.missingThumbnails === 'undefined') { photo.missingThumbnails = 0; } // this is a bitwise operation - photo.missingThumbnails += - ThumbnailGeneratorMWs.ThumbnailMap[size]; + photo.missingThumbnails += entry.bit; } } } diff --git a/src/common/config/public/ClientConfig.ts b/src/common/config/public/ClientConfig.ts index bbedb1c7..5d327cdd 100644 --- a/src/common/config/public/ClientConfig.ts +++ b/src/common/config/public/ClientConfig.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-inferrable-types */ import 'reflect-metadata'; -import { SortingMethods } from '../../entities/SortingMethods'; -import { UserRoles } from '../../entities/UserDTO'; -import { ConfigProperty, SubConfigClass } from 'typeconfig/common'; -import { IPrivateConfig } from '../private/PrivateConfig'; +import {SortingMethods} from '../../entities/SortingMethods'; +import {UserRoles} from '../../entities/UserDTO'; +import {ConfigProperty, SubConfigClass} from 'typeconfig/common'; +import {IPrivateConfig} from '../private/PrivateConfig'; export enum MapProviders { OpenStreetMap = 1, @@ -15,11 +15,11 @@ export enum MapProviders { export class AutoCompleteConfig { @ConfigProperty() enabled: boolean = true; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) targetItemsPerCategory: number = 5; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) maxItems: number = 30; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) cacheTimeout: number = 1000 * 60 * 60; } @@ -27,11 +27,11 @@ export class AutoCompleteConfig { export class ClientSearchConfig { @ConfigProperty() enabled: boolean = true; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) searchCacheTimeout: number = 1000 * 60 * 60; @ConfigProperty() AutoComplete: AutoCompleteConfig = new AutoCompleteConfig(); - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) maxMediaResult: number = 10000; @ConfigProperty({ description: 'Search returns also with directories, not just media', @@ -42,7 +42,7 @@ export class ClientSearchConfig { 'Search also returns with metafiles from directories that contain a media file of the matched search result', }) listMetafiles: boolean = true; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) maxDirectoryResult: number = 200; } @@ -62,7 +62,7 @@ export class ClientSharingConfig { @SubConfigClass() export class ClientRandomPhotoConfig { - @ConfigProperty({ description: 'Enables random link generation.' }) + @ConfigProperty({description: 'Enables random link generation.'}) enabled: boolean = true; } @@ -92,23 +92,23 @@ export class ClientMapConfig { maxPreviewMarkers: number = 50; @ConfigProperty() useImageMarkers: boolean = true; - @ConfigProperty({ type: MapProviders }) + @ConfigProperty({type: MapProviders}) mapProvider: MapProviders = MapProviders.OpenStreetMap; @ConfigProperty() mapboxAccessToken: string = ''; - @ConfigProperty({ arrayType: MapLayers }) + @ConfigProperty({arrayType: MapLayers}) customLayers: MapLayers[] = [new MapLayers()]; } @SubConfigClass() export class ClientThumbnailConfig { - @ConfigProperty({ type: 'unsignedInt', max: 100 }) + @ConfigProperty({type: 'unsignedInt', max: 100}) iconSize: number = 45; - @ConfigProperty({ type: 'unsignedInt' }) + @ConfigProperty({type: 'unsignedInt'}) personThumbnailSize: number = 200; - @ConfigProperty({ arrayType: 'unsignedInt' }) + @ConfigProperty({arrayType: 'unsignedInt'}) thumbnailSizes: number[] = [240, 480]; - @ConfigProperty({ volatile: true }) + @ConfigProperty({volatile: true}) concurrentThumbnailGenerations: number = 1; /** @@ -121,6 +121,13 @@ export class ClientThumbnailConfig { }); return m; } + + /** + * Generates a map for bitwise operation from icon and normal thumbnails + */ + generateThumbnailMapEntries(): { size: number, bit: number }[] { + return Object.entries(this.generateThumbnailMap()).map(v => ({size: parseInt(v[0]), bit: v[1]})); + } } @SubConfigClass() @@ -137,7 +144,7 @@ export class ClientOtherConfig { enableCache: boolean = true; @ConfigProperty() enableOnScrollRendering: boolean = true; - @ConfigProperty({ type: SortingMethods }) + @ConfigProperty({type: SortingMethods}) defaultPhotoSortingMethod: SortingMethods = SortingMethods.ascDate; @ConfigProperty({ description: @@ -227,9 +234,9 @@ export class ClientFacesConfig { enabled: boolean = true; @ConfigProperty() keywordsToPersons: boolean = true; - @ConfigProperty({ type: UserRoles }) + @ConfigProperty({type: UserRoles}) writeAccessMinRole: UserRoles = UserRoles.Admin; - @ConfigProperty({ type: UserRoles }) + @ConfigProperty({type: UserRoles}) readAccessMinRole: UserRoles = UserRoles.User; } @@ -255,9 +262,9 @@ export class ClientConfig { Other: ClientOtherConfig = new ClientOtherConfig(); @ConfigProperty() authenticationRequired: boolean = true; - @ConfigProperty({ type: UserRoles }) + @ConfigProperty({type: UserRoles}) unAuthenticatedUserRole: UserRoles = UserRoles.Admin; - @ConfigProperty({ arrayType: 'string', volatile: true }) + @ConfigProperty({arrayType: 'string', volatile: true}) languages: string[] | undefined; @ConfigProperty() Media: ClientMediaConfig = new ClientMediaConfig(); diff --git a/src/common/entities/ConentWrapper.ts b/src/common/entities/ConentWrapper.ts index 3f7d128d..258802ec 100644 --- a/src/common/entities/ConentWrapper.ts +++ b/src/common/entities/ConentWrapper.ts @@ -164,6 +164,10 @@ export class ContentWrapper { } } + if (m.missingThumbnails === 0) { + delete m.missingThumbnails; + } + if (MediaDTOUtils.isPhoto(m)) { delete (m as VideoDTO).metadata.bitRate; delete (m as VideoDTO).metadata.duration; diff --git a/test/common/unit/ContentWrapper.spec.ts b/test/common/unit/ContentWrapper.spec.ts index b95f915c..1d0f00bf 100644 --- a/test/common/unit/ContentWrapper.spec.ts +++ b/test/common/unit/ContentWrapper.spec.ts @@ -34,6 +34,9 @@ describe('ContentWrapper', () => { delete (m as PhotoDTO).metadata.faces; delete (m as PhotoDTO).metadata.positionData; } + if (m.missingThumbnails === 0) { + delete m.missingThumbnails; + } } for (let i = 0; i < content.metaFile.length; ++i) { delete content.metaFile[i].id;