diff --git a/.dockerignore b/.dockerignore index aff14128..6acb6b87 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ .idea/ .git/ node_modules/ -benchmark/ release/ demo/ dist/ diff --git a/benchmark/BMConfig.ts b/benchmark/BMConfig.ts index 26a19f71..07b00965 100644 --- a/benchmark/BMConfig.ts +++ b/benchmark/BMConfig.ts @@ -11,6 +11,7 @@ import {ConfigProperty} from 'typeconfig/common'; enumsAsString: true, softReadonly: true, cli: { + prefix: 'bm-config', enable: { configPath: true, attachState: true, @@ -31,6 +32,8 @@ export class PrivateConfigClass { path: string = 'demo/images'; @ConfigProperty({description: 'Describe your system setup'}) system: string = ''; + @ConfigProperty({description: 'Number of times to run the benchmark'}) + RUNS: number = 50; } diff --git a/benchmark/Benchmark.ts b/benchmark/Benchmark.ts new file mode 100644 index 00000000..2624fd1b --- /dev/null +++ b/benchmark/Benchmark.ts @@ -0,0 +1,125 @@ +import {BenchmarkResult} from './BenchmarkRunner'; +import {ContentWrapper} from '../src/common/entities/ConentWrapper'; + +export interface BenchmarkStep { + name: string; + fn: ((input?: any) => Promise); +} + +export class Benchmark { + + + steps: BenchmarkStep[] = []; + name: string; + inputCB: () => any; + beforeEach: () => Promise; + afterEach: () => Promise; + + + constructor(name: string, + inputCB?: () => any, + beforeEach?: () => Promise, + afterEach?: () => Promise) { + this.name = name; + this.inputCB = inputCB; + this.beforeEach = beforeEach; + this.afterEach = afterEach; + } + + async run(RUNS: number): Promise { + console.log('Running benchmark: ' + this.name); + const scanned = await this.scanSteps(); + const start = process.hrtime(); + let skip = 0; + const stepTimer = new Array(this.steps.length).fill(0); + for (let i = 0; i < RUNS; i++) { + if (this.beforeEach) { + const startSkip = process.hrtime(); + await this.beforeEach(); + const endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + } + await this.runOneRound(stepTimer); + if (this.afterEach) { + const startSkip = process.hrtime(); + await this.afterEach(); + const endSkip = process.hrtime(startSkip); + skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); + } + } + const end = process.hrtime(start); + const duration = (end[0] * 1000 + end[1] / 1000000 - skip) / RUNS; + + + const ret = this.outputToBMResult(this.name, scanned[scanned.length - 1]); + ret.duration = duration; + ret.subBenchmarks = scanned.map((o, i) => { + const stepBm = this.outputToBMResult(this.steps[i].name, o); + stepBm.duration = stepTimer[i] / RUNS; + return stepBm; + } + ); + + return ret; + } + + outputToBMResult(name: string, output: any[] | ContentWrapper): BenchmarkResult { + if (output) { + if (Array.isArray(output)) { + return { + name: name, + duration: null, + items: output.length, + }; + } + + if (output.directory || output.searchResult) { + return { + name: name, + duration: null, + contentWrapper: output + }; + } + + } + return { + name: name, + duration: null + }; + } + + async scanSteps(): Promise { + let pipe = this.inputCB ? this.inputCB() : null; + const stepOutput = new Array(this.steps.length); + + for (let j = 0; j < this.steps.length; ++j) { + if (this.beforeEach) { + await this.beforeEach(); + } + for (let i = 0; i <= j; ++i) { + pipe = await this.steps[i].fn(pipe); + } + stepOutput[j] = pipe; + if (this.afterEach) { + await this.afterEach(); + } + } + return stepOutput; + } + + async runOneRound(stepTimer: number[]): Promise { + let pipe = this.inputCB ? this.inputCB() : null; + for (let i = 0; i < this.steps.length; ++i) { + const start = process.hrtime(); + pipe = await this.steps[i].fn(pipe); + const end = process.hrtime(start); + stepTimer[i] += (end[0] * 1000 + end[1] / 1000000); + } + return stepTimer; + } + + addAStep(step: BenchmarkStep) { + this.steps.push(step); + } + +} diff --git a/benchmark/BenchmarkRunner.ts b/benchmark/BenchmarkRunner.ts new file mode 100644 index 00000000..75d9aaa5 --- /dev/null +++ b/benchmark/BenchmarkRunner.ts @@ -0,0 +1,215 @@ +import {Config} from '../src/common/config/private/Config'; +import {ObjectManagers} from '../src/backend/model/ObjectManagers'; +import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'; +import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager'; +import {SearchManager} from '../src/backend/model/database/sql/SearchManager'; +import * as util from 'util'; +import * as rimraf from 'rimraf'; +import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; +import {Utils} from '../src/common/Utils'; +import {DirectoryDTO} from '../src/common/entities/DirectoryDTO'; +import {ServerConfig} from '../src/common/config/private/PrivateConfig'; +import {ProjectPath} from '../src/backend/ProjectPath'; +import {PersonMWs} from '../src/backend/middlewares/PersonMWs'; +import {ThumbnailGeneratorMWs} from '../src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs'; +import {Benchmark} from './Benchmark'; +import {IndexingJob} from '../src/backend/model/jobs/jobs/IndexingJob'; +import {IJob} from '../src/backend/model/jobs/jobs/IJob'; +import {JobProgressStates} from '../src/common/entities/job/JobProgressDTO'; +import {JobProgress} from '../src/backend/model/jobs/jobs/JobProgress'; +import {GalleryMWs} from '../src/backend/middlewares/GalleryMWs'; +import {UserDTO, UserRoles} from '../src/common/entities/UserDTO'; +import {ContentWrapper} from '../src/common/entities/ConentWrapper'; + +const rimrafPR = util.promisify(rimraf); + +export interface BenchmarkResult { + name: string; + duration: number; + contentWrapper?: ContentWrapper; + items?: number; + subBenchmarks?: BenchmarkResult[]; +} + +export class BMIndexingManager extends IndexingManager { + + public async saveToDB(scannedDirectory: DirectoryDTO): Promise { + return super.saveToDB(scannedDirectory); + } +} + +export class BenchmarkRunner { + + + constructor(public RUNS: number) { + + } + + async bmSaveDirectory(): Promise { + await this.resetDB(); + const dir = await DiskMangerWorker.scanDirectory('./'); + const bm = new Benchmark('Saving directory to DB', null, () => this.resetDB()); + bm.addAStep({ + name: 'Saving directory to DB', + fn: () => { + const im = new BMIndexingManager(); + return im.saveToDB(dir); + } + }); + return await bm.run(this.RUNS); + } + + async bmScanDirectory(): Promise { + const bm = new Benchmark('Scanning directory'); + bm.addAStep({ + name: 'Scanning directory', + fn: async () => new ContentWrapper(await DiskMangerWorker.scanDirectory('./')) + }); + return await bm.run(this.RUNS); + } + + async bmListDirectory(): Promise { + await this.setupDB(); + Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; + const bm = new Benchmark('List directory', + null, + async () => { + await ObjectManagers.reset(); + await ObjectManagers.InitSQLManagers(); + }); + bm.addAStep({ + name: 'List directory', + fn: (input) => this.nextToPromise(GalleryMWs.listDirectory, input, {directory: '/'}) + }); + bm.addAStep({ + name: 'Add Thumbnail information', + fn: (input) => this.nextToPromise(ThumbnailGeneratorMWs.addThumbnailInformation, input) + }); + bm.addAStep({ + name: 'Clean Up Gallery Result', + fn: (input) => this.nextToPromise(GalleryMWs.cleanUpGalleryResults, input) + }); + return await bm.run(this.RUNS); + } + + async bmListPersons(): Promise { + await this.setupDB(); + Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; + const bm = new Benchmark('Listing Faces', null, async () => { + await ObjectManagers.reset(); + await ObjectManagers.InitSQLManagers(); + }); + bm.addAStep({ + name: 'List Persons', + fn: (input) => this.nextToPromise(PersonMWs.listPersons, input) + }); + bm.addAStep({ + name: 'Add sample photo', + fn: (input) => this.nextToPromise(PersonMWs.addSamplePhotoForAll, input) + }); + bm.addAStep({ + name: 'Add thumbnail info', + fn: (input) => this.nextToPromise(ThumbnailGeneratorMWs.addThumbnailInfoForPersons, input) + }); + bm.addAStep({ + name: 'Remove sample photo', + fn: (input) => this.nextToPromise(PersonMWs.removeSamplePhotoForAll, input) + }); + return await bm.run(this.RUNS); + } + + async bmAllSearch(text: string): Promise<{ result: BenchmarkResult, searchType: SearchTypes }[]> { + await this.setupDB(); + const types = Utils.enumToArray(SearchTypes).map(a => a.key).concat([null]); + const results: { result: BenchmarkResult, searchType: SearchTypes }[] = []; + + for (let i = 0; i < types.length; i++) { + const bm = new Benchmark('Searching'); + bm.addAStep({ + name: 'Searching', + fn: async () => { + const sm = new SearchManager(); + return new ContentWrapper(null, await sm.search(text, types[i])); + } + }); + results.push({result: await bm.run(this.RUNS), searchType: types[i]}); + } + return results; + } + + async bmInstantSearch(text: string): Promise { + await this.setupDB(); + const bm = new Benchmark('Instant search'); + bm.addAStep({ + name: 'Instant search', + fn: async () => { + const sm = new SearchManager(); + return new ContentWrapper(null, await sm.instantSearch(text)); + } + }); + return await bm.run(this.RUNS); + } + + async bmAutocomplete(text: string): Promise { + await this.setupDB(); + const bm = new Benchmark('Auto complete'); + bm.addAStep({ + name: 'Auto complete', + fn: () => { + const sm = new SearchManager(); + return sm.autocomplete(text); + } + }); + return await bm.run(this.RUNS); + } + + private nextToPromise(fn: (req: any, res: any, next: Function) => void, input?: any, params = {}) { + return new Promise((resolve, reject) => { + const request = { + resultPipe: input, + params: params, + query: {}, + session: {user: {name: UserRoles[UserRoles.Admin], role: UserRoles.Admin}} + }; + fn(request, resolve, (err?: any) => { + if (err) { + return reject(err); + } + resolve(request.resultPipe); + }); + }); + } + + private resetDB = async () => { + Config.Server.Threading.enabled = false; + await ObjectManagers.reset(); + await rimrafPR(ProjectPath.DBFolder); + Config.Server.Database.type = ServerConfig.DatabaseType.sqlite; + Config.Server.Jobs.scheduled = []; + await ObjectManagers.InitSQLManagers(); + }; + + private setupDB(): Promise { + Config.Server.Threading.enabled = false; + return new Promise(async (resolve, reject) => { + try { + await this.resetDB(); + const indexingJob = new IndexingJob(); + + indexingJob.JobListener = { + onJobFinished: (job: IJob, state: JobProgressStates, soloRun: boolean) => { + resolve(); + }, + + onProgressUpdate: (progress: JobProgress) => { + } + }; + indexingJob.start().catch(console.error); + } catch (e) { + console.error(e); + reject(e); + } + }); + } + +} diff --git a/benchmark/Benchmarks.ts b/benchmark/Benchmarks.ts deleted file mode 100644 index 9764a5cc..00000000 --- a/benchmark/Benchmarks.ts +++ /dev/null @@ -1,164 +0,0 @@ -import {SQLConnection} from '../src/backend/model/database/sql/SQLConnection'; -import {Config} from '../src/common/config/private/Config'; -import {ObjectManagers} from '../src/backend/model/ObjectManagers'; -import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'; -import {IndexingManager} from '../src/backend/model/database/sql/IndexingManager'; -import {SearchManager} from '../src/backend/model/database/sql/SearchManager'; -import * as util from 'util'; -import * as rimraf from 'rimraf'; -import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; -import {Utils} from '../src/common/Utils'; -import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager'; -import {DirectoryDTO} from '../src/common/entities/DirectoryDTO'; -import {ServerConfig} from '../src/common/config/private/PrivateConfig'; -import {ProjectPath} from '../src/backend/ProjectPath'; -import {PersonMWs} from '../src/backend/middlewares/PersonMWs'; -import {ThumbnailGeneratorMWs} from '../src/backend/middlewares/thumbnail/ThumbnailGeneratorMWs'; - -const rimrafPR = util.promisify(rimraf); - -export interface BenchmarkResult { - duration: number; - directories?: number; - media?: number; - items?: number; -} - -export class BMIndexingManager extends IndexingManager { - - public async saveToDB(scannedDirectory: DirectoryDTO): Promise { - return super.saveToDB(scannedDirectory); - } -} - -export class Benchmarks { - - constructor(public RUNS: number) { - - } - - async bmSaveDirectory(): Promise { - await this.resetDB(); - const dir = await DiskMangerWorker.scanDirectory('./'); - const im = new BMIndexingManager(); - return await this.benchmark(() => im.saveToDB(dir), () => this.resetDB()); - } - - async bmScanDirectory(): Promise { - return await this.benchmark(() => DiskMangerWorker.scanDirectory('./')); - } - - async bmListDirectory(): Promise { - const gm = new GalleryManager(); - await this.setupDB(); - Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; - return await this.benchmark(() => gm.listDirectory('./')); - } - - async bmListPersons(): Promise { - await this.setupDB(); - Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; - return await this.benchmark(async () => { - await ObjectManagers.reset(); - const req = {resultPipe: null}; - await this.nextToPromise(PersonMWs.listPersons, req, null); - await this.nextToPromise(PersonMWs.addSamplePhotoForAll, req, null); - await this.nextToPromise(ThumbnailGeneratorMWs.addThumbnailInfoForPersons, req, null); - await this.nextToPromise(PersonMWs.removeSamplePhotoForAll, req, null); - return req.resultPipe; - }); - } - - async bmAllSearch(text: string): Promise<{ result: BenchmarkResult, searchType: SearchTypes }[]> { - await this.setupDB(); - const types = Utils.enumToArray(SearchTypes).map(a => a.key).concat([null]); - const results: { result: BenchmarkResult, searchType: SearchTypes }[] = []; - const sm = new SearchManager(); - for (let i = 0; i < types.length; i++) { - results.push({result: await this.benchmark(() => sm.search(text, types[i])), searchType: types[i]}); - } - return results; - } - - async bmInstantSearch(text: string): Promise { - await this.setupDB(); - const sm = new SearchManager(); - return await this.benchmark(() => sm.instantSearch(text)); - } - - async bmAutocomplete(text: string): Promise { - await this.setupDB(); - const sm = new SearchManager(); - return await this.benchmark(() => sm.autocomplete(text)); - } - - private nextToPromise(fn: (req: any, res: any, next: Function) => void, request: any, response: any) { - return new Promise((resolve, reject) => { - fn(request, resolve, (err?: any) => { - if (err) { - return reject(err); - } - resolve(); - }); - }); - } - - private async benchmark(fn: () => Promise<{ media: any[], directories: any[] } | any[] | void>, - beforeEach: () => Promise = null, - afterEach: () => Promise = null) { - const scanned = await fn(); - const start = process.hrtime(); - let skip = 0; - for (let i = 0; i < this.RUNS; i++) { - if (beforeEach) { - const startSkip = process.hrtime(); - await beforeEach(); - const endSkip = process.hrtime(startSkip); - skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); - } - await fn(); - if (afterEach) { - const startSkip = process.hrtime(); - await afterEach(); - const endSkip = process.hrtime(startSkip); - skip += (endSkip[0] * 1000 + endSkip[1] / 1000000); - } - } - const end = process.hrtime(start); - const duration = (end[0] * 1000 + end[1] / 1000000) / this.RUNS; - - if (!scanned) { - return { - duration: duration - }; - } - - if (Array.isArray(scanned)) { - return { - duration: duration, - items: scanned.length - }; - } - - return { - duration: duration, - media: scanned.media.length, - directories: scanned.directories.length - }; - } - - private resetDB = async () => { - await SQLConnection.close(); - await rimrafPR(ProjectPath.DBFolder); - Config.Server.Database.type = ServerConfig.DatabaseType.sqlite; - await ObjectManagers.InitSQLManagers(); - }; - - private async setupDB() { - const im = new BMIndexingManager(); - await this.resetDB(); - const dir = await DiskMangerWorker.scanDirectory('./'); - await im.saveToDB(dir); - } - -} diff --git a/benchmark/README.md b/benchmark/README.md index 46edac06..81e5266d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -2,6 +2,93 @@ These results are created mostly for development, but the results are public for curious users. +## PiGallery2 v1.8.2, 30.12.2020 +**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm + +**Gallery**: directories: 0 media: 341, faces: 65 + +| Action | Sub action | Action details | Average Duration | Details | +|:------:|:----------:|:--------------:|:----------------:|:-------:| +| **Scanning directory** | | | 1526.7 ms | media: 341, directories:0 | +| **Saving directory to DB** | | | 679.9 ms | - | +| **List directory** | | | 61.1 ms | media: 341, directories:0 | +| | List directory | | 39.7 ms | media: 341, directories:0 | +| | Add Thumbnail information | | 19.3 ms | media: 341, directories:0 | +| | Clean Up Gallery Result | | 2.0 ms | media: 341, directories:0 | +| **Listing Faces** | | | 8.4 ms | items: 1 | +| | List Persons | | 1.0 ms | items: 1 | +| | Add sample photo | | 7.1 ms | items: 1 | +| | Add thumbnail info | | 0.1 ms | items: 1 | +| | Remove sample photo | | 0.0 ms | items: 1 | +| **Searching** | | `a` as `directory` | 3.3 ms | media: 0, directories:0 | +| **Searching** | | `a` as `person` | 11.6 ms | media: 65, directories:0 | +| **Searching** | | `a` as `keyword` | 38.0 ms | media: 339, directories:0 | +| **Searching** | | `a` as `position` | 31.7 ms | media: 282, directories:0 | +| **Searching** | | `a` as `photo` | 3.2 ms | media: 0, directories:0 | +| **Searching** | | `a` as `video` | 3.2 ms | media: 0, directories:0 | +| **Searching** | | `a` as `any` | 39.3 ms | media: 339, directories:0 | +| **Instant search** | | `a` | 6.7 ms | media: 10, directories:0 | +| **Auto complete** | | `a` | 6.7 ms | items: 10 | +*Measurements run 2 times and an average was calculated. + + +## PiGallery2 v1.8.2, 30.12.2020 +**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm + +**Gallery**: directories: 0 media: 341, faces: 65 + +| Action | Sub action | Action details | Average Duration | Details | +|:------:|:----------:|:--------------:|:----------------:|:-------:| +| **Scanning directory** | | | 1459.3 ms | media: 341, directories:0 | +| **Saving directory to DB** | | | 654.9 ms | - | +| **List directory** | | | 70.4 ms | - | +| | List directory | | 42.1 ms | - | +| | Add Thumbnail information | | 25.8 ms | - | +| | Clean Up Gallery Result | | 2.3 ms | - | +| **Listing Faces** | | | 10.5 ms | items: 1 | +| | List Persons | | 1.0 ms | items: 1 | +| | Add sample photo | | 9.1 ms | items: 1 | +| | Add thumbnail info | | 0.2 ms | items: 1 | +| | Remove sample photo | | 0.0 ms | items: 1 | +| **Searching** | | `a` as `directory` | 3.3 ms | - | +| **Searching** | | `a` as `person` | 11.7 ms | media: 65, directories:0 | +| **Searching** | | `a` as `keyword` | 40.4 ms | media: 339, directories:0 | +| **Searching** | | `a` as `position` | 30.4 ms | media: 282, directories:0 | +| **Searching** | | `a` as `photo` | 2.7 ms | - | +| **Searching** | | `a` as `video` | 3.5 ms | - | +| **Searching** | | `a` as `any` | 36.9 ms | media: 339, directories:0 | +| **Instant search** | | `a` | 5.4 ms | media: 10, directories:0 | +| **Auto complete** | | `a` | 6.7 ms | items: 10 | +*Measurements run 2 times and an average was calculated. + + +## PiGallery2 v1.8.2, 30.12.2020 +**System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm + +**Gallery**: directories: 0 media: 341, faces: 65 + +| Action | Sub action | Action details | Average Duration | Details | +|:------:|:----------:|:--------------:|:----------------:|:-------:| +| Scanning directory | | | 1746.5 ms | media: 341, directories:0 | +| Saving directory to DB | | | 994.1 ms | - | +| Scanning directory | | | 65.3 ms | media: 341, directories:0 | +| Listing Faces | | | 19.0 ms | items: 1 | +| | List Persons | | 1.9 ms | items: 1 | +| | Add sample photo | | 16.6 ms | items: 1 | +| | Add thumbnail info | | 0.3 ms | items: 1 | +| | Remove sample photo | | 0.0 ms | items: 1 | +| Searching | | `a` as `directory` | 4.1 ms | - | +| Searching | | `a` as `person` | 16.1 ms | media: 65, directories:0 | +| Searching | | `a` as `keyword` | 41.6 ms | media: 339, directories:0 | +| Searching | | `a` as `position` | 67.1 ms | media: 282, directories:0 | +| Searching | | `a` as `photo` | 5.4 ms | - | +| Searching | | `a` as `video` | 4.3 ms | - | +| Searching | | `a` as `any` | 53.5 ms | media: 339, directories:0 | +| Instant search | | `a` | 5.3 ms | media: 10, directories:0 | +| Auto complete | | `a` | 7.2 ms | items: 10 | +*Measurements run 2 times and an average was calculated. + + ## PiGallery2 v1.5.8, 26.01.2019 **System**: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz, 16GB Ram, SHDD: 1TB, 5400 rpm diff --git a/benchmark/docker-compose/docker-compose.yml b/benchmark/docker-compose/docker-compose.yml new file mode 100644 index 00000000..584262e8 --- /dev/null +++ b/benchmark/docker-compose/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' +services: + pigallery2: + entrypoint: [ "node", "./benchmark/index", "--config-path=/app/data/config/config.json", "--bm-config-path=/app/data/config/bm_config.json" ] + image: bpatrik/pigallery2:latest + container_name: pigallery2 + environment: + - NODE_ENV=production + volumes: + - "./pigallery2/benchmark_config:/app/data/config" # CHANGE ME + - "db-benchmark-data:/app/data/db" + - "./pigallery2/images:/app/data/images" # CHANGE ME + - "./pigallery2/tmp:/app/data/tmp" # CHANGE ME + expose: + - "80" + restart: always + +volumes: + db-benchmark-data: diff --git a/benchmark/index.ts b/benchmark/index.ts index 5b081ec1..c73cb165 100644 --- a/benchmark/index.ts +++ b/benchmark/index.ts @@ -1,18 +1,17 @@ import {Config} from '../src/common/config/private/Config'; -import * as path from 'path'; import {ProjectPath} from '../src/backend/ProjectPath'; -import {BenchmarkResult, Benchmarks} from './Benchmarks'; +import {BenchmarkResult, BenchmarkRunner} from './BenchmarkRunner'; import {SearchTypes} from '../src/common/entities/AutoCompleteItem'; import {Utils} from '../src/common/Utils'; import {DiskMangerWorker} from '../src/backend/model/threading/DiskMangerWorker'; import {BMConfig} from './BMConfig'; import {GalleryManager} from '../src/backend/model/database/sql/GalleryManager'; +import {PersonManager} from '../src/backend/model/database/sql/PersonManager'; Config.Server.Media.folder = BMConfig.path; -const dbFolder = path.join(__dirname, './../'); ProjectPath.reset(); -const RUNS = 50; +const RUNS = BMConfig.RUNS; let resultsText = ''; const printLine = (text: string) => { @@ -28,54 +27,75 @@ const printHeader = async () => { printLine('**System**: ' + BMConfig.system); const dir = await DiskMangerWorker.scanDirectory('./'); const gm = new GalleryManager(); - printLine('**Gallery**: directories: ' + + const pm = new PersonManager(); + printLine('\n**Gallery**: directories: ' + dir.directories.length + ' media: ' + dir.media.length + // @ts-ignore - ', faces: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0)); + ', all persons: ' + dir.media.reduce((p, c) => p + (c.metadata.faces || []).length, 0) + + ',unique persons (faces): ' + (await pm.getAll()).length + '\n'); }; const printTableHeader = () => { - printLine('| action | action details | average time | details |'); - printLine('|:------:|:--------------:|:------------:|:-------:|'); + printLine('| Action | Sub action | Action details | Average Duration | Details |'); + printLine('|:------:|:----------:|:--------------:|:----------------:|:-------:|'); }; -const printResult = (result: BenchmarkResult, action: string, actionDetails: string = '') => { - console.log('benchmarked: ' + action); +const printResult = (result: BenchmarkResult, actionDetails: string = '', isSubResult = false) => { + console.log('benchmarked: ' + result.name); let details = '-'; if (result.items) { details = 'items: ' + result.items; } - if (result.media) { - details = 'media: ' + result.media + ', directories:' + result.directories; + if (result.contentWrapper) { + if (result.contentWrapper.directory) { + details = 'media: ' + result.contentWrapper.directory.media.length + + ', directories:' + result.contentWrapper.directory.directories.length; + } else { + details = 'media: ' + result.contentWrapper.searchResult.media.length + + ', directories:' + result.contentWrapper.searchResult.directories.length; + } + } + if (isSubResult) { + printLine('| | ' + result.name + ' | ' + actionDetails + + ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |'); + } else { + printLine('| **' + result.name + '** | | ' + actionDetails + + ' | ' + (result.duration).toFixed(1) + ' ms | ' + details + ' |'); + } + if (result.subBenchmarks && result.subBenchmarks.length > 1) { + for (let i = 0; i < result.subBenchmarks.length; i++) { + printResult(result.subBenchmarks[i], '', true); + } } - printLine('| ' + action + ' | ' + actionDetails + - ' | ' + (result.duration).toFixed(1) + 'ms | ' + details + ' |'); }; const run = async () => { + console.log('Running, RUNS:' + RUNS); const start = Date.now(); - const bm = new Benchmarks(RUNS); + const bm = new BenchmarkRunner(RUNS); // header await printHeader(); printTableHeader(); - printResult(await bm.bmScanDirectory(), 'Scanning directory'); - printResult(await bm.bmSaveDirectory(), 'Saving directory'); - printResult(await bm.bmListDirectory(), 'Listing Directory'); - printResult(await bm.bmListPersons(), 'Listing Faces'); + + printResult(await bm.bmScanDirectory()); + printResult(await bm.bmSaveDirectory()); + printResult(await bm.bmListDirectory()); + printResult(await bm.bmListPersons()); (await bm.bmAllSearch('a')).forEach(res => { if (res.searchType !== null) { - printResult(res.result, 'searching', '`a` as `' + SearchTypes[res.searchType] + '`'); + printResult(res.result, '`a` as `' + SearchTypes[res.searchType] + '`'); } else { - printResult(res.result, 'searching', '`a` as `any`'); + printResult(res.result, '`a` as `any`'); } }); - printResult(await bm.bmInstantSearch('a'), 'instant search', '`a`'); - printResult(await bm.bmAutocomplete('a'), 'auto complete', '`a`'); + printResult(await bm.bmInstantSearch('a'), '`a`'); + printResult(await bm.bmAutocomplete('a'), '`a`'); + printLine('*Measurements run ' + RUNS + ' times and an average was calculated.'); console.log(resultsText); console.log('run for : ' + ((Date.now() - start)).toFixed(1) + 'ms'); }; -run(); +run().then(console.log).catch(console.error); diff --git a/gulpfile.ts b/gulpfile.ts index 6e2300b1..efd34b14 100644 --- a/gulpfile.ts +++ b/gulpfile.ts @@ -38,7 +38,8 @@ const getSwitch = (name: string, def: string = null): string => { gulp.task('build-backend', function () { return gulp.src([ 'src/common/**/*.ts', - 'src/backend/**/*.ts'], {base: '.'}) + 'src/backend/**/*.ts', + 'benchmark/**/*.ts'], {base: '.'}) .pipe(tsBackendProject()) .js .pipe(gulp.dest('./release')); diff --git a/src/backend/model/database/interfaces/IIndexingManager.ts b/src/backend/model/database/interfaces/IIndexingManager.ts index b02618cc..faf0e2a3 100644 --- a/src/backend/model/database/interfaces/IIndexingManager.ts +++ b/src/backend/model/database/interfaces/IIndexingManager.ts @@ -1,6 +1,9 @@ import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; export interface IIndexingManager { + SavingReady: Promise + IsSavingInProgress: boolean; + indexDirectory(relativeDirectoryName: string): Promise; resetDB(): Promise; diff --git a/src/backend/model/database/sql/IndexingManager.ts b/src/backend/model/database/sql/IndexingManager.ts index 48a71f79..b69b6306 100644 --- a/src/backend/model/database/sql/IndexingManager.ts +++ b/src/backend/model/database/sql/IndexingManager.ts @@ -22,9 +22,15 @@ const LOG_TAG = '[IndexingManager]'; export class IndexingManager implements IIndexingManager { + SavingReady: Promise = null; + SavingReadyPR: () => void = null; private savingQueue: DirectoryDTO[] = []; private isSaving = false; + get IsSavingInProgress() { + return this.SavingReady !== null; + } + public indexDirectory(relativeDirectoryName: string): Promise { return new Promise(async (resolve, reject) => { try { @@ -67,10 +73,19 @@ export class IndexingManager implements IIndexingManager { return; } this.savingQueue.push(scannedDirectory); + if (!this.SavingReady) { + this.SavingReady = new Promise((resolve) => { + this.SavingReadyPR = resolve; + }); + } while (this.isSaving === false && this.savingQueue.length > 0) { await this.saveToDB(this.savingQueue[0]); this.savingQueue.shift(); } + if (this.savingQueue.length === 0) { + this.SavingReady = null; + this.SavingReadyPR(); + } } diff --git a/src/backend/model/database/sql/SQLConnection.ts b/src/backend/model/database/sql/SQLConnection.ts index 447491b5..b775289f 100644 --- a/src/backend/model/database/sql/SQLConnection.ts +++ b/src/backend/model/database/sql/SQLConnection.ts @@ -106,6 +106,7 @@ export class SQLConnection { this.connection = null; } } catch (err) { + console.error('Error during closing sql db:'); console.error(err); } } diff --git a/src/backend/model/jobs/jobs/IndexingJob.ts b/src/backend/model/jobs/jobs/IndexingJob.ts index b170dbe0..7158b982 100644 --- a/src/backend/model/jobs/jobs/IndexingJob.ts +++ b/src/backend/model/jobs/jobs/IndexingJob.ts @@ -23,6 +23,9 @@ export class IndexingJob extends Job { protected async step(): Promise { if (this.directoriesToIndex.length === 0) { + if (ObjectManagers.getInstance().IndexingManager.IsSavingInProgress) { + await ObjectManagers.getInstance().IndexingManager.SavingReady; + } return false; } const directory = this.directoriesToIndex.shift();