From 9fdd732ce5736c7d1e0931f9196f2c423abb83e0 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sun, 18 Dec 2022 23:47:13 +0100 Subject: [PATCH] Making file job load files from DB in batches. #494 --- src/backend/model/jobs/jobs/FileJob.ts | 113 ++++++++++++++++++--- src/backend/model/jobs/jobs/JobProgress.ts | 3 +- src/common/config/private/PrivateConfig.ts | 4 +- 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index 9a41bb49..481218b7 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -25,6 +25,12 @@ export abstract class FileJob { this.directoryQueue = []; this.fileQueue = []; + this.DBProcessing = { + mediaLoaded: 0, + hasMoreMedia: true, + initiated: false + }; this.directoryQueue.push('/'); } @@ -59,22 +70,17 @@ export abstract class FileJob; protected async step(): Promise { - if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { + const DBBased = this.config.indexedOnly && + Config.Server.Database.type !== DatabaseType.memory; + if ( + this.fileQueue.length === 0 && + ((this.directoryQueue.length === 0 && !DBBased) || + (DBBased && + this.DBProcessing.hasMoreMedia === false))) { return false; } - if (this.directoryQueue.length > 0) { - if ( - this.config.indexedOnly === true && - Config.Server.Database.type !== DatabaseType.memory - ) { - await this.loadAllMediaFilesFromDB(); - this.directoryQueue = []; - } else { - await this.loadADirectoryFromDisk(); - } - } else if (this.fileQueue.length > 0) { - this.Progress.Left = this.fileQueue.length; + const processOneFile = async () => { const filePath = this.fileQueue.shift(); try { if ((await this.shouldProcess(filePath)) === true) { @@ -95,7 +101,32 @@ export abstract class FileJob 0) { + await this.loadADirectoryFromDisk(); + } else if (this.fileQueue.length > 0) { + this.Progress.Left = this.fileQueue.length; + await processOneFile(); + } + } else { + if (!this.DBProcessing.initiated) { + this.Progress.log('Counting files from db'); + Logger.silly(LOG_TAG, 'Counting files from db'); + this.Progress.All = await this.countMediaFromDB(); + Logger.silly(LOG_TAG, 'Found:' + this.Progress.All); + this.DBProcessing.initiated = true; + return true; + } + if (this.fileQueue.length === 0) { + await this.loadMediaFilesFromDB(); + } else { + this.Progress.Left = this.fileQueue.length; + await processOneFile(); + } } + return true; } @@ -138,16 +169,21 @@ export abstract class FileJob { + private async loadMediaFilesFromDB(): Promise { if (this.scanFilter.noVideo === true && this.scanFilter.noPhoto === true && this.scanFilter.noMetaFile === true) { return; } - this.Progress.log('Loading files from db'); - Logger.silly(LOG_TAG, 'Loading files from db'); + const logStr = `Loading next batch of files from db. Skipping: ${this.DBProcessing.mediaLoaded}, looking for more: ${Config.Server.Jobs.mediaProcessingBatchSize}`; + this.Progress.log(logStr); + Logger.silly(LOG_TAG, logStr); + const hasMoreFile = { + media: false, + metafile: false + }; const connection = await SQLConnection.getConnection(); if (!this.scanFilter.noVideo || !this.scanFilter.noPhoto) { @@ -165,8 +201,12 @@ export abstract class FileJob 0; + this.DBProcessing.mediaLoaded += result.length; const scannedAndFiltered = await this.filterMediaFiles(result); for (const item of scannedAndFiltered) { this.fileQueue.push( @@ -186,9 +226,13 @@ export abstract class FileJob 0; + this.DBProcessing.mediaLoaded += result.length; const scannedAndFiltered = await this.filterMetaFiles(result); for (const item of scannedAndFiltered) { this.fileQueue.push( @@ -201,5 +245,42 @@ export abstract class FileJob { + if (this.scanFilter.noVideo === true && + this.scanFilter.noPhoto === true && + this.scanFilter.noMetaFile === true) { + return; + } + let count = 0; + const connection = await SQLConnection.getConnection(); + if (!this.scanFilter.noVideo || + !this.scanFilter.noPhoto) { + + let usedEntity = MediaEntity; + + if (this.scanFilter.noVideo === true) { + usedEntity = PhotoEntity; + } else if (this.scanFilter.noPhoto === true) { + usedEntity = VideoEntity; + } + + count += await connection + .getRepository(usedEntity) + .createQueryBuilder('media') + .getCount(); + + if (!this.scanFilter.noMetaFile) { + + count += await connection + .getRepository(FileEntity) + .createQueryBuilder('file') + .getCount(); + + } + return count; + } } } diff --git a/src/backend/model/jobs/jobs/JobProgress.ts b/src/backend/model/jobs/jobs/JobProgress.ts index ae758bc5..e38c80bc 100644 --- a/src/backend/model/jobs/jobs/JobProgress.ts +++ b/src/backend/model/jobs/jobs/JobProgress.ts @@ -3,6 +3,7 @@ import { JobProgressLogDTO, JobProgressStates, } from '../../../../common/entities/job/JobProgressDTO'; +import {Config} from '../../../../common/config/private/Config'; export class JobProgress { private steps = { @@ -86,7 +87,7 @@ export class JobProgress { }; log(log: string): void { - while (this.logs.length > 10) { + while (this.logs.length > Config.Server.Jobs.maxSavedProgress) { this.logs.shift(); } this.logs.push({ diff --git a/src/common/config/private/PrivateConfig.ts b/src/common/config/private/PrivateConfig.ts index 8319ac63..9d40a717 100644 --- a/src/common/config/private/PrivateConfig.ts +++ b/src/common/config/private/PrivateConfig.ts @@ -316,7 +316,9 @@ export class JobScheduleConfig implements JobScheduleDTO { @SubConfigClass() export class ServerJobConfig { @ConfigProperty({type: 'integer', description: 'Job history size'}) - maxSavedProgress: number = 10; + maxSavedProgress: number = 20; + @ConfigProperty({type: 'integer', description: 'Job loads this many photos or videos form the DB for processing'}) + mediaProcessingBatchSize: number = 1000; @ConfigProperty({arrayType: JobScheduleConfig}) scheduled: JobScheduleConfig[] = [ new JobScheduleConfig(