From c716ff4ca7fb1e3d97febf6f9e51e38343174804 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Mon, 30 Dec 2019 17:52:58 +0100 Subject: [PATCH] improving jobs --- package.json | 2 +- src/backend/model/jobs/jobs/FileJob.ts | 28 ++++++------ .../model/jobs/jobs/PhotoConvertingJob.ts | 6 +-- .../model/jobs/jobs/ThumbnailGenerationJob.ts | 11 ++--- .../model/jobs/jobs/VideoConvertingJob.ts | 13 +++--- src/common/entities/MediaDTO.ts | 10 +++++ .../button/job-button.settings.component.ts | 4 +- .../job-progress.settings.component.html | 4 +- .../photo/photo.settings.component.html | 2 +- .../app/ui/settings/scheduled-jobs.service.ts | 43 ++++++++++++++++--- .../thumbnail.settings.component.html | 2 +- .../video/video.settings.component.html | 2 +- src/frontend/translate/messages.en.xlf | 12 ++++-- src/frontend/translate/messages.fr.xlf | 12 ++++-- src/frontend/translate/messages.hu.xlf | 8 ++++ src/frontend/translate/messages.ro.xlf | 12 ++++-- src/frontend/translate/messages.ru.xlf | 12 ++++-- 17 files changed, 126 insertions(+), 57 deletions(-) diff --git a/package.json b/package.json index a56725c3..f985baac 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "pretest": "tsc", "test": "ng test && mocha --recursive test/backend/unit && mocha --recursive test/backend/integration && mocha --recursive test/common/unit ", "start": "node ./src/backend/index", - "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale hu --i18n-file src/frontend/translate/messages.hu.xlf --i18n-missing-translation warning", + "run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning", "build-stats": "ng build --aot --prod --stats-json --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning", "merge-new-translation": "gulp merge-new-translation", "add-translation": "gulp add-translation" diff --git a/src/backend/model/jobs/jobs/FileJob.ts b/src/backend/model/jobs/jobs/FileJob.ts index b6e0d55d..7bf2860a 100644 --- a/src/backend/model/jobs/jobs/FileJob.ts +++ b/src/backend/model/jobs/jobs/FileJob.ts @@ -12,7 +12,7 @@ import {MediaEntity} from '../../database/sql/enitites/MediaEntity'; import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity'; import {VideoEntity} from '../../database/sql/enitites/VideoEntity'; import {backendTexts} from '../../../../common/BackendTexts'; -import DatabaseType = ServerConfig.DatabaseType; +import {ProjectPath} from '../../../ProjectPath'; declare var global: NodeJS.Global; @@ -23,12 +23,12 @@ const LOG_TAG = '[FileJob]'; export abstract class FileJob extends Job { public readonly ConfigTemplate: ConfigTemplateEntry[] = []; directoryQueue: string[] = []; - fileQueue: FileDTO[] = []; + fileQueue: string[] = []; protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) { super(); - if (Config.Server.Database.type !== DatabaseType.memory) { + if (Config.Server.Database.type !== ServerConfig.DatabaseType.memory) { this.ConfigTemplate.push({ id: 'indexedOnly', type: 'boolean', @@ -54,9 +54,9 @@ export abstract class FileJob; + protected abstract async shouldProcess(filePath: string): Promise; - protected abstract async processFile(file: FileDTO): Promise; + protected abstract async processFile(filePath: string): Promise; protected async step(): Promise { if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) { @@ -66,7 +66,7 @@ export abstract class FileJob 0) { if (this.config.indexedOnly === true && - Config.Server.Database.type !== DatabaseType.memory) { + Config.Server.Database.type !== ServerConfig.DatabaseType.memory) { await this.loadAllMediaFilesFromDB(); this.directoryQueue = []; } else { @@ -74,13 +74,12 @@ export abstract class FileJob 0) { this.Progress.Left = this.fileQueue.length; - const file = this.fileQueue.shift(); - const filePath = path.join(file.directory.path, file.directory.name, file.name); + const filePath = this.fileQueue.shift(); try { - if ((await this.shouldProcess(file)) === true) { + if ((await this.shouldProcess(filePath)) === true) { this.Progress.Processed++; this.Progress.log('processing: ' + filePath); - await this.processFile(file); + await this.processFile(filePath); } else { this.Progress.log('skipping: ' + filePath); this.Progress.Skipped++; @@ -102,10 +101,12 @@ export abstract class FileJob path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))); } if (this.scanFilter.noMetaFile !== true) { - this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile)); + this.fileQueue.push(...(await this.filterMetaFiles(scanned.metaFile)) + .map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))); } } @@ -134,6 +135,7 @@ export abstract class FileJob path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)))); } } diff --git a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts index 46746b48..f1757fa5 100644 --- a/src/backend/model/jobs/jobs/PhotoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/PhotoConvertingJob.ts @@ -21,14 +21,12 @@ export class PhotoConvertingJob extends FileJob { } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution)); } - protected async processFile(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { await PhotoProcessing.convertPhoto(mPath); } diff --git a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts index e22c4b4e..c347cd6d 100644 --- a/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts +++ b/src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts @@ -1,7 +1,5 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; -import {ProjectPath} from '../../../ProjectPath'; -import * as path from 'path'; import {FileJob} from './FileJob'; import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing'; import {ThumbnailSourceType} from '../../threading/PhotoWorker'; @@ -49,8 +47,7 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn return undefined; } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { for (let i = 0; i < this.config.sizes.length; ++i) { if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) { return true; @@ -58,13 +55,11 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn } } - protected async processFile(file: FileDTO): Promise { - - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { for (let i = 0; i < this.config.sizes.length; ++i) { await PhotoProcessing.generateThumbnail(mPath, this.config.sizes[i], - MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, + MediaDTO.isVideoPath(mPath) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo, false); } diff --git a/src/backend/model/jobs/jobs/VideoConvertingJob.ts b/src/backend/model/jobs/jobs/VideoConvertingJob.ts index 691532f7..95f1bcf7 100644 --- a/src/backend/model/jobs/jobs/VideoConvertingJob.ts +++ b/src/backend/model/jobs/jobs/VideoConvertingJob.ts @@ -1,12 +1,10 @@ import {Config} from '../../../../common/config/private/Config'; import {DefaultsJobs} from '../../../../common/entities/job/JobDTO'; -import {ProjectPath} from '../../../ProjectPath'; -import * as path from 'path'; import {FileJob} from './FileJob'; import {VideoProcessing} from '../../fileprocessing/VideoProcessing'; -import {FileDTO} from '../../../../common/entities/FileDTO'; const LOG_TAG = '[VideoConvertingJob]'; +declare const global: any; export class VideoConvertingJob extends FileJob { @@ -20,14 +18,15 @@ export class VideoConvertingJob extends FileJob { return Config.Client.Media.Video.enabled === true; } - protected async shouldProcess(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async shouldProcess(mPath: string): Promise { return !(await VideoProcessing.convertedVideoExist(mPath)); } - protected async processFile(file: FileDTO): Promise { - const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name); + protected async processFile(mPath: string): Promise { await VideoProcessing.convertVideo(mPath); + if (global.gc) { + global.gc(); + } } diff --git a/src/common/entities/MediaDTO.ts b/src/common/entities/MediaDTO.ts index 71833484..08a15e55 100644 --- a/src/common/entities/MediaDTO.ts +++ b/src/common/entities/MediaDTO.ts @@ -65,6 +65,16 @@ export module MediaDTO { return false; }; + export const isVideoPath = (path: string): boolean => { + const lower = path.toLowerCase(); + for (const ext of SupportedFormats.WithDots.Videos) { + if (lower.endsWith(ext)) { + return true; + } + } + return false; + }; + export const isVideoTranscodingNeeded = (media: FileDTO): boolean => { const lower = media.name.toLowerCase(); for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) { diff --git a/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts b/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts index d9db5acd..23bdf312 100644 --- a/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts +++ b/src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts @@ -41,7 +41,7 @@ export class JobButtonComponent { this.error.emit(''); try { await this.jobsService.start(this.jobName, this.config, this.soloRun); - this.notification.info(this.i18n('Job started') + ': ' + this.jobName); + this.notification.success(this.i18n('Job started') + ': ' + this.backendTextService.getJobName(this.jobName)); return true; } catch (err) { console.log(err); @@ -57,7 +57,7 @@ export class JobButtonComponent { this.error.emit(''); try { await this.jobsService.stop(this.jobName); - this.notification.info(this.i18n('Job stopped') + ': ' + this.jobName); + this.notification.info(this.i18n('Job stopped') + ': ' + this.backendTextService.getJobName(this.jobName)); return true; } catch (err) { console.log(err); diff --git a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html index e1e54998..aa452750 100644 --- a/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html +++ b/src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html @@ -25,7 +25,7 @@
-

+

...

diff --git a/src/frontend/app/ui/settings/photo/photo.settings.component.html b/src/frontend/app/ui/settings/photo/photo.settings.component.html index f5d2f62f..74e6fe3c 100644 --- a/src/frontend/app/ui/settings/photo/photo.settings.component.html +++ b/src/frontend/app/ui/settings/photo/photo.settings.component.html @@ -100,7 +100,7 @@

- diff --git a/src/frontend/app/ui/settings/scheduled-jobs.service.ts b/src/frontend/app/ui/settings/scheduled-jobs.service.ts index 8cd0b108..5ada4906 100644 --- a/src/frontend/app/ui/settings/scheduled-jobs.service.ts +++ b/src/frontend/app/ui/settings/scheduled-jobs.service.ts @@ -1,9 +1,12 @@ import {EventEmitter, Injectable} from '@angular/core'; import {BehaviorSubject} from 'rxjs'; -import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO'; +import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO'; import {NetworkService} from '../../model/network/network.service'; import {JobScheduleDTO} from '../../../../common/entities/job/JobScheduleDTO'; import {JobDTO} from '../../../../common/entities/job/JobDTO'; +import {BackendtextService} from '../../model/backendtext.service'; +import {NotificationService} from '../../model/notification.service'; +import {I18n} from '@ngx-translate/i18n-polyfill'; @Injectable() export class ScheduledJobsService { @@ -15,7 +18,10 @@ export class ScheduledJobsService { public jobStartingStopping: { [key: string]: boolean } = {}; private subscribers = 0; - constructor(private _networkService: NetworkService) { + constructor(private _networkService: NetworkService, + private notification: NotificationService, + private backendTextService: BackendtextService, + private i18n: I18n) { this.progress = new BehaviorSubject({}); } @@ -39,6 +45,8 @@ export class ScheduledJobsService { this.jobStartingStopping[jobName] = true; await this._networkService.postJson('/admin/jobs/scheduled/' + jobName + '/' + (soloStart === true ? 'soloStart' : 'start'), {config: config}); + // placeholder to force showing running job + this.addDummyProgress(jobName, config); delete this.jobStartingStopping[jobName]; this.forceUpdate(); } @@ -53,14 +61,20 @@ export class ScheduledJobsService { protected async loadProgress(): Promise { const prevPrg = this.progress.value; this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress')); - for (const prg in prevPrg) { - if (!this.progress.value.hasOwnProperty(prg)) { + for (const prg of Object.keys(prevPrg)) { + if (!this.progress.value.hasOwnProperty(prg) || + // state changed from running to finished + ((prevPrg[prg].state === JobProgressStates.running || + prevPrg[prg].state === JobProgressStates.cancelling) && + !(this.progress.value[prg].state === JobProgressStates.running || + this.progress.value[prg].state === JobProgressStates.cancelling) + )) { this.onJobFinish.emit(prg); + this.notification.info(this.i18n('Job finished') + ': ' + this.backendTextService.getJobName(prevPrg[prg].jobName)); } } } - protected getProgressPeriodically() { if (this.timer != null || this.subscribers === 0) { return; @@ -76,6 +90,25 @@ export class ScheduledJobsService { this.loadProgress().catch(console.error); } + private addDummyProgress(jobName: string, config: any) { + const prgs = this.progress.value; + prgs[JobDTO.getHashName(jobName, config)] = { + jobName: jobName, + state: JobProgressStates.running, + HashName: JobDTO.getHashName(jobName, config), + logs: [], steps: { + skipped: 0, + processed: 0, + all: 0 + }, + time: { + start: Date.now(), + end: Date.now() + } + }; + this.progress.next(prgs); + } + private incSubscribers() { this.subscribers++; this.getProgressPeriodically(); diff --git a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html index fde9cd8c..6e5ea85e 100644 --- a/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html +++ b/src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html @@ -74,7 +74,7 @@ - Reset - diff --git a/src/frontend/translate/messages.en.xlf b/src/frontend/translate/messages.en.xlf index af29f867..98313dfd 100644 --- a/src/frontend/translate/messages.en.xlf +++ b/src/frontend/translate/messages.en.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.fr.xlf b/src/frontend/translate/messages.fr.xlf index 86bfc1ca..f8d16257 100644 --- a/src/frontend/translate/messages.fr.xlf +++ b/src/frontend/translate/messages.fr.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.hu.xlf b/src/frontend/translate/messages.hu.xlf index 018b5e79..93b6f769 100644 --- a/src/frontend/translate/messages.hu.xlf +++ b/src/frontend/translate/messages.hu.xlf @@ -2590,6 +2590,14 @@ Véletleg Fotó + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Feladat végzett + Thumbnail diff --git a/src/frontend/translate/messages.ro.xlf b/src/frontend/translate/messages.ro.xlf index e5ff8ebe..486f7a68 100644 --- a/src/frontend/translate/messages.ro.xlf +++ b/src/frontend/translate/messages.ro.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail diff --git a/src/frontend/translate/messages.ru.xlf b/src/frontend/translate/messages.ru.xlf index 5f33f402..3e522dab 100644 --- a/src/frontend/translate/messages.ru.xlf +++ b/src/frontend/translate/messages.ru.xlf @@ -904,9 +904,7 @@ app/ui/settings/thumbnail/thumbnail.settings.component.html 59 - ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 - pixels. - + ';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels. Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or @@ -2592,6 +2590,14 @@ Random Photo + + Job finished + + src/frontend/app/ui/settings/scheduled-jobs.service.ts + 1 + + Job finished + Thumbnail