xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00
Code Actions Projects Activity

improving jobs

Browse Source
This commit is contained in:
Patrik J. Braun 2019-12-30 17:52:58 +01:00
parent 34267e8ed8
commit c716ff4ca7
17 changed files with 126 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -14,7 +14,7 @@
"pretest": "tsc",
"test": "ng test && mocha --recursive test/backend/unit && mocha --recursive test/backend/integration && mocha --recursive test/common/unit ",
"start": "node ./src/backend/index",
"run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale hu --i18n-file src/frontend/translate/messages.hu.xlf --i18n-missing-translation warning",
"run-dev": "ng build --aot --watch --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning",
"build-stats": "ng build --aot --prod --stats-json --output-path=./dist --i18n-locale en --i18n-file src/frontend/translate/messages.en.xlf --i18n-missing-translation warning",
"merge-new-translation": "gulp merge-new-translation",
"add-translation": "gulp add-translation"

28
src/backend/model/jobs/jobs/FileJob.ts
View File

@ -12,7 +12,7 @@ import {MediaEntity} from '../../database/sql/enitites/MediaEntity';
import {PhotoEntity} from '../../database/sql/enitites/PhotoEntity';
import {VideoEntity} from '../../database/sql/enitites/VideoEntity';
import {backendTexts} from '../../../../common/BackendTexts';
import DatabaseType = ServerConfig.DatabaseType;
import {ProjectPath} from '../../../ProjectPath';
declare var global: NodeJS.Global;
@ -23,12 +23,12 @@ const LOG_TAG = '[FileJob]';
export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly: boolean }> extends Job<S> {
public readonly ConfigTemplate: ConfigTemplateEntry[] = [];
directoryQueue: string[] = [];
fileQueue: FileDTO[] = [];
fileQueue: string[] = [];
protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) {
super();
if (Config.Server.Database.type !== DatabaseType.memory) {
if (Config.Server.Database.type !== ServerConfig.DatabaseType.memory) {
this.ConfigTemplate.push({
id: 'indexedOnly',
type: 'boolean',
@ -54,9 +54,9 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
return files;
}
protected abstract async shouldProcess(file: FileDTO): Promise<boolean>;
protected abstract async shouldProcess(filePath: string): Promise<boolean>;
protected abstract async processFile(file: FileDTO): Promise<void>;
protected abstract async processFile(filePath: string): Promise<void>;
protected async step(): Promise<boolean> {
if (this.directoryQueue.length === 0 && this.fileQueue.length === 0) {
@ -66,7 +66,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
if (this.directoryQueue.length > 0) {
if (this.config.indexedOnly === true &&
Config.Server.Database.type !== DatabaseType.memory) {
Config.Server.Database.type !== ServerConfig.DatabaseType.memory) {
await this.loadAllMediaFilesFromDB();
this.directoryQueue = [];
} else {
@ -74,13 +74,12 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
}
} else if (this.fileQueue.length > 0) {
this.Progress.Left = this.fileQueue.length;
const file = this.fileQueue.shift();
const filePath = path.join(file.directory.path, file.directory.name, file.name);
const filePath = this.fileQueue.shift();
try {
if ((await this.shouldProcess(file)) === true) {
if ((await this.shouldProcess(filePath)) === true) {
this.Progress.Processed++;
this.Progress.log('processing: ' + filePath);
await this.processFile(file);
await this.processFile(filePath);
} else {
this.Progress.log('skipping: ' + filePath);
this.Progress.Skipped++;
@ -102,10 +101,12 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
}
if (this.scanFilter.noVideo !== true || this.scanFilter.noVideo !== true) {
this.fileQueue.push(...await this.filterMediaFiles(scanned.media));
this.fileQueue.push(...(await this.filterMediaFiles(scanned.media))
.map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)));
}
if (this.scanFilter.noMetaFile !== true) {
this.fileQueue.push(...await this.filterMetaFiles(scanned.metaFile));
this.fileQueue.push(...(await this.filterMetaFiles(scanned.metaFile))
.map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name)));
}
}
@ -134,6 +135,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
.leftJoinAndSelect('media.directory', 'directory')
.getMany();
this.fileQueue.push(...await this.filterMediaFiles(result));
this.fileQueue.push(...(result
.map(f => path.join(ProjectPath.ImageFolder, f.directory.path, f.directory.name, f.name))));
}
}

6
src/backend/model/jobs/jobs/PhotoConvertingJob.ts
View File

@ -21,14 +21,12 @@ export class PhotoConvertingJob extends FileJob {
}
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async shouldProcess(mPath: string): Promise<boolean> {
return !(await PhotoProcessing.convertedPhotoExist(mPath, Config.Server.Media.Photo.Converting.resolution));
}
protected async processFile(file: FileDTO): Promise<void> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async processFile(mPath: string): Promise<void> {
await PhotoProcessing.convertPhoto(mPath);
}

11
src/backend/model/jobs/jobs/ThumbnailGenerationJob.ts
View File

@ -1,7 +1,5 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {ProjectPath} from '../../../ProjectPath';
import * as path from 'path';
import {FileJob} from './FileJob';
import {PhotoProcessing} from '../../fileprocessing/PhotoProcessing';
import {ThumbnailSourceType} from '../../threading/PhotoWorker';
@ -49,8 +47,7 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
return undefined;
}
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async shouldProcess(mPath: string): Promise<boolean> {
for (let i = 0; i < this.config.sizes.length; ++i) {
if (!(await PhotoProcessing.convertedPhotoExist(mPath, this.config.sizes[i]))) {
return true;
@ -58,13 +55,11 @@ export class ThumbnailGenerationJob extends FileJob<{ sizes: number[], indexedOn
}
}
protected async processFile(file: FileDTO): Promise<void> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async processFile(mPath: string): Promise<void> {
for (let i = 0; i < this.config.sizes.length; ++i) {
await PhotoProcessing.generateThumbnail(mPath,
this.config.sizes[i],
MediaDTO.isVideo(file) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
MediaDTO.isVideoPath(mPath) ? ThumbnailSourceType.Video : ThumbnailSourceType.Photo,
false);
}

13
src/backend/model/jobs/jobs/VideoConvertingJob.ts
View File

@ -1,12 +1,10 @@
import {Config} from '../../../../common/config/private/Config';
import {DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {ProjectPath} from '../../../ProjectPath';
import * as path from 'path';
import {FileJob} from './FileJob';
import {VideoProcessing} from '../../fileprocessing/VideoProcessing';
import {FileDTO} from '../../../../common/entities/FileDTO';
const LOG_TAG = '[VideoConvertingJob]';
declare const global: any;
export class VideoConvertingJob extends FileJob {
@ -20,14 +18,15 @@ export class VideoConvertingJob extends FileJob {
return Config.Client.Media.Video.enabled === true;
}
protected async shouldProcess(file: FileDTO): Promise<boolean> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async shouldProcess(mPath: string): Promise<boolean> {
return !(await VideoProcessing.convertedVideoExist(mPath));
}
protected async processFile(file: FileDTO): Promise<void> {
const mPath = path.join(ProjectPath.ImageFolder, file.directory.path, file.directory.name, file.name);
protected async processFile(mPath: string): Promise<void> {
await VideoProcessing.convertVideo(mPath);
if (global.gc) {
global.gc();
}
}

10
src/common/entities/MediaDTO.ts
View File

@ -65,6 +65,16 @@ export module MediaDTO {
return false;
};
export const isVideoPath = (path: string): boolean => {
const lower = path.toLowerCase();
for (const ext of SupportedFormats.WithDots.Videos) {
if (lower.endsWith(ext)) {
return true;
}
}
return false;
};
export const isVideoTranscodingNeeded = (media: FileDTO): boolean => {
const lower = media.name.toLowerCase();
for (const ext of SupportedFormats.WithDots.TranscodeNeed.Videos) {

4
src/frontend/app/ui/settings/jobs/button/job-button.settings.component.ts
View File

@ -41,7 +41,7 @@ export class JobButtonComponent {
this.error.emit('');
try {
await this.jobsService.start(this.jobName, this.config, this.soloRun);
this.notification.info(this.i18n('Job started') + ': ' + this.jobName);
this.notification.success(this.i18n('Job started') + ': ' + this.backendTextService.getJobName(this.jobName));
return true;
} catch (err) {
console.log(err);
@ -57,7 +57,7 @@ export class JobButtonComponent {
this.error.emit('');
try {
await this.jobsService.stop(this.jobName);
this.notification.info(this.i18n('Job stopped') + ': ' + this.jobName);
this.notification.info(this.i18n('Job stopped') + ': ' + this.backendTextService.getJobName(this.jobName));
return true;
} catch (err) {
console.log(err);

4
src/frontend/app/ui/settings/jobs/progress/job-progress.settings.component.html
View File

@ -25,7 +25,7 @@
<div class="input-group form-group">
<input
*ngIf="progress.state === JobProgressStates.running" type="text" class="form-control" disabled
*ngIf="progress.state === JobProgressStates.running && progress.logs.length > 0" type="text" class="form-control" disabled
[ngModel]="progress.logs[progress.logs.length-1].comment" name="details">
<input
*ngIf="progress.state === JobProgressStates.cancelling" type="text" class="form-control" disabled
@ -112,7 +112,7 @@
</div>
<div class="card-body text-secondary">
<ng-container *ngFor="let log of progress.logs; let i = index;">
<p class="card-text" *ngIf="(i==0 && log.id > 0) || ( i> 0 && progress.logs[i-1].id+1!=log.id)">
<p class="card-text" *ngIf="(i==0 && log.id > 0) || ( i> 0 && progress.logs[i-1].id+1!=log.id)">
...
</p>
<p class="card-text">

2
src/frontend/app/ui/settings/photo/photo.settings.component.html
View File

@ -100,7 +100,7 @@
</button>
<div [hidden]="!settings.client.Converting.enabled">
<app-settings-job-button class="mt-2 mt-md-2 float-left"
<app-settings-job-button class="mt-2 mt-md-0 float-left"
[soloRun]="true"
(error)="error=$event"
[jobName]="jobName"></app-settings-job-button>

43
src/frontend/app/ui/settings/scheduled-jobs.service.ts
View File

@ -1,9 +1,12 @@
import {EventEmitter, Injectable} from '@angular/core';
import {BehaviorSubject} from 'rxjs';
import {JobProgressDTO} from '../../../../common/entities/job/JobProgressDTO';
import {JobProgressDTO, JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
import {NetworkService} from '../../model/network/network.service';
import {JobScheduleDTO} from '../../../../common/entities/job/JobScheduleDTO';
import {JobDTO} from '../../../../common/entities/job/JobDTO';
import {BackendtextService} from '../../model/backendtext.service';
import {NotificationService} from '../../model/notification.service';
import {I18n} from '@ngx-translate/i18n-polyfill';
@Injectable()
export class ScheduledJobsService {
@ -15,7 +18,10 @@ export class ScheduledJobsService {
public jobStartingStopping: { [key: string]: boolean } = {};
private subscribers = 0;
constructor(private _networkService: NetworkService) {
constructor(private _networkService: NetworkService,
private notification: NotificationService,
private backendTextService: BackendtextService,
private i18n: I18n) {
this.progress = new BehaviorSubject({});
}
@ -39,6 +45,8 @@ export class ScheduledJobsService {
this.jobStartingStopping[jobName] = true;
await this._networkService.postJson('/admin/jobs/scheduled/' + jobName + '/' + (soloStart === true ? 'soloStart' : 'start'),
{config: config});
// placeholder to force showing running job
this.addDummyProgress(jobName, config);
delete this.jobStartingStopping[jobName];
this.forceUpdate();
}
@ -53,14 +61,20 @@ export class ScheduledJobsService {
protected async loadProgress(): Promise<void> {
const prevPrg = this.progress.value;
this.progress.next(await this._networkService.getJson<{ [key: string]: JobProgressDTO }>('/admin/jobs/scheduled/progress'));
for (const prg in prevPrg) {
if (!this.progress.value.hasOwnProperty(prg)) {
for (const prg of Object.keys(prevPrg)) {
if (!this.progress.value.hasOwnProperty(prg) ||
// state changed from running to finished
((prevPrg[prg].state === JobProgressStates.running ||
prevPrg[prg].state === JobProgressStates.cancelling) &&
!(this.progress.value[prg].state === JobProgressStates.running ||
this.progress.value[prg].state === JobProgressStates.cancelling)
)) {
this.onJobFinish.emit(prg);
this.notification.info(this.i18n('Job finished') + ': ' + this.backendTextService.getJobName(prevPrg[prg].jobName));
}
}
}
protected getProgressPeriodically() {
if (this.timer != null || this.subscribers === 0) {
return;
@ -76,6 +90,25 @@ export class ScheduledJobsService {
this.loadProgress().catch(console.error);
}
private addDummyProgress(jobName: string, config: any) {
const prgs = this.progress.value;
prgs[JobDTO.getHashName(jobName, config)] = {
jobName: jobName,
state: JobProgressStates.running,
HashName: JobDTO.getHashName(jobName, config),
logs: [], steps: {
skipped: 0,
processed: 0,
all: 0
},
time: {
start: Date.now(),
end: Date.now()
}
};
this.progress.next(prgs);
}
private incSubscribers() {
this.subscribers++;
this.getProgressPeriodically();

2
src/frontend/app/ui/settings/thumbnail/thumbnail.settings.component.html
View File

@ -74,7 +74,7 @@
</button>
<app-settings-job-button class="mt-2 mt-md-2 float-left"
<app-settings-job-button class="mt-2 mt-md-0 float-left"
[soloRun]="true"
(error)="error=$event"
[jobName]="jobName"

2
src/frontend/app/ui/settings/video/video.settings.component.html
View File

@ -127,7 +127,7 @@
(click)="reset()" i18n>Reset
</button>
<app-settings-job-button class="mt-2 mt-md-2 float-left"
<app-settings-job-button class="mt-2 mt-md-0 float-left"
[soloRun]="true"
(error)="error=$event"
[jobName]="jobName"></app-settings-job-button>

12
src/frontend/translate/messages.en.xlf
View File

@ -904,9 +904,7 @@
<context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
<context context-type="linenumber">59</context>
</context-group>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
pixels.
</target>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
</trans-unit>
<trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
<source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@ -2592,6 +2590,14 @@
</context-group>
<target>Random Photo</target>
</trans-unit>
<trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
<source>Job finished</source>
<context-group purpose="location">
<context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
<context context-type="linenumber">1</context>
</context-group>
<target>Job finished</target>
</trans-unit>
<trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
<source>Thumbnail</source>
<context-group purpose="location">

12
src/frontend/translate/messages.fr.xlf
View File

@ -904,9 +904,7 @@
<context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
<context context-type="linenumber">59</context>
</context-group>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
pixels.
</target>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
</trans-unit>
<trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
<source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@ -2592,6 +2590,14 @@
</context-group>
<target>Random Photo</target>
</trans-unit>
<trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
<source>Job finished</source>
<context-group purpose="location">
<context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
<context context-type="linenumber">1</context>
</context-group>
<target>Job finished</target>
</trans-unit>
<trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
<source>Thumbnail</source>
<context-group purpose="location">

8
src/frontend/translate/messages.hu.xlf
View File

@ -2590,6 +2590,14 @@
</context-group>
<target>Véletleg Fotó</target>
</trans-unit>
<trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
<source>Job finished</source>
<context-group purpose="location">
<context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
<context context-type="linenumber">1</context>
</context-group>
<target>Feladat végzett</target>
</trans-unit>
<trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
<source>Thumbnail</source>
<context-group purpose="location">

12
src/frontend/translate/messages.ro.xlf
View File

@ -904,9 +904,7 @@
<context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
<context context-type="linenumber">59</context>
</context-group>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
pixels.
</target>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
</trans-unit>
<trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
<source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@ -2592,6 +2590,14 @@
</context-group>
<target>Random Photo</target>
</trans-unit>
<trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
<source>Job finished</source>
<context-group purpose="location">
<context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
<context context-type="linenumber">1</context>
</context-group>
<target>Job finished</target>
</trans-unit>
<trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
<source>Thumbnail</source>
<context-group purpose="location">

12
src/frontend/translate/messages.ru.xlf
View File

@ -904,9 +904,7 @@
<context context-type="sourcefile">app/ui/settings/thumbnail/thumbnail.settings.component.html</context>
<context context-type="linenumber">59</context>
</context-group>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160
pixels.
</target>
<target>';' separated integers. If size is 240, that shorter side of the thumbnail will have 160 pixels.</target>
</trans-unit>
<trans-unit id="69149fe434cf1b0d9bafe5ead5d126101bb5162e" datatype="html">
<source>Video support uses ffmpeg. ffmpeg and ffprobe binaries need to be available in the PATH or
@ -2592,6 +2590,14 @@
</context-group>
<target>Random Photo</target>
</trans-unit>
<trans-unit id="bf5ce15a43de0abbce2193b3061a208eb35f7195" datatype="html">
<source>Job finished</source>
<context-group purpose="location">
<context context-type="sourcefile">src/frontend/app/ui/settings/scheduled-jobs.service.ts</context>
<context context-type="linenumber">1</context>
</context-group>
<target>Job finished</target>
</trans-unit>
<trans-unit id="45dda89cf029b7d7b457a8dff01dc4b9a6485816" datatype="html">
<source>Thumbnail</source>
<context-group purpose="location">