xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00
Code Actions Projects Activity

improving indexing

Browse Source
This commit is contained in:
Patrik Braun 2017-07-27 23:10:16 +02:00
parent 6cd5056292
commit 1d7bc3c489
12 changed files with 193 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
backend/middlewares/AdminMWs.ts
View File

@ -3,7 +3,12 @@ import {ErrorCodes, ErrorDTO} from "../../common/entities/Error";
import {ObjectManagerRepository} from "../model/ObjectManagerRepository";
import {Logger} from "../Logger";
import {SQLConnection} from "../model/sql/SQLConnection";
import {DataBaseConfig, DatabaseType, ThumbnailConfig} from "../../common/config/private/IPrivateConfig";
import {
DataBaseConfig,
DatabaseType,
IndexingConfig,
ThumbnailConfig
} from "../../common/config/private/IPrivateConfig";
import {Config} from "../../common/config/private/Config";
import {ConfigDiagnostics} from "../model/ConfigDiagnostics";
import {ClientConfig} from "../../common/config/public/ConfigClass";
@ -237,6 +242,28 @@ export class AdminMWs {
}
}
public static async updateIndexingSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
}
try {
const settings: IndexingConfig = req.body.settings;
Config.Server.indexing = settings;
//only updating explicitly set config (not saving config set by the diagnostics)
const original = Config.original();
original.Server.indexing = settings;
original.save();
await ConfigDiagnostics.runDiagnostics();
Logger.info(LOG_TAG, "new config:");
Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t'));
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Settings error: " + JSON.stringify(err, null, ' '), err));
}
}
public static startIndexing(req: Request, res: Response, next: NextFunction) {
try {

5
backend/model/memory/GalleryManager.ts
View File

@ -5,6 +5,7 @@ import * as fs from "fs";
import {DiskManager} from "../DiskManger";
import {ProjectPath} from "../../ProjectPath";
import {Config} from "../../../common/config/private/Config";
import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig";
export class GalleryManager implements IGalleryManager {
@ -13,7 +14,9 @@ export class GalleryManager implements IGalleryManager {
if (knownLastModified && knownLastScanned) {
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
const lastModified = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout && lastModified == knownLastModified) {
if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout &&
lastModified == knownLastModified &&
Config.Server.indexing.reIndexingSensitivity < ReIndexingSensitivity.high) {
return Promise.resolve(null);
}
}

20
backend/model/sql/GalleryManager.ts
View File

@ -10,6 +10,7 @@ import {Utils} from "../../../common/Utils";
import {ProjectPath} from "../../ProjectPath";
import {Config} from "../../../common/config/private/Config";
import {ISQLGalleryManager} from "./IGalleryManager";
import {ReIndexingSensitivity} from "../../../common/config/private/IPrivateConfig";
export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
@ -37,10 +38,15 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
if (dir && dir.scanned == true) {
//iF it seems that the content did not changed, do not work on it
if (knownLastModified && knownLastScanned) {
if (Date.now() - knownLastScanned <= Config.Server.cachedFolderTimeout &&
lastModified == knownLastModified &&
dir.lastScanned == knownLastScanned) {
if (knownLastModified && knownLastScanned
&& lastModified == knownLastModified &&
dir.lastScanned == knownLastScanned) {
if (Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.low) {
return null;
}
if (Date.now() - knownLastScanned <= Config.Server.indexing.cachedFolderTimeout &&
Config.Server.indexing.reIndexingSensitivity == ReIndexingSensitivity.medium) {
return null;
}
}
@ -61,7 +67,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
dir: dir.directories[i].id
})
.orderBy("photo.metadata.creationDate", "ASC")
.setLimit(Config.Server.folderPreviewSize)
.setLimit(Config.Server.indexing.folderPreviewSize)
.getMany();
dir.directories[i].isPartial = true;
@ -79,7 +85,9 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
return this.indexDirectory(relativeDirectoryName);
}
if (Date.now() - dir.lastScanned > Config.Server.cachedFolderTimeout) {
if ((Date.now() - dir.lastScanned > Config.Server.indexing.cachedFolderTimeout &&
Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.medium) ||
Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.high) {
//on the fly reindexing
this.indexDirectory(relativeDirectoryName).catch((err) => {
console.error(err);

2
backend/model/threading/DiskMangerWorker.ts
View File

@ -55,7 +55,7 @@ export class DiskMangerWorker {
const fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
const d = await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
Config.Server.folderPreviewSize, true
Config.Server.indexing.folderPreviewSize, true
);
d.lastScanned = 0; //it was not a fully scan
d.isPartial = true;

6
backend/routes/AdminRouter.ts
View File

@ -96,6 +96,12 @@ export class AdminRouter {
AdminMWs.updateOtherSettings,
RenderingMWs.renderOK
);
app.put("/api/settings/indexing",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.updateIndexingSettings,
RenderingMWs.renderOK
);
};

13
common/config/private/IPrivateConfig.ts
View File

@ -41,6 +41,16 @@ export interface SharingConfig {
updateTimeout: number;
}
export enum ReIndexingSensitivity {
low, medium, high
}
export interface IndexingConfig {
folderPreviewSize: number;
cachedFolderTimeout: number;//Do not rescans the folder if seems ok
reIndexingSensitivity: ReIndexingSensitivity;
}
export interface ServerConfig {
port: number;
imagesFolder: string;
@ -49,8 +59,7 @@ export interface ServerConfig {
enableThreading: boolean;
sharing: SharingConfig;
sessionTimeout: number
folderPreviewSize: number;
cachedFolderTimeout: number;//Do not rescans the folder if seems ok
indexing: IndexingConfig;
}
export interface IPrivateConfig {

17
common/config/private/PrivateConfigClass.ts
View File

@ -1,5 +1,11 @@
import {PublicConfigClass} from "../public/ConfigClass";
import {DatabaseType, IPrivateConfig, ServerConfig, ThumbnailProcessingLib} from "./IPrivateConfig";
import {
DatabaseType,
IPrivateConfig,
ReIndexingSensitivity,
ServerConfig,
ThumbnailProcessingLib
} from "./IPrivateConfig";
import * as path from "path";
import {ConfigLoader} from "typeconfig";
@ -33,9 +39,12 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
sharing: {
updateTimeout: 1000 * 60 * 5
},
enableThreading: true,
folderPreviewSize: 2,
cachedFolderTimeout: 1000 * 60 * 60
indexing: {
folderPreviewSize: 2,
cachedFolderTimeout: 1000 * 60 * 60,
reIndexingSensitivity: ReIndexingSensitivity.high
},
enableThreading: true
};
private ConfigLoader: any;

16
frontend/app/settings/_abstract/abstract.settings.component.ts
View File

@ -23,8 +23,8 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
public inProgress = false;
public error: string = null;
public changed: boolean = false;
private subscription = null;
private settingsSubscription = null;
private _subscription = null;
private _settingsSubscription = null;
public settings: T = <any>{};
public original: T = <any>{};
@ -36,7 +36,7 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
protected notification: NotificationService,
private sliceFN?: (s: IPrivateConfig) => T) {
if (this.sliceFN) {
this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
this._settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
this.onNewSettings(this._settingsService._settingsService.settings.value);
}
}
@ -55,7 +55,7 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
}
this.getSettings();
this.subscription = this.form.valueChanges.subscribe((data) => {
this._subscription = this.form.valueChanges.subscribe((data) => {
this.changed = !Utils.equalsFilter(this.settings, this.original);
});
@ -67,11 +67,11 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
ngOnDestroy() {
if (this.subscription != null) {
this.subscription.unsubscribe();
if (this._subscription != null) {
this._subscription.unsubscribe();
}
if (this.settingsSubscription != null) {
this.settingsSubscription.unsubscribe();
if (this._settingsSubscription != null) {
this._settingsSubscription.unsubscribe();
}
}

54
frontend/app/settings/indexing/indexing.settings.component.html
View File

@ -6,6 +6,56 @@
<div class="panel-body">
<div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
<ng-container *ngIf="!simplifiedMode">
<div class="form-group">
<label class="col-sm-2 control-label" for="cachedFolderTimeout">Index cache timeout [ms]</label>
<div class="col-sm-10">
<input type="number" class="form-control" placeholder="36000"
id="cachedFolderTimeout"
min="0"
step="1"
[(ngModel)]="settings.cachedFolderTimeout"
name="cachedFolderTimeout" required>
<span class="help-block">If there was no indexing in this time, it reindexes. (skipped if indexen in DB and sensitivity is low)</span>
</div>
</div>
<div class="form-group">
<label class="col-sm-2 control-label" for="folderPreviewSize">Sub folder preview size</label>
<div class="col-sm-10">
<input type="number" class="form-control" placeholder="1"
id="folderPreviewSize"
min="0"
step="1"
[(ngModel)]="settings.folderPreviewSize"
name="folderPreviewSize" required>
<span class="help-block">Reads this many photos from sub folders</span>
</div>
</div>
<div class="form-group">
<label class="col-sm-2 control-label" for="reIndexingSensitivity">Folder reindexing sensitivity</label>
<div class="col-sm-10">
<select id="reIndexingSensitivity" class="form-control" [(ngModel)]="settings.reIndexingSensitivity"
name="reIndexingSensitivity" required>
<option *ngFor="let type of types" [ngValue]="type.key">{{type.value}}
</option>
</select>
<span
class="help-block">Set the reindexing sensitivity. High value check the folders for change more often</span>
</div>
</div>
<button class="btn btn-success pull-right"
[disabled]="!settingsForm.form.valid || !changed || inProgress"
(click)="save()">Save
</button>
<button class="btn btn-default pull-right"
(click)="reset()">Reset
</button>
<br/>
<hr/>
</ng-container>
If you add a new folder to your gallery, the site indexes it automatically.
If you would like to trigger indexing manually, click index button.<br/>
(Note: search ony searched among the indexed directories)<br/>
@ -34,11 +84,11 @@
<button class="btn btn-default"
*ngIf="_settingsService.progress.value != null"
[disabled]="inProgress"
(click)="cancel()">Cancel
(click)="cancelIndexing()">Cancel
</button>
<button class="btn btn-danger"
[disabled]="inProgress"
(click)="reset()">Reset Indexes
(click)="resetDatabase()">Reset Indexes
</button>
</div>
</div>

66
frontend/app/settings/indexing/indexing.settings.component.ts
View File

@ -1,11 +1,13 @@
import {Component, Input, OnDestroy, OnInit, Output} from "@angular/core";
import {Component} from "@angular/core";
import {IndexingSettingsService} from "./indexing.settings.service";
import {AuthenticationService} from "../../model/network/authentication.service";
import {NavigationService} from "../../model/navigation.service";
import {NotificationService} from "../../model/notification.service";
import {ErrorDTO} from "../../../../common/entities/Error";
import {UserRoles} from "../../../../common/entities/UserDTO";
import {Observable} from "rxjs/Rx";
import {IndexingConfig, ReIndexingSensitivity} from "../../../../common/config/private/IPrivateConfig";
import {SettingsComponent} from "../_abstract/abstract.settings.component";
import {Utils} from "../../../../common/Utils";
@Component({
selector: 'settings-indexing',
@ -14,61 +16,55 @@ import {Observable} from "rxjs/Rx";
'./../_abstract/abstract.settings.component.css'],
providers: [IndexingSettingsService],
})
export class IndexingSettingsComponent implements OnInit, OnDestroy {
export class IndexingSettingsComponent extends SettingsComponent<IndexingConfig> {
@Input()
public simplifiedMode: boolean = true;
@Output('hasAvailableSettings')
hasAvailableSettings: boolean = true;
public inProgress = false;
public error: string = null;
types: Array<any> = [];
private subscription: { timer: any, settings: any } = {
timer: null,
settings: null
};
private $counter: Observable<number> = null;
updateProgress = async () => {
try {
await this._settingsService.getProgress();
await (<IndexingSettingsService>this._settingsService).getProgress();
} catch (err) {
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
}
if (this._settingsService.progress.value != null && this.subscription.timer == null) {
if ((<IndexingSettingsService>this._settingsService).progress.value != null && this.subscription.timer == null) {
if (!this.$counter) {
this.$counter = Observable.interval(5000);
}
this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress());
}
if (this._settingsService.progress.value == null && this.subscription.timer != null) {
if ((<IndexingSettingsService>this._settingsService).progress.value == null && this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
};
private subscription: { timer: any, settings: any } = {
timer: null,
settings: null
};
private $counter: Observable<number> = null;
constructor(private _authService: AuthenticationService,
private _navigation: NavigationService,
public _settingsService: IndexingSettingsService,
private notification: NotificationService) {
constructor(_authService: AuthenticationService,
_navigation: NavigationService,
_settingsService: IndexingSettingsService,
notification: NotificationService) {
super("Indexing", _authService, _navigation, <any>_settingsService, notification, s => s.Server.indexing);
}
async ngOnInit() {
if (!this._authService.isAuthenticated() ||
this._authService.user.value.role < UserRoles.Admin) {
this._navigation.toLogin();
return;
}
this.subscription.settings = this._settingsService.Settings.subscribe(() => {
this.hasAvailableSettings = (this._settingsService.isSupported() || !this.simplifiedMode);
});
super.ngOnInit();
this.types = Utils
.enumToArray(ReIndexingSensitivity);
this.updateProgress();
}
ngOnDestroy() {
super.ngOnDestroy();
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
@ -83,7 +79,7 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.index();
await (<IndexingSettingsService>this._settingsService).index();
this.updateProgress();
this.notification.success("Folder indexed", "Success");
this.inProgress = false;
@ -99,11 +95,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
return false;
}
async cancel() {
async cancelIndexing() {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.cancel();
await (<IndexingSettingsService>this._settingsService).cancel();
this.notification.success("Folder indexed", "Success");
this.inProgress = false;
return true;
@ -118,11 +114,11 @@ export class IndexingSettingsComponent implements OnInit, OnDestroy {
return false;
}
async reset() {
async resetDatabase() {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.reset();
await (<IndexingSettingsService>this._settingsService).reset();
this.notification.success('Database reset', "Success");
this.inProgress = false;
return true;

12
frontend/app/settings/indexing/indexing.settings.service.ts
View File

@ -2,12 +2,12 @@ import {Injectable} from "@angular/core";
import {NetworkService} from "../../model/network/network.service";
import {SettingsService} from "../settings.service";
import {AbstractSettingsService} from "../_abstract/abstract.settings.service";
import {DatabaseType} from "../../../../common/config/private/IPrivateConfig";
import {DatabaseType, IndexingConfig} from "../../../../common/config/private/IPrivateConfig";
import {IndexingProgressDTO} from "../../../../common/entities/settings/IndexingProgressDTO";
import {BehaviorSubject} from "rxjs/BehaviorSubject";
@Injectable()
export class IndexingSettingsService extends AbstractSettingsService<void> {
export class IndexingSettingsService extends AbstractSettingsService<IndexingConfig> {
public progress: BehaviorSubject<IndexingProgressDTO>;
@ -18,6 +18,11 @@ export class IndexingSettingsService extends AbstractSettingsService<void> {
this.progress = new BehaviorSubject(null);
}
public updateSettings(settings: IndexingConfig): Promise<void> {
return this._networkService.putJson("/settings/indexing", {settings: settings});
}
public isSupported(): boolean {
return this._settingsService.settings.value.Server.database.type != DatabaseType.memory;
}
@ -39,7 +44,4 @@ export class IndexingSettingsService extends AbstractSettingsService<void> {
}
updateSettings(settings: void): Promise<void> {
throw new Error("Method not implemented.");
}
}

22
frontend/app/settings/settings.service.ts
View File

@ -1,19 +1,27 @@
import {Injectable} from "@angular/core";
import {BehaviorSubject} from "rxjs/BehaviorSubject";
import {DatabaseType, IPrivateConfig, ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
import {
DatabaseType,
IPrivateConfig,
ReIndexingSensitivity,
ThumbnailProcessingLib
} from "../../../common/config/private/IPrivateConfig";
import {NetworkService} from "../model/network/network.service";
@Injectable()
export class SettingsService {
public settings: BehaviorSubject<IPrivateConfig>;
constructor(private _networkService: NetworkService) {
this.settings = new BehaviorSubject<IPrivateConfig>(<any>{
this.settings = new BehaviorSubject<IPrivateConfig>({
Client: {
Search: {
enabled: true,
autocompleteEnabled: true,
instantSearchEnabled: true
instantSearchEnabled: true,
InstantSearchTimeout: 0
},
concurrentThumbnailGenerations: null,
Thumbnail: {
iconSize: 30,
thumbnailSizes: []
@ -47,12 +55,18 @@ export class SettingsService {
folder: "",
qualityPriority: true,
processingLibrary: ThumbnailProcessingLib.sharp
},
sessionTimeout: 0,
indexing: {
cachedFolderTimeout: 0,
folderPreviewSize: 0,
reIndexingSensitivity: ReIndexingSensitivity.medium
}
}
});
}
public async getSettings(): Promise<void> {
public async getSettings(): Promise<void> {
this.settings.next(await <Promise<IPrivateConfig>>this._networkService.getJson("/settings"));
}