1
0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00

implementing manual directory indexing

This commit is contained in:
Patrik Braun 2017-07-25 21:09:37 +02:00
parent d2fdd2fe10
commit 2569286915
20 changed files with 481 additions and 46 deletions

View File

@ -13,6 +13,7 @@ import {ProjectPath} from "../ProjectPath";
const LOG_TAG = "[AdminMWs]";
export class AdminMWs {
@ -125,7 +126,7 @@ export class AdminMWs {
}
public static async updateAuthenticationSettings(req: Request, res: Response, next: NextFunction) {
public static async updateAuthenticationSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
}
@ -145,7 +146,7 @@ export class AdminMWs {
}
}
public static async updateThumbnailSettings(req: Request, res: Response, next: NextFunction) {
public static async updateThumbnailSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
}
@ -179,7 +180,7 @@ export class AdminMWs {
}
public static async updateBasicSettings(req: Request, res: Response, next: NextFunction) {
public static async updateBasicSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
}
@ -209,7 +210,7 @@ export class AdminMWs {
}
public static async updateOtherSettings(req: Request, res: Response, next: NextFunction) {
public static async updateOtherSettings(req: Request, res: Response, next: NextFunction) {
if ((typeof req.body === 'undefined') || (typeof req.body.settings === 'undefined')) {
return next(new ErrorDTO(ErrorCodes.INPUT_ERROR, "settings is needed"));
}
@ -237,4 +238,42 @@ export class AdminMWs {
}
public static startIndexing(req: Request, res: Response, next: NextFunction) {
try {
ObjectManagerRepository.getInstance().IndexingManager.startIndexing();
req.resultPipe = "ok";
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, ' '), err));
}
}
public static getIndexingProgress(req: Request, res: Response, next: NextFunction) {
try {
req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress();
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, ' '), err));
}
}
public static cancelIndexing(req: Request, res: Response, next: NextFunction) {
try {
ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing();
req.resultPipe = "ok";
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, ' '), err));
}
}
public static async resetIndexes(req: Request, res: Response, next: NextFunction) {
try {
await ObjectManagerRepository.getInstance().IndexingManager.reset();
req.resultPipe = "ok";
return next();
} catch (err) {
return next(new ErrorDTO(ErrorCodes.SETTINGS_ERROR, "Indexing error: " + JSON.stringify(err, null, ' '), err));
}
}
}

View File

@ -12,7 +12,7 @@ import {Logger} from "../Logger";
export class RenderingMWs {
public static renderResult(req: Request, res: Response, next: NextFunction) {
if (!req.resultPipe)
if (typeof req.resultPipe == "undefined")
return next();
return RenderingMWs.renderMessage(res, req.resultPipe);

View File

@ -7,8 +7,6 @@ import {DiskMangerWorker} from "./threading/DiskMangerWorker";
const LOG_TAG = "[DiskManager]";
export class DiskManager {
static threadPool: DiskManagerTH = null;

View File

@ -4,6 +4,7 @@ import {ISearchManager} from "./interfaces/ISearchManager";
import {SQLConnection} from "./sql/SQLConnection";
import {ISharingManager} from "./interfaces/ISharingManager";
import {Logger} from "../Logger";
import {IIndexingManager} from "./interfaces/IIndexingManager";
export class ObjectManagerRepository {
@ -11,33 +12,15 @@ export class ObjectManagerRepository {
private _userManager: IUserManager;
private _searchManager: ISearchManager;
private _sharingManager: ISharingManager;
private _indexingManager: IIndexingManager;
private static _instance: ObjectManagerRepository = null;
public static async InitMemoryManagers() {
await ObjectManagerRepository.reset();
const GalleryManager = require("./memory/GalleryManager").GalleryManager;
const UserManager = require("./memory/UserManager").UserManager;
const SearchManager = require("./memory/SearchManager").SearchManager;
const SharingManager = require("./memory/SharingManager").SharingManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
get IndexingManager(): IIndexingManager {
return this._indexingManager;
}
public static async InitSQLManagers() {
await ObjectManagerRepository.reset();
await SQLConnection.init();
const GalleryManager = require("./sql/GalleryManager").GalleryManager;
const UserManager = require("./sql/UserManager").UserManager;
const SearchManager = require("./sql/SearchManager").SearchManager;
const SharingManager = require("./sql/SharingManager").SharingManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
Logger.debug("SQL DB inited");
set IndexingManager(value: IIndexingManager) {
this._indexingManager = value;
}
public static getInstance() {
@ -85,4 +68,34 @@ export class ObjectManagerRepository {
this._sharingManager = value;
}
public static async InitMemoryManagers() {
await ObjectManagerRepository.reset();
const GalleryManager = require("./memory/GalleryManager").GalleryManager;
const UserManager = require("./memory/UserManager").UserManager;
const SearchManager = require("./memory/SearchManager").SearchManager;
const SharingManager = require("./memory/SharingManager").SharingManager;
const IndexingManager = require("./memory/IndexingManager").IndexingManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
}
public static async InitSQLManagers() {
await ObjectManagerRepository.reset();
await SQLConnection.init();
const GalleryManager = require("./sql/GalleryManager").GalleryManager;
const UserManager = require("./sql/UserManager").UserManager;
const SearchManager = require("./sql/SearchManager").SearchManager;
const SharingManager = require("./sql/SharingManager").SharingManager;
const IndexingManager = require("./sql/IndexingManager").IndexingManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
Logger.debug("SQL DB inited");
}
}

View File

@ -4,4 +4,5 @@ export interface IGalleryManager {
listDirectory(relativeDirectoryName: string,
knownLastModified?: number,
knownLastScanned?: number): Promise<DirectoryDTO>;
}

View File

@ -0,0 +1,11 @@
import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
export interface IIndexingManager {
startIndexing(): void;
getProgress(): IndexingProgressDTO;
cancelIndexing(): void;
reset(): Promise<void> ;
}

View File

@ -0,0 +1,21 @@
import {IIndexingManager} from "../interfaces/IIndexingManager";
import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
export class IndexingManager implements IIndexingManager {
startIndexing(): void {
throw new Error("not supported by memory DB");
}
getProgress(): IndexingProgressDTO {
throw new Error("not supported by memory DB");
}
cancelIndexing(): void {
throw new Error("not supported by memory DB");
}
reset(): Promise<void> {
throw new Error("Method not implemented.");
}
}

View File

@ -9,8 +9,9 @@ import {PhotoEntity, PhotoMetadataEntity} from "./enitites/PhotoEntity";
import {Utils} from "../../../common/Utils";
import {ProjectPath} from "../../ProjectPath";
import {Config} from "../../../common/config/private/Config";
import {ISQLGalleryManager} from "./IGalleryManager";
export class GalleryManager implements IGalleryManager {
export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
public async listDirectory(relativeDirectoryName: string,

View File

@ -0,0 +1,10 @@
import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
export interface ISQLGalleryManager {
listDirectory(relativeDirectoryName: string,
knownLastModified?: number,
knownLastScanned?: number): Promise<DirectoryDTO>;
indexDirectory(relativeDirectoryName): Promise<DirectoryDTO>;
}

View File

@ -0,0 +1,80 @@
import {IIndexingManager} from "../interfaces/IIndexingManager";
import {IndexingProgressDTO} from "../../../common/entities/settings/IndexingProgressDTO";
import {ObjectManagerRepository} from "../ObjectManagerRepository";
import {ISQLGalleryManager} from "./IGalleryManager";
import * as path from "path";
import {SQLConnection} from "./SQLConnection";
import {DirectoryEntity} from "./enitites/DirectoryEntity";
import {Logger} from "../../Logger";
const LOG_TAG = "[IndexingManager]";
export class IndexingManager implements IIndexingManager {
directoriesToIndex: string[] = [];
indexingProgress = null;
enabled = false;
private indexNewDirectory = async () => {
if (this.directoriesToIndex.length == 0) {
this.indexingProgress = null;
if (global.gc) {
global.gc();
}
return;
}
const directory = this.directoriesToIndex.shift();
this.indexingProgress.current = directory;
this.indexingProgress.left = this.directoriesToIndex.length;
const scanned = await (<ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory);
if (this.enabled == false) {
return;
}
this.indexingProgress.indexed++;
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name))
}
process.nextTick(this.indexNewDirectory);
};
startIndexing(): void {
if (this.directoriesToIndex.length == 0 && this.enabled == false) {
Logger.info(LOG_TAG, "Starting indexing");
this.indexingProgress = <IndexingProgressDTO>{
indexed: 0,
left: 0,
current: ""
};
this.directoriesToIndex.push("/");
this.enabled = true;
this.indexNewDirectory();
} else {
Logger.info(LOG_TAG, "Already indexing..");
}
}
getProgress(): IndexingProgressDTO {
return this.indexingProgress;
}
cancelIndexing(): void {
Logger.info(LOG_TAG, "Canceling indexing");
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
if (global.gc) {
global.gc();
}
}
async reset(): Promise<void> {
Logger.info(LOG_TAG, "Resetting DB");
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
const connection = await SQLConnection.getConnection();
return connection
.getRepository(DirectoryEntity)
.createQueryBuilder("directory")
.delete()
.execute();
}
}

View File

@ -155,7 +155,7 @@ export class DiskMangerWorker {
metadata.creationDate = <number> (iptcData.date_time ? iptcData.date_time.getTime() : metadata.creationDate);
} catch (err) {
Logger.debug(LOG_TAG, "Error parsing iptc data", fullPath, err);
// Logger.debug(LOG_TAG, "Error parsing iptc data", fullPath, err);
}
metadata.creationDate = metadata.creationDate || 0;

View File

@ -6,24 +6,35 @@ import {AdminMWs} from "../middlewares/AdminMWs";
export class AdminRouter {
public static route(app: any) {
this.addResetDB(app);
this.addIndexGallery(app);
this.addSettings(app);
}
private static addResetDB(app) {
app.post("/api/admin/db/reset",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin)
//TODO: implement
);
};
private static addIndexGallery(app) {
app.post("/api/admin/gallery/index",
app.get("/api/admin/indexes/job/progress",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin)
//TODO: implement
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.getIndexingProgress,
RenderingMWs.renderResult
);
app.put("/api/admin/indexes/job",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.startIndexing,
RenderingMWs.renderResult
);
app.delete("/api/admin/indexes/job",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.cancelIndexing,
RenderingMWs.renderResult
);
app.delete("/api/admin/indexes",
AuthenticationMWs.authenticate,
AuthenticationMWs.authorise(UserRoles.Admin),
AdminMWs.resetIndexes,
RenderingMWs.renderResult
);
};

View File

@ -0,0 +1,5 @@
export interface IndexingProgressDTO {
indexed: number;
left: number;
current: string;
}

View File

@ -50,5 +50,7 @@
<settings-share #share [hidden]="!share.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-share>
<settings-map #map [hidden]="!map.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-map>
<settings-other #other [hidden]="!other.hasAvailableSettings" [simplifiedMode]="simplifiedMode"></settings-other>
<settings-indexing #other [hidden]="!other.hasAvailableSettings"
[simplifiedMode]="simplifiedMode"></settings-indexing>
</div>
</app-frame>

View File

@ -54,6 +54,7 @@ import {ShareSettingsComponent} from "./settings/share/share.settings.component"
import {BasicSettingsComponent} from "./settings/basic/basic.settings.component";
import {OtherSettingsComponent} from "./settings/other/other.settings.component";
import {DefaultUrlSerializer, UrlSerializer, UrlTree} from '@angular/router';
import {IndexingSettingsComponent} from "./settings/indexing/indexing.settings.component";
@Injectable()
export class GoogleMapsConfig {
@ -129,6 +130,7 @@ export class CustomUrlSerializer implements UrlSerializer {
ShareSettingsComponent,
BasicSettingsComponent,
OtherSettingsComponent,
IndexingSettingsComponent,
StringifyRole],
providers: [
{provide: UrlSerializer, useClass: CustomUrlSerializer},

View File

@ -34,9 +34,11 @@ export abstract class SettingsComponent<T> implements OnInit, OnDestroy, OnChang
private _navigation: NavigationService,
public _settingsService: AbstractSettingsService<T>,
protected notification: NotificationService,
private sliceFN: (s: IPrivateConfig) => T) {
this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
this.onNewSettings(this._settingsService._settingsService.settings.value);
private sliceFN?: (s: IPrivateConfig) => T) {
if (this.sliceFN) {
this.settingsSubscription = this._settingsService.Settings.subscribe(this.onNewSettings);
this.onNewSettings(this._settingsService._settingsService.settings.value);
}
}
onNewSettings = (s) => {

View File

@ -0,0 +1,3 @@
.buttons {
text-align: center;
}

View File

@ -0,0 +1,47 @@
<form #settingsForm="ngForm" class="form-horizontal">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title col-xs-4">Folder indexing</h3>
</div>
<div class="panel-body">
<div [hidden]="!error" class="alert alert-danger" role="alert"><strong>Error: </strong>{{error}}</div>
If you add a new folder to your gallery, the site indexes it automatically.
If you would like to trigger indexing manually, click index button.<br/>
(Note: search ony searched among the indexed directories)<br/>
<div *ngIf="_settingsService.progress.value != null">
indexing: {{_settingsService.progress.value.current}}
<div class="progress">
<div class="progress-bar progress-bar-success"
role="progressbar"
aria-valuenow="2"
aria-valuemin="0"
aria-valuemax="100"
style="min-width: 2em;"
[style.width.%]="(_settingsService.progress.value.indexed/(_settingsService.progress.value.left+_settingsService.progress.value.indexed))*100">
{{_settingsService.progress.value.indexed}}/{{_settingsService.progress.value.indexed+_settingsService.progress.value.left}}
</div>
</div>
</div>
<div class="row buttons">
<button class="btn btn-success"
*ngIf="_settingsService.progress.value == null"
[disabled]="inProgress"
(click)="index()">Index
</button>
<button class="btn btn-default"
*ngIf="_settingsService.progress.value != null"
[disabled]="inProgress"
(click)="cancel()">Cancel
</button>
<button class="btn btn-danger"
[disabled]="inProgress"
(click)="reset()">Reset Indexes
</button>
</div>
</div>
</div>
</form>

View File

@ -0,0 +1,144 @@
import {Component, Input, OnDestroy, OnInit, Output} from "@angular/core";
import {IndexingSettingsService} from "./indexing.settings.service";
import {AuthenticationService} from "../../model/network/authentication.service";
import {NavigationService} from "../../model/navigation.service";
import {NotificationService} from "../../model/notification.service";
import {ErrorDTO} from "../../../../common/entities/Error";
import {UserRoles} from "../../../../common/entities/UserDTO";
import {Observable} from "rxjs/Rx";
@Component({
selector: 'settings-indexing',
templateUrl: './indexing.settings.component.html',
styleUrls: ['./indexing.settings.component.css',
'./../_abstract/abstract.settings.component.css'],
providers: [IndexingSettingsService],
})
export class IndexingSettingsComponent implements OnInit, OnDestroy {
@Input()
public simplifiedMode: boolean = true;
@Output('hasAvailableSettings')
hasAvailableSettings: boolean = true;
public inProgress = false;
public error: string = null;
updateProgress = async () => {
try {
await this._settingsService.getProgress();
} catch (err) {
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
}
if (this._settingsService.progress.value != null && this.subscription.timer == null) {
if (!this.$counter) {
this.$counter = Observable.interval(5000);
}
this.subscription.timer = this.$counter.subscribe((x) => this.updateProgress());
}
if (this._settingsService.progress.value == null && this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
};
private subscription: { timer: any, settings: any } = {
timer: null,
settings: null
};
private $counter: Observable<number> = null;
constructor(private _authService: AuthenticationService,
private _navigation: NavigationService,
public _settingsService: IndexingSettingsService,
private notification: NotificationService) {
}
async ngOnInit() {
if (!this._authService.isAuthenticated() ||
this._authService.user.value.role < UserRoles.Admin) {
this._navigation.toLogin();
return;
}
this.subscription.settings = this._settingsService.Settings.subscribe(() => {
this.hasAvailableSettings = (this._settingsService.isSupported() || !this.simplifiedMode);
});
this.updateProgress();
}
ngOnDestroy() {
if (this.subscription.timer != null) {
this.subscription.timer.unsubscribe();
this.subscription.timer = null;
}
if (this.subscription.settings != null) {
this.subscription.settings.unsubscribe();
this.subscription.settings = null;
}
}
async index() {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.index();
this.updateProgress();
this.notification.success("Folder indexed", "Success");
this.inProgress = false;
return true;
} catch (err) {
console.log(err);
if (err.message) {
this.error = (<ErrorDTO>err).message;
}
}
this.inProgress = false;
return false;
}
async cancel() {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.cancel();
this.notification.success("Folder indexed", "Success");
this.inProgress = false;
return true;
} catch (err) {
console.log(err);
if (err.message) {
this.error = (<ErrorDTO>err).message;
}
}
this.inProgress = false;
return false;
}
async reset() {
this.inProgress = true;
this.error = "";
try {
await this._settingsService.reset();
this.notification.success('Database reset', "Success");
this.inProgress = false;
return true;
} catch (err) {
console.log(err);
if (err.message) {
this.error = (<ErrorDTO>err).message;
}
}
this.inProgress = false;
return false;
}
}

View File

@ -0,0 +1,45 @@
import {Injectable} from "@angular/core";
import {NetworkService} from "../../model/network/network.service";
import {SettingsService} from "../settings.service";
import {AbstractSettingsService} from "../_abstract/abstract.settings.service";
import {DatabaseType} from "../../../../common/config/private/IPrivateConfig";
import {IndexingProgressDTO} from "../../../../common/entities/settings/IndexingProgressDTO";
import {BehaviorSubject} from "rxjs/BehaviorSubject";
@Injectable()
export class IndexingSettingsService extends AbstractSettingsService<void> {
public progress: BehaviorSubject<IndexingProgressDTO>;
constructor(private _networkService: NetworkService,
_settingsService: SettingsService) {
super(_settingsService);
this.progress = new BehaviorSubject(null);
}
public isSupported(): boolean {
return this._settingsService.settings.value.Server.database.type != DatabaseType.memory;
}
public index() {
return this._networkService.putJson("/admin/indexes/job");
}
public cancel() {
return this._networkService.deleteJson("/admin/indexes/job");
}
public async getProgress() {
this.progress.next(await this._networkService.getJson<IndexingProgressDTO>("/admin/indexes/job/progress"));
}
public reset() {
return this._networkService.deleteJson("/admin/indexes");
}
updateSettings(settings: void): Promise<void> {
throw new Error("Method not implemented.");
}
}