From 0f4eb10c9153c80f087207d49615ea049eca571d Mon Sep 17 00:00:00 2001 From: Braun Patrik Date: Mon, 17 Jul 2017 23:12:12 +0200 Subject: [PATCH] fixing database index bugs --- backend/middlewares/GalleryMWs.ts | 8 +- backend/model/mysql/GalleryManager.ts | 104 ++++++++++++------ backend/model/mysql/MySQLConnection.ts | 18 +-- .../model/mysql/enitites/DirectoryEntity.ts | 2 +- backend/model/mysql/enitites/PhotoEntity.ts | 18 ++- backend/model/threading/DiskMangerWorker.ts | 8 +- backend/routes/ErrorRouter.ts | 3 +- common/config/private/IPrivateConfig.ts | 1 + common/config/private/PrivateConfigClass.ts | 3 +- 9 files changed, 104 insertions(+), 61 deletions(-) diff --git a/backend/middlewares/GalleryMWs.ts b/backend/middlewares/GalleryMWs.ts index cbd5c7bd..96d3ebd4 100644 --- a/backend/middlewares/GalleryMWs.ts +++ b/backend/middlewares/GalleryMWs.ts @@ -74,14 +74,16 @@ export class GalleryMWs { } let fullImagePath = path.join(ProjectPath.ImageFolder, req.params.imagePath); - if (fs.statSync(fullImagePath).isDirectory()) { - return next(); - } //check if thumbnail already exist if (fs.existsSync(fullImagePath) === false) { return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, "no such file:" + fullImagePath)); } + if (fs.statSync(fullImagePath).isDirectory()) { + return next(); + } + + req.resultPipe = fullImagePath; return next(); diff --git a/backend/model/mysql/GalleryManager.ts b/backend/model/mysql/GalleryManager.ts index 61642c8a..b992130c 100644 --- a/backend/model/mysql/GalleryManager.ts +++ b/backend/model/mysql/GalleryManager.ts @@ -1,12 +1,14 @@ import {IGalleryManager} from "../interfaces/IGalleryManager"; import {DirectoryDTO} from "../../../common/entities/DirectoryDTO"; import * as path from "path"; +import * as fs from "fs"; import {DirectoryEntity} from "./enitites/DirectoryEntity"; import {MySQLConnection} from "./MySQLConnection"; import {DiskManager} from "../DiskManger"; -import {PhotoEntity} from "./enitites/PhotoEntity"; +import {PhotoEntity, PhotoMetadataEntity} from "./enitites/PhotoEntity"; import {Utils} from "../../../common/Utils"; import {ProjectPath} from "../../ProjectPath"; +import {Config} from "../../../common/config/private/Config"; export class GalleryManager implements IGalleryManager { @@ -15,8 +17,6 @@ export class GalleryManager implements IGalleryManager { relativeDirectoryName = path.normalize(path.join("." + path.sep, relativeDirectoryName)); const directoryName = path.basename(relativeDirectoryName); const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); - console.log("GalleryManager:listDirectory"); - console.log(directoryName, directoryParent, path.dirname(relativeDirectoryName), ProjectPath.normalizeRelative(path.dirname(relativeDirectoryName))); const connection = await MySQLConnection.getConnection(); let dir = await connection .getRepository(DirectoryEntity) @@ -33,18 +33,41 @@ export class GalleryManager implements IGalleryManager { if (dir.photos) { for (let i = 0; i < dir.photos.length; i++) { dir.photos[i].directory = dir; - dir.photos[i].metadata.keywords = JSON.parse(dir.photos[i].metadata.keywords); - dir.photos[i].metadata.cameraData = JSON.parse(dir.photos[i].metadata.cameraData); - dir.photos[i].metadata.positionData = JSON.parse(dir.photos[i].metadata.positionData); - dir.photos[i].metadata.size = JSON.parse(dir.photos[i].metadata.size); + PhotoMetadataEntity.open(dir.photos[i].metadata); dir.photos[i].readyThumbnails = []; dir.photos[i].readyIcon = false; } } + if (dir.directories) { + for (let i = 0; i < dir.directories.length; i++) { + dir.directories[i].photos = await connection + .getRepository(PhotoEntity) + .createQueryBuilder("photo") + .where("photo.directory = :dir", { + dir: dir.directories[i].id + }) + .setLimit(Config.Server.folderPreviewSize) + .getMany(); + for (let j = 0; j < dir.directories[i].photos.length; j++) { + dir.directories[i].photos[j].directory = dir.directories[i]; + PhotoMetadataEntity.open(dir.directories[i].photos[j].metadata); + dir.directories[i].photos[j].readyThumbnails = []; + dir.directories[i].photos[j].readyIcon = false; + } + } + } + + const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); + const lastUpdate = Math.max(stat.ctime.getTime(), stat.mtime.getTime()); + + if (dir.lastUpdate != lastUpdate) { + return this.indexDirectory(relativeDirectoryName); + } + + console.log("lazy"); //on the fly updating this.indexDirectory(relativeDirectoryName).catch((err) => { - console.error(err); }); @@ -57,12 +80,10 @@ export class GalleryManager implements IGalleryManager { } - public indexDirectory(relativeDirectoryName): Promise { + public indexDirectory(relativeDirectoryName): Promise { return new Promise(async (resolve, reject) => { try { const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName); - - const connection = await MySQLConnection.getConnection(); //returning with the result @@ -82,32 +103,50 @@ export class GalleryManager implements IGalleryManager { if (!!parentDir) { parentDir.scanned = true; - parentDir.lastUpdate = Date.now(); + parentDir.lastUpdate = scannedDirectory.lastUpdate; parentDir = await directoryRepository.persist(parentDir); } else { (scannedDirectory).scanned = true; parentDir = await directoryRepository.persist(scannedDirectory); } + let indexedDirectories = await directoryRepository.createQueryBuilder("directory") + .where("directory.parent = :dir", { + dir: parentDir.id + }).getMany(); for (let i = 0; i < scannedDirectory.directories.length; i++) { - //TODO: simplify algorithm - let dir = await directoryRepository.createQueryBuilder("directory") - .where("directory.name = :name AND directory.path = :path", { - name: scannedDirectory.directories[i].name, - path: scannedDirectory.directories[i].path - }).getOne(); - if (dir) { - dir.parent = parentDir; - await directoryRepository.persist(dir); + let directory: DirectoryEntity = null; + for (let j = 0; j < indexedDirectories.length; j++) { + if (indexedDirectories[j].name == scannedDirectory.directories[i].name) { + directory = indexedDirectories[j]; + indexedDirectories.splice(j, 1); + break; + } + } + + if (directory) { //update existing directory + if (!directory.parent && !directory.parent.id) { + directory.parent = parentDir; + delete directory.photos; + await directoryRepository.persist(directory); + } } else { scannedDirectory.directories[i].parent = parentDir; (scannedDirectory.directories[i]).scanned = false; - await directoryRepository.persist(scannedDirectory.directories[i]); + const d = await directoryRepository.persist(scannedDirectory.directories[i]); + for (let j = 0; j < scannedDirectory.directories[i].photos.length; j++) { + PhotoMetadataEntity.close(scannedDirectory.directories[i].photos[j].metadata); + scannedDirectory.directories[i].photos[j].directory = d; + + } + await photosRepository.persist(scannedDirectory.directories[i].photos); } } + await directoryRepository.remove(indexedDirectories); + let indexedPhotos = await photosRepository.createQueryBuilder("photo") .where("photo.directory = :dir", { @@ -133,20 +172,17 @@ export class GalleryManager implements IGalleryManager { } //typeorm not supports recursive embended: TODO:fix it - let keyStr = JSON.stringify(scannedDirectory.photos[i].metadata.keywords); - let camStr = JSON.stringify(scannedDirectory.photos[i].metadata.cameraData); - let posStr = JSON.stringify(scannedDirectory.photos[i].metadata.positionData); - let sizeStr = JSON.stringify(scannedDirectory.photos[i].metadata.size); + PhotoMetadataEntity.close(scannedDirectory.photos[i].metadata); - if (photo.metadata.keywords != keyStr || - photo.metadata.cameraData != camStr || - photo.metadata.positionData != posStr || - photo.metadata.size != sizeStr) { + if (photo.metadata.keywords != scannedDirectory.photos[i].metadata.keywords || + photo.metadata.cameraData != scannedDirectory.photos[i].metadata.cameraData || + photo.metadata.positionData != scannedDirectory.photos[i].metadata.positionData || + photo.metadata.size != scannedDirectory.photos[i].metadata.size) { - photo.metadata.keywords = keyStr; - photo.metadata.cameraData = camStr; - photo.metadata.positionData = posStr; - photo.metadata.size = sizeStr; + photo.metadata.keywords = scannedDirectory.photos[i].metadata.keywords; + photo.metadata.cameraData = scannedDirectory.photos[i].metadata.cameraData; + photo.metadata.positionData = scannedDirectory.photos[i].metadata.positionData; + photo.metadata.size = scannedDirectory.photos[i].metadata.size; photosToSave.push(photo); } } diff --git a/backend/model/mysql/MySQLConnection.ts b/backend/model/mysql/MySQLConnection.ts index 20a9b418..ca0d522e 100644 --- a/backend/model/mysql/MySQLConnection.ts +++ b/backend/model/mysql/MySQLConnection.ts @@ -40,12 +40,12 @@ export class MySQLConnection { SharingEntity ], autoSchemaSync: true, - logging: { + /* logging: { logQueries: true, logOnlyFailedQueries: true, logFailedQueryError: true, logSchemaCreation: true - } + }*/ }); } return this.connection; @@ -66,20 +66,6 @@ export class MySQLConnection { username: config.mysql.username, password: config.mysql.password, database: config.mysql.database - }, - entities: [ - UserEntity, - DirectoryEntity, - PhotoMetadataEntity, - PhotoEntity, - SharingEntity - ], - autoSchemaSync: true, - logging: { - logQueries: true, - logOnlyFailedQueries: true, - logFailedQueryError: true, - logSchemaCreation: true } }); await conn.close(); diff --git a/backend/model/mysql/enitites/DirectoryEntity.ts b/backend/model/mysql/enitites/DirectoryEntity.ts index 1dc8a67d..85967e3d 100644 --- a/backend/model/mysql/enitites/DirectoryEntity.ts +++ b/backend/model/mysql/enitites/DirectoryEntity.ts @@ -25,7 +25,7 @@ export class DirectoryEntity implements DirectoryDTO { @Column({type: 'smallint', length: 1}) public scanned: boolean; - @ManyToOne(type => DirectoryEntity, directory => directory.directories) + @ManyToOne(type => DirectoryEntity, directory => directory.directories, {onDelete: "CASCADE"}) public parent: DirectoryEntity; @OneToMany(type => DirectoryEntity, dir => dir.parent) diff --git a/backend/model/mysql/enitites/PhotoEntity.ts b/backend/model/mysql/enitites/PhotoEntity.ts index 46e1421f..6acb0eee 100644 --- a/backend/model/mysql/enitites/PhotoEntity.ts +++ b/backend/model/mysql/enitites/PhotoEntity.ts @@ -18,7 +18,7 @@ export class PhotoEntity implements PhotoDTO { @Column("string") name: string; - @ManyToOne(type => DirectoryEntity, directory => directory.photos) + @ManyToOne(type => DirectoryEntity, directory => directory.photos, {onDelete: "CASCADE"}) directory: DirectoryDTO; @Embedded(type => PhotoMetadataEntity) @@ -51,6 +51,22 @@ export class PhotoMetadataEntity implements PhotoMetadata { @Column("number") fileSize: number; + + //TODO: fixit after typeorm update + public static open(m: PhotoMetadataEntity) { + m.keywords = JSON.parse(m.keywords); + m.cameraData = JSON.parse(m.cameraData); + m.positionData = JSON.parse(m.positionData); + m.size = JSON.parse(m.size); + } + + //TODO: fixit after typeorm update + public static close(m: PhotoMetadataEntity) { + m.keywords = JSON.stringify(m.keywords); + m.cameraData = JSON.stringify(m.cameraData); + m.positionData = JSON.stringify(m.positionData); + m.size = JSON.stringify(m.size); + } } /* diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 9b692809..3ae7fe41 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -7,6 +7,7 @@ import * as path from "path"; import {IptcParser} from "ts-node-iptc"; import * as exif_parser from "exif-parser"; import {ProjectPath} from "../../ProjectPath"; +import {Config} from "../../../common/config/private/Config"; const LOG_TAG = "[DiskManagerTask]"; export class DiskMangerWorker { @@ -51,7 +52,6 @@ export class DiskMangerWorker { try { const exif = exif_parser.create(data).parse(); - console.log(exif); metadata.cameraData = { ISO: exif.tags.ISO, model: exif.tags.Model, @@ -120,11 +120,11 @@ export class DiskMangerWorker { const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); - // let promises: Array> = []; + const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); let directory = { name: directoryName, path: directoryParent, - lastUpdate: Date.now(), + lastUpdate: Math.max(stat.ctime.getTime(), stat.mtime.getTime()), directories: [], photos: [] }; @@ -139,7 +139,7 @@ export class DiskMangerWorker { let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file)); if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file), - 5, true + Config.Server.folderPreviewSize, true )); } else if (DiskMangerWorker.isImage(fullFilePath)) { directory.photos.push({ diff --git a/backend/routes/ErrorRouter.ts b/backend/routes/ErrorRouter.ts index 67fe6333..406f116a 100644 --- a/backend/routes/ErrorRouter.ts +++ b/backend/routes/ErrorRouter.ts @@ -20,7 +20,8 @@ export class ErrorRouter { private static addGenericHandler(app) { app.use((err: any, req: Request, res: Response, next: Function) => { //Flush out the stack to the console - Logger.error("Unexpected error:", err); + Logger.error("Unexpected error:"); + console.error(err); next(new ErrorDTO(ErrorCodes.SERVER_ERROR, "Unknown server side error", err)); }, RenderingMWs.renderError diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index a3d10c1d..202124b9 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -38,6 +38,7 @@ export interface ServerConfig { enableThreading: boolean; sharing: SharingConfig; sessionTimeout: number + folderPreviewSize: number; } export interface IPrivateConfig { Server: ServerConfig; diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index a6846989..f1db580b 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -30,7 +30,8 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon sharing: { updateTimeout: 1000 * 60 * 5 }, - enableThreading: true + enableThreading: true, + folderPreviewSize: 5 }; private ConfigLoader: any;