xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00
Code Actions Projects Activity

fixing database index bugs

Browse Source
This commit is contained in:
Braun Patrik 2017-07-17 23:12:12 +02:00
parent d274613e38
commit 0f4eb10c91
9 changed files with 104 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/middlewares/GalleryMWs.ts
View File

@ -74,14 +74,16 @@ export class GalleryMWs {
}
let fullImagePath = path.join(ProjectPath.ImageFolder, req.params.imagePath);
if (fs.statSync(fullImagePath).isDirectory()) {
return next();
}
//check if thumbnail already exist
if (fs.existsSync(fullImagePath) === false) {
return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, "no such file:" + fullImagePath));
}
if (fs.statSync(fullImagePath).isDirectory()) {
return next();
}
req.resultPipe = fullImagePath;
return next();

104
backend/model/mysql/GalleryManager.ts
View File

@ -1,12 +1,14 @@
import {IGalleryManager} from "../interfaces/IGalleryManager";
import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
import * as path from "path";
import * as fs from "fs";
import {DirectoryEntity} from "./enitites/DirectoryEntity";
import {MySQLConnection} from "./MySQLConnection";
import {DiskManager} from "../DiskManger";
import {PhotoEntity} from "./enitites/PhotoEntity";
import {PhotoEntity, PhotoMetadataEntity} from "./enitites/PhotoEntity";
import {Utils} from "../../../common/Utils";
import {ProjectPath} from "../../ProjectPath";
import {Config} from "../../../common/config/private/Config";
export class GalleryManager implements IGalleryManager {
@ -15,8 +17,6 @@ export class GalleryManager implements IGalleryManager {
relativeDirectoryName = path.normalize(path.join("." + path.sep, relativeDirectoryName));
const directoryName = path.basename(relativeDirectoryName);
const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
console.log("GalleryManager:listDirectory");
console.log(directoryName, directoryParent, path.dirname(relativeDirectoryName), ProjectPath.normalizeRelative(path.dirname(relativeDirectoryName)));
const connection = await MySQLConnection.getConnection();
let dir = await connection
.getRepository(DirectoryEntity)
@ -33,18 +33,41 @@ export class GalleryManager implements IGalleryManager {
if (dir.photos) {
for (let i = 0; i < dir.photos.length; i++) {
dir.photos[i].directory = dir;
dir.photos[i].metadata.keywords = <any>JSON.parse(<any>dir.photos[i].metadata.keywords);
dir.photos[i].metadata.cameraData = <any>JSON.parse(<any>dir.photos[i].metadata.cameraData);
dir.photos[i].metadata.positionData = <any>JSON.parse(<any>dir.photos[i].metadata.positionData);
dir.photos[i].metadata.size = <any>JSON.parse(<any>dir.photos[i].metadata.size);
PhotoMetadataEntity.open(dir.photos[i].metadata);
dir.photos[i].readyThumbnails = [];
dir.photos[i].readyIcon = false;
}
}
if (dir.directories) {
for (let i = 0; i < dir.directories.length; i++) {
dir.directories[i].photos = await connection
.getRepository(PhotoEntity)
.createQueryBuilder("photo")
.where("photo.directory = :dir", {
dir: dir.directories[i].id
})
.setLimit(Config.Server.folderPreviewSize)
.getMany();
for (let j = 0; j < dir.directories[i].photos.length; j++) {
dir.directories[i].photos[j].directory = dir.directories[i];
PhotoMetadataEntity.open(dir.directories[i].photos[j].metadata);
dir.directories[i].photos[j].readyThumbnails = [];
dir.directories[i].photos[j].readyIcon = false;
}
}
}
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
const lastUpdate = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
if (dir.lastUpdate != lastUpdate) {
return this.indexDirectory(relativeDirectoryName);
}
console.log("lazy");
//on the fly updating
this.indexDirectory(relativeDirectoryName).catch((err) => {
console.error(err);
});
@ -57,12 +80,10 @@ export class GalleryManager implements IGalleryManager {
}
public indexDirectory(relativeDirectoryName): Promise<DirectoryDTO> {
public indexDirectory(relativeDirectoryName): Promise<DirectoryDTO> {
return new Promise(async (resolve, reject) => {
try {
const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
const connection = await MySQLConnection.getConnection();
//returning with the result
@ -82,32 +103,50 @@ export class GalleryManager implements IGalleryManager {
if (!!parentDir) {
parentDir.scanned = true;
parentDir.lastUpdate = Date.now();
parentDir.lastUpdate = scannedDirectory.lastUpdate;
parentDir = await directoryRepository.persist(parentDir);
} else {
(<DirectoryEntity>scannedDirectory).scanned = true;
parentDir = await directoryRepository.persist(<DirectoryEntity>scannedDirectory);
}
let indexedDirectories = await directoryRepository.createQueryBuilder("directory")
.where("directory.parent = :dir", {
dir: parentDir.id
}).getMany();
for (let i = 0; i < scannedDirectory.directories.length; i++) {
//TODO: simplify algorithm
let dir = await directoryRepository.createQueryBuilder("directory")
.where("directory.name = :name AND directory.path = :path", {
name: scannedDirectory.directories[i].name,
path: scannedDirectory.directories[i].path
}).getOne();
if (dir) {
dir.parent = parentDir;
await directoryRepository.persist(dir);
let directory: DirectoryEntity = null;
for (let j = 0; j < indexedDirectories.length; j++) {
if (indexedDirectories[j].name == scannedDirectory.directories[i].name) {
directory = indexedDirectories[j];
indexedDirectories.splice(j, 1);
break;
}
}
if (directory) { //update existing directory
if (!directory.parent && !directory.parent.id) {
directory.parent = parentDir;
delete directory.photos;
await directoryRepository.persist(directory);
}
} else {
scannedDirectory.directories[i].parent = parentDir;
(<DirectoryEntity>scannedDirectory.directories[i]).scanned = false;
await directoryRepository.persist(<DirectoryEntity>scannedDirectory.directories[i]);
const d = await directoryRepository.persist(<DirectoryEntity>scannedDirectory.directories[i]);
for (let j = 0; j < scannedDirectory.directories[i].photos.length; j++) {
PhotoMetadataEntity.close(scannedDirectory.directories[i].photos[j].metadata);
scannedDirectory.directories[i].photos[j].directory = d;
}
await photosRepository.persist(scannedDirectory.directories[i].photos);
}
}
await directoryRepository.remove(indexedDirectories);
let indexedPhotos = await photosRepository.createQueryBuilder("photo")
.where("photo.directory = :dir", {
@ -133,20 +172,17 @@ export class GalleryManager implements IGalleryManager {
}
//typeorm not supports recursive embended: TODO:fix it
let keyStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.keywords);
let camStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.cameraData);
let posStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.positionData);
let sizeStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.size);
PhotoMetadataEntity.close(scannedDirectory.photos[i].metadata);
if (photo.metadata.keywords != keyStr ||
photo.metadata.cameraData != camStr ||
photo.metadata.positionData != posStr ||
photo.metadata.size != sizeStr) {
if (photo.metadata.keywords != scannedDirectory.photos[i].metadata.keywords ||
photo.metadata.cameraData != scannedDirectory.photos[i].metadata.cameraData ||
photo.metadata.positionData != scannedDirectory.photos[i].metadata.positionData ||
photo.metadata.size != scannedDirectory.photos[i].metadata.size) {
photo.metadata.keywords = keyStr;
photo.metadata.cameraData = camStr;
photo.metadata.positionData = posStr;
photo.metadata.size = sizeStr;
photo.metadata.keywords = scannedDirectory.photos[i].metadata.keywords;
photo.metadata.cameraData = scannedDirectory.photos[i].metadata.cameraData;
photo.metadata.positionData = scannedDirectory.photos[i].metadata.positionData;
photo.metadata.size = scannedDirectory.photos[i].metadata.size;
photosToSave.push(photo);
}
}

18
backend/model/mysql/MySQLConnection.ts
View File

@ -40,12 +40,12 @@ export class MySQLConnection {
SharingEntity
],
autoSchemaSync: true,
logging: {
/* logging: {
logQueries: true,
logOnlyFailedQueries: true,
logFailedQueryError: true,
logSchemaCreation: true
}
}*/
});
}
return this.connection;
@ -66,20 +66,6 @@ export class MySQLConnection {
username: config.mysql.username,
password: config.mysql.password,
database: config.mysql.database
},
entities: [
UserEntity,
DirectoryEntity,
PhotoMetadataEntity,
PhotoEntity,
SharingEntity
],
autoSchemaSync: true,
logging: {
logQueries: true,
logOnlyFailedQueries: true,
logFailedQueryError: true,
logSchemaCreation: true
}
});
await conn.close();

2
backend/model/mysql/enitites/DirectoryEntity.ts
View File

@ -25,7 +25,7 @@ export class DirectoryEntity implements DirectoryDTO {
@Column({type: 'smallint', length: 1})
public scanned: boolean;
@ManyToOne(type => DirectoryEntity, directory => directory.directories)
@ManyToOne(type => DirectoryEntity, directory => directory.directories, {onDelete: "CASCADE"})
public parent: DirectoryEntity;
@OneToMany(type => DirectoryEntity, dir => dir.parent)

18
backend/model/mysql/enitites/PhotoEntity.ts
View File

@ -18,7 +18,7 @@ export class PhotoEntity implements PhotoDTO {
@Column("string")
name: string;
@ManyToOne(type => DirectoryEntity, directory => directory.photos)
@ManyToOne(type => DirectoryEntity, directory => directory.photos, {onDelete: "CASCADE"})
directory: DirectoryDTO;
@Embedded(type => PhotoMetadataEntity)
@ -51,6 +51,22 @@ export class PhotoMetadataEntity implements PhotoMetadata {
@Column("number")
fileSize: number;
//TODO: fixit after typeorm update
public static open(m: PhotoMetadataEntity) {
m.keywords = <any>JSON.parse(<any>m.keywords);
m.cameraData = <any>JSON.parse(<any>m.cameraData);
m.positionData = <any>JSON.parse(<any>m.positionData);
m.size = <any>JSON.parse(<any>m.size);
}
//TODO: fixit after typeorm update
public static close(m: PhotoMetadataEntity) {
m.keywords = <any>JSON.stringify(<any>m.keywords);
m.cameraData = <any>JSON.stringify(<any>m.cameraData);
m.positionData = <any>JSON.stringify(<any>m.positionData);
m.size = <any>JSON.stringify(<any>m.size);
}
}
/*

8
backend/model/threading/DiskMangerWorker.ts
View File

@ -7,6 +7,7 @@ import * as path from "path";
import {IptcParser} from "ts-node-iptc";
import * as exif_parser from "exif-parser";
import {ProjectPath} from "../../ProjectPath";
import {Config} from "../../../common/config/private/Config";
const LOG_TAG = "[DiskManagerTask]";
export class DiskMangerWorker {
@ -51,7 +52,6 @@ export class DiskMangerWorker {
try {
const exif = exif_parser.create(data).parse();
console.log(exif);
metadata.cameraData = <CameraMetadata> {
ISO: exif.tags.ISO,
model: exif.tags.Model,
@ -120,11 +120,11 @@ export class DiskMangerWorker {
const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
// let promises: Array<Promise<any>> = [];
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
let directory = <DirectoryDTO>{
name: directoryName,
path: directoryParent,
lastUpdate: Date.now(),
lastUpdate: Math.max(stat.ctime.getTime(), stat.mtime.getTime()),
directories: [],
photos: []
};
@ -139,7 +139,7 @@ export class DiskMangerWorker {
let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
5, true
Config.Server.folderPreviewSize, true
));
} else if (DiskMangerWorker.isImage(fullFilePath)) {
directory.photos.push(<PhotoDTO>{

3
backend/routes/ErrorRouter.ts
View File

@ -20,7 +20,8 @@ export class ErrorRouter {
private static addGenericHandler(app) {
app.use((err: any, req: Request, res: Response, next: Function) => {
//Flush out the stack to the console
Logger.error("Unexpected error:", err);
Logger.error("Unexpected error:");
console.error(err);
next(new ErrorDTO(ErrorCodes.SERVER_ERROR, "Unknown server side error", err));
},
RenderingMWs.renderError

1
common/config/private/IPrivateConfig.ts
View File

@ -38,6 +38,7 @@ export interface ServerConfig {
enableThreading: boolean;
sharing: SharingConfig;
sessionTimeout: number
folderPreviewSize: number;
}
export interface IPrivateConfig {
Server: ServerConfig;

3
common/config/private/PrivateConfigClass.ts
View File

@ -30,7 +30,8 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
sharing: {
updateTimeout: 1000 * 60 * 5
},
enableThreading: true
enableThreading: true,
folderPreviewSize: 5
};
private ConfigLoader: any;