fixing database index bugs

backend/middlewares/GalleryMWs.ts

@@ -74,14 +74,16 @@ export class GalleryMWs {
     }
 
     let fullImagePath = path.join(ProjectPath.ImageFolder, req.params.imagePath);
-    if (fs.statSync(fullImagePath).isDirectory()) {
-      return next();
-    }
 
     //check if thumbnail already exist
     if (fs.existsSync(fullImagePath) === false) {
       return next(new ErrorDTO(ErrorCodes.GENERAL_ERROR, "no such file:" + fullImagePath));
     }
+    if (fs.statSync(fullImagePath).isDirectory()) {
+      return next();
+    }
+
+
 
     req.resultPipe = fullImagePath;
     return next();

backend/model/mysql/GalleryManager.ts

@@ -1,12 +1,14 @@
 import {IGalleryManager} from "../interfaces/IGalleryManager";
 import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import * as path from "path";
+import * as fs from "fs";
 import {DirectoryEntity} from "./enitites/DirectoryEntity";
 import {MySQLConnection} from "./MySQLConnection";
 import {DiskManager} from "../DiskManger";
-import {PhotoEntity} from "./enitites/PhotoEntity";
+import {PhotoEntity, PhotoMetadataEntity} from "./enitites/PhotoEntity";
 import {Utils} from "../../../common/Utils";
 import {ProjectPath} from "../../ProjectPath";
+import {Config} from "../../../common/config/private/Config";
 
 export class GalleryManager implements IGalleryManager {
 
@@ -15,8 +17,6 @@ export class GalleryManager implements IGalleryManager {
     relativeDirectoryName = path.normalize(path.join("." + path.sep, relativeDirectoryName));
     const directoryName = path.basename(relativeDirectoryName);
     const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
-    console.log("GalleryManager:listDirectory");
-    console.log(directoryName, directoryParent, path.dirname(relativeDirectoryName), ProjectPath.normalizeRelative(path.dirname(relativeDirectoryName)));
     const connection = await MySQLConnection.getConnection();
     let dir = await connection
       .getRepository(DirectoryEntity)
@@ -33,18 +33,41 @@ export class GalleryManager implements IGalleryManager {
       if (dir.photos) {
         for (let i = 0; i < dir.photos.length; i++) {
           dir.photos[i].directory = dir;
-          dir.photos[i].metadata.keywords = <any>JSON.parse(<any>dir.photos[i].metadata.keywords);
+          PhotoMetadataEntity.open(dir.photos[i].metadata);
-          dir.photos[i].metadata.cameraData = <any>JSON.parse(<any>dir.photos[i].metadata.cameraData);
-          dir.photos[i].metadata.positionData = <any>JSON.parse(<any>dir.photos[i].metadata.positionData);
-          dir.photos[i].metadata.size = <any>JSON.parse(<any>dir.photos[i].metadata.size);
           dir.photos[i].readyThumbnails = [];
           dir.photos[i].readyIcon = false;
         }
       }
+      if (dir.directories) {
+        for (let i = 0; i < dir.directories.length; i++) {
+          dir.directories[i].photos = await connection
+            .getRepository(PhotoEntity)
+            .createQueryBuilder("photo")
+            .where("photo.directory = :dir", {
+              dir: dir.directories[i].id
+            })
+            .setLimit(Config.Server.folderPreviewSize)
+            .getMany();
 
+          for (let j = 0; j < dir.directories[i].photos.length; j++) {
+            dir.directories[i].photos[j].directory = dir.directories[i];
+            PhotoMetadataEntity.open(dir.directories[i].photos[j].metadata);
+            dir.directories[i].photos[j].readyThumbnails = [];
+            dir.directories[i].photos[j].readyIcon = false;
+          }
+        }
+      }
+
+      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
+      const lastUpdate = Math.max(stat.ctime.getTime(), stat.mtime.getTime());
+
+      if (dir.lastUpdate != lastUpdate) {
+        return this.indexDirectory(relativeDirectoryName);
+      }
+
+      console.log("lazy");
       //on the fly updating
       this.indexDirectory(relativeDirectoryName).catch((err) => {
-
         console.error(err);
       });
 
@@ -57,12 +80,10 @@ export class GalleryManager implements IGalleryManager {
 
   }
 
-  public   indexDirectory(relativeDirectoryName): Promise<DirectoryDTO> {
+  public indexDirectory(relativeDirectoryName): Promise<DirectoryDTO> {
     return new Promise(async (resolve, reject) => {
       try {
         const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
-
-
         const connection = await MySQLConnection.getConnection();
 
         //returning with the result
@@ -82,32 +103,50 @@ export class GalleryManager implements IGalleryManager {
 
         if (!!parentDir) {
           parentDir.scanned = true;
-          parentDir.lastUpdate = Date.now();
+          parentDir.lastUpdate = scannedDirectory.lastUpdate;
           parentDir = await directoryRepository.persist(parentDir);
         } else {
           (<DirectoryEntity>scannedDirectory).scanned = true;
           parentDir = await directoryRepository.persist(<DirectoryEntity>scannedDirectory);
         }
 
+        let indexedDirectories = await directoryRepository.createQueryBuilder("directory")
+          .where("directory.parent = :dir", {
+            dir: parentDir.id
+          }).getMany();
 
         for (let i = 0; i < scannedDirectory.directories.length; i++) {
-          //TODO: simplify algorithm
-          let dir = await directoryRepository.createQueryBuilder("directory")
-            .where("directory.name = :name AND directory.path = :path", {
-              name: scannedDirectory.directories[i].name,
-              path: scannedDirectory.directories[i].path
-            }).getOne();
 
-          if (dir) {
+          let directory: DirectoryEntity = null;
-            dir.parent = parentDir;
+          for (let j = 0; j < indexedDirectories.length; j++) {
-            await directoryRepository.persist(dir);
+            if (indexedDirectories[j].name == scannedDirectory.directories[i].name) {
+              directory = indexedDirectories[j];
+              indexedDirectories.splice(j, 1);
+              break;
+            }
+          }
+
+          if (directory) { //update existing directory
+            if (!directory.parent && !directory.parent.id) {
+              directory.parent = parentDir;
+              delete directory.photos;
+              await directoryRepository.persist(directory);
+            }
           } else {
             scannedDirectory.directories[i].parent = parentDir;
             (<DirectoryEntity>scannedDirectory.directories[i]).scanned = false;
-            await directoryRepository.persist(<DirectoryEntity>scannedDirectory.directories[i]);
+            const d = await directoryRepository.persist(<DirectoryEntity>scannedDirectory.directories[i]);
+            for (let j = 0; j < scannedDirectory.directories[i].photos.length; j++) {
+              PhotoMetadataEntity.close(scannedDirectory.directories[i].photos[j].metadata);
+              scannedDirectory.directories[i].photos[j].directory = d;
+
+            }
+            await photosRepository.persist(scannedDirectory.directories[i].photos);
           }
         }
 
+        await directoryRepository.remove(indexedDirectories);
+
 
         let indexedPhotos = await photosRepository.createQueryBuilder("photo")
           .where("photo.directory = :dir", {
@@ -133,20 +172,17 @@ export class GalleryManager implements IGalleryManager {
           }
 
           //typeorm not supports recursive embended: TODO:fix it
-          let keyStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.keywords);
+          PhotoMetadataEntity.close(scannedDirectory.photos[i].metadata);
-          let camStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.cameraData);
-          let posStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.positionData);
-          let sizeStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.size);
 
-          if (photo.metadata.keywords != keyStr ||
+          if (photo.metadata.keywords != scannedDirectory.photos[i].metadata.keywords ||
-            photo.metadata.cameraData != camStr ||
+            photo.metadata.cameraData != scannedDirectory.photos[i].metadata.cameraData ||
-            photo.metadata.positionData != posStr ||
+            photo.metadata.positionData != scannedDirectory.photos[i].metadata.positionData ||
-            photo.metadata.size != sizeStr) {
+            photo.metadata.size != scannedDirectory.photos[i].metadata.size) {
 
-            photo.metadata.keywords = keyStr;
+            photo.metadata.keywords = scannedDirectory.photos[i].metadata.keywords;
-            photo.metadata.cameraData = camStr;
+            photo.metadata.cameraData = scannedDirectory.photos[i].metadata.cameraData;
-            photo.metadata.positionData = posStr;
+            photo.metadata.positionData = scannedDirectory.photos[i].metadata.positionData;
-            photo.metadata.size = sizeStr;
+            photo.metadata.size = scannedDirectory.photos[i].metadata.size;
             photosToSave.push(photo);
           }
         }

backend/model/mysql/MySQLConnection.ts

@@ -40,12 +40,12 @@ export class MySQLConnection {
           SharingEntity
         ],
         autoSchemaSync: true,
-        logging: {
+        /*  logging: {
           logQueries: true,
           logOnlyFailedQueries: true,
           logFailedQueryError: true,
           logSchemaCreation: true
-        }
+         }*/
       });
     }
     return this.connection;
@@ -66,20 +66,6 @@ export class MySQLConnection {
         username: config.mysql.username,
         password: config.mysql.password,
         database: config.mysql.database
-      },
-      entities: [
-        UserEntity,
-        DirectoryEntity,
-        PhotoMetadataEntity,
-        PhotoEntity,
-        SharingEntity
-      ],
-      autoSchemaSync: true,
-      logging: {
-        logQueries: true,
-        logOnlyFailedQueries: true,
-        logFailedQueryError: true,
-        logSchemaCreation: true
       }
     });
     await conn.close();

backend/model/mysql/enitites/DirectoryEntity.ts

@@ -25,7 +25,7 @@ export class DirectoryEntity implements DirectoryDTO {
   @Column({type: 'smallint', length: 1})
   public scanned: boolean;
 
-  @ManyToOne(type => DirectoryEntity, directory => directory.directories)
+  @ManyToOne(type => DirectoryEntity, directory => directory.directories, {onDelete: "CASCADE"})
   public parent: DirectoryEntity;
 
   @OneToMany(type => DirectoryEntity, dir => dir.parent)

backend/model/mysql/enitites/PhotoEntity.ts

@@ -18,7 +18,7 @@ export class PhotoEntity implements PhotoDTO {
   @Column("string")
   name: string;
 
-  @ManyToOne(type => DirectoryEntity, directory => directory.photos)
+  @ManyToOne(type => DirectoryEntity, directory => directory.photos, {onDelete: "CASCADE"})
   directory: DirectoryDTO;
 
   @Embedded(type => PhotoMetadataEntity)
@@ -51,6 +51,22 @@ export class PhotoMetadataEntity implements PhotoMetadata {
 
   @Column("number")
   fileSize: number;
+
+  //TODO: fixit after typeorm update
+  public static open(m: PhotoMetadataEntity) {
+    m.keywords = <any>JSON.parse(<any>m.keywords);
+    m.cameraData = <any>JSON.parse(<any>m.cameraData);
+    m.positionData = <any>JSON.parse(<any>m.positionData);
+    m.size = <any>JSON.parse(<any>m.size);
+  }
+
+  //TODO: fixit after typeorm update
+  public static close(m: PhotoMetadataEntity) {
+    m.keywords = <any>JSON.stringify(<any>m.keywords);
+    m.cameraData = <any>JSON.stringify(<any>m.cameraData);
+    m.positionData = <any>JSON.stringify(<any>m.positionData);
+    m.size = <any>JSON.stringify(<any>m.size);
+  }
 }
 
 /*

backend/model/threading/DiskMangerWorker.ts

@@ -7,6 +7,7 @@ import * as path from "path";
 import {IptcParser} from "ts-node-iptc";
 import * as exif_parser from "exif-parser";
 import {ProjectPath} from "../../ProjectPath";
+import {Config} from "../../../common/config/private/Config";
 
 const LOG_TAG = "[DiskManagerTask]";
 export class DiskMangerWorker {
@@ -51,7 +52,6 @@ export class DiskMangerWorker {
 
             try {
               const exif = exif_parser.create(data).parse();
-              console.log(exif);
               metadata.cameraData = <CameraMetadata> {
                 ISO: exif.tags.ISO,
                 model: exif.tags.Model,
@@ -120,11 +120,11 @@ export class DiskMangerWorker {
       const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
       const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
 
-      // let promises: Array<Promise<any>> = [];
+      const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));
       let directory = <DirectoryDTO>{
         name: directoryName,
         path: directoryParent,
-        lastUpdate: Date.now(),
+        lastUpdate: Math.max(stat.ctime.getTime(), stat.mtime.getTime()),
         directories: [],
         photos: []
       };
@@ -139,7 +139,7 @@ export class DiskMangerWorker {
             let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
             if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
               directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
-                5, true
+                Config.Server.folderPreviewSize, true
               ));
             } else if (DiskMangerWorker.isImage(fullFilePath)) {
               directory.photos.push(<PhotoDTO>{

backend/routes/ErrorRouter.ts

@@ -20,7 +20,8 @@ export class ErrorRouter {
   private static addGenericHandler(app) {
     app.use((err: any, req: Request, res: Response, next: Function) => {
         //Flush out the stack to the console
-        Logger.error("Unexpected error:", err);
+        Logger.error("Unexpected error:");
+        console.error(err);
         next(new ErrorDTO(ErrorCodes.SERVER_ERROR, "Unknown server side error", err));
       },
       RenderingMWs.renderError

common/config/private/IPrivateConfig.ts

@@ -38,6 +38,7 @@ export interface ServerConfig {
   enableThreading: boolean;
   sharing: SharingConfig;
   sessionTimeout: number
+  folderPreviewSize: number;
 }
 export interface IPrivateConfig {
   Server: ServerConfig;

common/config/private/PrivateConfigClass.ts

@@ -30,7 +30,8 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
     sharing: {
       updateTimeout: 1000 * 60 * 5
     },
-    enableThreading: true
+    enableThreading: true,
+    folderPreviewSize: 5
   };
   private ConfigLoader: any;