improving indexing algorithm

backend/model/DiskManger.ts

@@ -31,7 +31,7 @@ export class DiskManager {
         };
 
         let promises: Array< Promise<any> > = [];
-        fs.readdir(absoluteDirectoryName, function (err, list) {
+        fs.readdir(absoluteDirectoryName, (err, list) => {
 
             if (err) {
                 return cb(err, null);
@@ -63,6 +63,7 @@ export class DiskManager {
             }
 
             Promise.all(promises).then(() => {
+                console.log("DiskManager: scanDirectory finished");
                 return cb(err, directory);
             });
 
@@ -125,7 +126,7 @@ export class DiskManager {
         return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
             fs.readFile(fullPath, function (err, data) {
                 if (err) {
-                    reject(err);
+                    return reject(err);
                 } else {
                     let exif = exif_parser.create(data).parse();
                     let iptcData = iptc(data);
@@ -175,7 +176,7 @@ export class DiskManager {
                         size: imageSize,
                         creationDate: creationDate
                     };
-                    resolve(metadata);
+                    return resolve(metadata);
                 }
             });
         });

backend/model/ObjectManagerRepository.ts

@@ -29,6 +29,7 @@ export class ObjectManagerRepository {
                 ObjectManagerRepository.getInstance().setGalleryManager(new GalleryManager());
                 ObjectManagerRepository.getInstance().setUserManager(new UserManager());
                 ObjectManagerRepository.getInstance().setSearchManager(new SearchManager());
+                console.log("MySQL DB inited");
                 resolve(true);
             }).catch(err => reject(err));
         });

backend/model/mysql/GalleryManager.ts

@@ -44,7 +44,11 @@ export class GalleryManager implements IGalleryManager {
                 cb(null, dir); //WARNING: only on the fly indexing should happen after this point
 
                 //on the fly updating
-                return this.indexDirectory(relativeDirectoryName, cb);
+                return this.indexDirectory(relativeDirectoryName, (err, res) => {
+                    if (!!err || !res) {
+                        console.error(err);
+                    }
+                });
             }
             return this.indexDirectory(relativeDirectoryName, cb);
 
@@ -58,8 +62,16 @@ export class GalleryManager implements IGalleryManager {
 
     public indexDirectory(relativeDirectoryName, cb: (error: any, result: DirectoryDTO) => void) {
         DiskManager.scanDirectory(relativeDirectoryName, (err, scannedDirectory) => {
+            if (!!err || !scannedDirectory) {
+                return cb(err, null);
+            }
+
             MySQLConnection.getConnection().then(async connection => {
 
+                //returning with the result
+                cb(null, scannedDirectory);
+
+                //saving to db
                 let directoryRepository = connection.getRepository(DirectoryEntity);
                 let photosRepository = connection.getRepository(PhotoEntity);
 
@@ -79,7 +91,9 @@ export class GalleryManager implements IGalleryManager {
                     parentDir = await directoryRepository.persist(<DirectoryEntity>scannedDirectory);
                 }
 
+
                 for (let i = 0; i < scannedDirectory.directories.length; i++) {
+                    //TODO: simplify algorithm
                     if ((await directoryRepository.createQueryBuilder("directory")
                             .where("directory.name = :name AND directory.path = :path", {
                                 name: scannedDirectory.directories[i].name,
@@ -89,31 +103,53 @@ export class GalleryManager implements IGalleryManager {
                     }
                     scannedDirectory.directories[i].parent = parentDir;
                     (<DirectoryEntity>scannedDirectory.directories[i]).scanned = false;
-                    await directoryRepository.persist(<DirectoryEntity>scannedDirectory.directories[i]);
+                    await directoryRepository.persist(<Array<DirectoryEntity>>scannedDirectory.directories);
                 }
 
+
+                let indexedPhotos = await photosRepository.createQueryBuilder("photo")
+                    .where("photo.directory = :dir", {
+                        dir: parentDir.id
+                    }).getMany();
+
+
+                let photosToSave = [];
                 for (let i = 0; i < scannedDirectory.photos.length; i++) {
-                    //TODO: load as batch
+                    let photo = null;
-                    if ((await photosRepository.createQueryBuilder("photo")
+                    for (let j = 0; j < indexedPhotos.length; j++) {
-                            .where("photo.name = :name AND photo.directory = :dir", {
+                        if (indexedPhotos[j].name == scannedDirectory.photos[i].name) {
-                                name: scannedDirectory.photos[i].name,
+                            photo = indexedPhotos[j];
-                                dir: parentDir.id
+                            indexedPhotos.splice(j, 1);
-                            }).getCount()) > 0) {
+                            break;
-                        continue;
+                        }
+                    }
+                    if (photo == null) {
+                        scannedDirectory.photos[i].directory = null;
+                        photo = Utils.clone(scannedDirectory.photos[i]);
+                        scannedDirectory.photos[i].directory = scannedDirectory;
+                        photo.directory = parentDir;
                     }
-                    //typeorm not supports recursive embended: TODO:fix it
-                    scannedDirectory.photos[i].directory = null;
-                    let photo = Utils.clone(scannedDirectory.photos[i]);
-                    scannedDirectory.photos[i].directory = scannedDirectory;
-                    photo.directory = parentDir;
-                    photo.metadata.keywords = <any>JSON.stringify(photo.metadata.keywords);
-                    photo.metadata.cameraData = <any>JSON.stringify(photo.metadata.cameraData);
-                    photo.metadata.positionData = <any>JSON.stringify(photo.metadata.positionData);
-                    photo.metadata.size = <any>JSON.stringify(photo.metadata.size);
-                    await photosRepository.persist(photo);
-                }
 
-                return cb(null, scannedDirectory);
+                    //typeorm not supports recursive embended: TODO:fix it
+                    let keyStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.keywords);
+                    let camStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.cameraData);
+                    let posStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.positionData);
+                    let sizeStr = <any>JSON.stringify(scannedDirectory.photos[i].metadata.size);
+
+                    if (photo.metadata.keywords != keyStr ||
+                        photo.metadata.cameraData != camStr ||
+                        photo.metadata.positionData != posStr ||
+                        photo.metadata.size != sizeStr) {
+
+                        photo.metadata.keywords = keyStr;
+                        photo.metadata.cameraData = camStr;
+                        photo.metadata.positionData = posStr;
+                        photo.metadata.size = sizeStr;
+                        photosToSave.push(photo);
+                    }
+                }
+                await photosRepository.persist(photosToSave);
+                await photosRepository.remove(indexedPhotos);
 
 
             }).catch((error) => {

package.json

@@ -32,7 +32,7 @@
     "@angular/router": "~3.4.1",
     "body-parser": "^1.15.2",
     "core-js": "^2.4.1",
-    "debug": "^2.5.2",
+    "debug": "^2.6.0",
     "ejs": "^2.5.5",
     "exif-parser": "^0.1.9",
     "express": "^4.14.0",
@@ -49,24 +49,24 @@
     "reflect-metadata": "^0.1.9",
     "rxjs": "^5.0.2",
     "systemjs": "0.19.41",
-    "typeorm": "0.0.5",
+    "typeorm": "0.0.6",
     "zone.js": "^0.7.4"
   },
   "devDependencies": {
     "@types/express": "^4.0.34",
     "@types/express-session": "0.0.32",
-    "@types/jasmine": "^2.5.38",
+    "@types/jasmine": "^2.5.39",
     "@types/node": "^6.0.54",
     "@types/optimist": "0.0.29",
     "angular-cli": "^1.0.0-beta.24",
     "chai": "^3.5.0",
     "jasmine-core": "^2.5.2",
-    "karma": "^1.2.0",
+    "karma": "^1.3.0",
     "karma-cli": "^1.0.1",
-    "karma-jasmine": "^1.0.2",
+    "karma-jasmine": "^1.1.0",
     "karma-jasmine-html-reporter": "^0.2.2",
     "karma-phantomjs-launcher": "^1.0.2",
-    "karma-remap-istanbul": "^0.2.2",
+    "karma-remap-istanbul": "^0.4.0",
     "karma-systemjs": "^0.16.0",
     "mocha": "^3.2.0",
     "phantomjs-prebuilt": "^2.1.14",