improving mutithreading at thumbnail generation

implementing mutithread directory scanning

backend/middlewares/ThumbnailGeneratorMWs.ts

@@ -2,6 +2,7 @@
 import * as path from "path";
 import * as crypto from "crypto";
 import * as fs from "fs";
+import * as os from "os";
 import {NextFunction, Request, Response} from "express";
 import {Error, ErrorCodes} from "../../common/entities/Error";
 import {Config} from "../config/Config";
@@ -10,8 +11,11 @@ import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
 import {ProjectPath} from "../ProjectPath";
 import {PhotoDTO} from "../../common/entities/PhotoDTO";
 
+
+Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
+
 const Pool = require('threads').Pool;
-const pool = new Pool();
+const pool = new Pool(Config.Client.concurrentThumbnailGenerations);
 
 pool.run(
     (input: {imagePath: string, size: number, thPath: string}, done) => {
@@ -43,6 +47,7 @@ pool.run(
     }
 );
 
+
 export class ThumbnailGeneratorMWs {
 
 
@@ -117,6 +122,7 @@ export class ThumbnailGeneratorMWs {
         //check if thumbnail already exist
         if (fs.existsSync(thPath) === true) {
             return next();
+            //return setTimeout(()=>{next();},2000);
         }
 
         //create thumbnail folder if not exist
@@ -124,7 +130,7 @@ export class ThumbnailGeneratorMWs {
             fs.mkdirSync(ProjectPath.ThumbnailFolder);
         }
 
-
+        console.log("generating thumbnail", imagePath, size);
         //run on other thread
         pool.send({imagePath: imagePath, size: size, thPath: thPath})
             .on('done', (out) => {
@@ -134,8 +140,6 @@ export class ThumbnailGeneratorMWs {
         });
 
 
-
-
     }
 
     private static generateThumbnailName(imagePath: string, size: number): string {

backend/model/DiskManger.ts

@@ -1,9 +1,5 @@
 ///<reference path="exif.d.ts"/>
-import * as fs from "fs";
 import * as path from "path";
-import * as mime from "mime";
-import * as iptc from "node-iptc";
-import * as exif_parser from "exif-parser";
 import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
 import {
     PhotoDTO,
@@ -15,47 +11,135 @@ import {
 } from "../../common/entities/PhotoDTO";
 import {ProjectPath} from "../ProjectPath";
 
-export class DiskManager {
+const Pool = require('threads').Pool;
-    public static scanDirectory(relativeDirectoryName: string, cb: (error: any, result: DirectoryDTO) => void) {
+const pool = new Pool();
-        console.log("DiskManager: scanDirectory");
+pool.run(
-        let directoryName = path.basename(relativeDirectoryName);
+    (input: {
-        let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
+        relativeDirectoryName: string,
-        let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
+        directoryName: string,
+        directoryParent: string,
+        absoluteDirectoryName: string
+    }, done) => {
+        const fs = require("fs");
+        const path = require("path");
+        const mime = require("mime");
+        const iptc = require("node-iptc");
+        const exif_parser = require("exif-parser");
+
+
+        let isImage = (fullPath: string) => {
+            let imageMimeTypes = [
+                'image/bmp',
+                'image/gif',
+                'image/jpeg',
+                'image/png',
+                'image/pjpeg',
+                'image/tiff',
+                'image/webp',
+                'image/x-tiff',
+                'image/x-windows-bmp'
+            ];
+
+            let extension = mime.lookup(fullPath);
+
+            return imageMimeTypes.indexOf(extension) !== -1;
+        };
+
+        let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
+            return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
+                fs.readFile(fullPath, function (err, data) {
+                    if (err) {
+                        return reject(err);
+                    } else {
+                        let exif = exif_parser.create(data).parse();
+                        let iptcData = iptc(data);
+
+                        let imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height};
+                        let cameraData: CameraMetadata = {
+                            ISO: exif.tags.ISO,
+                            model: exif.tags.Modeol,
+                            maker: exif.tags.Make,
+                            fStop: exif.tags.FNumber,
+                            exposure: exif.tags.ExposureTime,
+                            focalLength: exif.tags.FocalLength,
+                            lens: exif.tags.LensModel,
+                        };
+                        let GPS: GPSMetadata = {
+                            latitude: exif.tags.GPSLatitude,
+                            longitude: exif.tags.GPSLongitude,
+                            altitude: exif.tags.GPSAltitude
+
+                        };
+
+                        let positionData: PositionMetaData = {
+                            GPSData: GPS,
+                            country: iptcData.country_or_primary_location_name,
+                            state: iptcData.province_or_state,
+                            city: iptcData.city
+                        };
+
+                        //Decode characters to UTF8
+                        let decode = (s: any) => {
+                            for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
+                                 ((a = s[i][c](0)) & 0x80) &&
+                                 (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
+                                     o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
+                            );
+                            return s.join("");
+                        };
+
+                        let keywords: [string] = iptcData.keywords.map((s: string) => decode(s));
+                        let creationDate: number = iptcData.date_time.getTime();
+
+
+                        let metadata: PhotoMetadata = <PhotoMetadata>{
+                            keywords: keywords,
+                            cameraData: cameraData,
+                            positionData: positionData,
+                            size: imageSize,
+                            creationDate: creationDate
+                        };
+                        return resolve(metadata);
+                    }
+                });
+            });
+        };
+
 
         let directory = <DirectoryDTO>{
-            name: directoryName,
+            name: input.directoryName,
-            path: directoryParent,
+            path: input.directoryParent,
             lastUpdate: Date.now(),
             directories: [],
             photos: []
         };
 
         let promises: Array< Promise<any> > = [];
-        fs.readdir(absoluteDirectoryName, (err, list) => {
+        fs.readdir(input.absoluteDirectoryName, (err, list) => {
 
             if (err) {
-                return cb(err, null);
+                return done(err, null);
             }
 
 
             for (let i = 0; i < list.length; i++) {
                 let file = list[i];
-                let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
+                let fullFilePath = path.normalize(path.resolve(input.absoluteDirectoryName, file));
                 if (fs.statSync(fullFilePath).isDirectory()) {
                     directory.directories.push(<DirectoryDTO>{
                         name: file,
-                        path: path.join(relativeDirectoryName, path.sep),
+                        path: path.join(input.relativeDirectoryName, path.sep),
                         lastUpdate: Date.now(),
                         directories: [],
                         photos: []
                     });
                 }
 
-                if (DiskManager.isImage(fullFilePath)) {
+                if (isImage(fullFilePath)) {
 
 
-                    let promise = DiskManager.loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
+                    let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
-                        directory.photos.push(<PhotoDTO>{name: file, directory: directory, metadata: photoMetadata});
+                        directory.photos.push(<PhotoDTO>{name: file, directory: null, metadata: photoMetadata});
                     });
 
                     promises.push(promise);
@@ -64,123 +148,34 @@ export class DiskManager {
 
             Promise.all(promises).then(() => {
                 console.log("DiskManager: scanDirectory finished");
-                return cb(err, directory);
+                return done(err, directory);
+            }).catch((err) => {
+                console.error(err);
             });
 
         });
-    }
+    });
 
-    private static isImage(fullPath: string) {
+export class DiskManager {
-        let imageMimeTypes = [
+    public static scanDirectory(relativeDirectoryName: string, cb: (error: any, result: DirectoryDTO) => void) {
-            'image/bmp',
+        console.log("DiskManager: scanDirectory");
-            'image/gif',
+        let directoryName = path.basename(relativeDirectoryName);
-            'image/jpeg',
+        let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
-            'image/png',
+        let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
-            'image/pjpeg',
-            'image/tiff',
-            'image/webp',
-            'image/x-tiff',
-            'image/x-windows-bmp'
-        ];
 
-        let extension = mime.lookup(fullPath);
+        pool.send({
-
+            relativeDirectoryName,
-        return imageMimeTypes.indexOf(extension) !== -1;
+            directoryName,
-
+            directoryParent,
-
+            absoluteDirectoryName
-    }
+        }) .on('done', (error: any, result: DirectoryDTO) => {
-
+            result.photos.forEach((ph) => {
-    /*
+                ph.directory = result;
-     UTF8 = {
-
-     encode: function(s){
-
-     for(var c, i = -1, l = (s = s.split("")).length, o = String.fromCharCode; ++i < l;
-
-     s[i] = (c = s[i].charCodeAt(0)) >= 127 ? o(0xc0 | (c >>> 6)) + o(0x80 | (c & 0x3f)) : s[i]
-
-     );
-
-     return s.join("");
-
-     },
-
-     decode: function(s){
-
-     for(var a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
-
-     ((a = s[i][c](0)) & 0x80) &&
-
-     (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
-
-     o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
-
-     );
-
-     return s.join("");
-
-     }
-
-     };*/
-    private static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
-        return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
-            fs.readFile(fullPath, function (err, data) {
-                if (err) {
-                    return reject(err);
-                } else {
-                    let exif = exif_parser.create(data).parse();
-                    let iptcData = iptc(data);
-
-                    let imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height};
-                    let cameraData: CameraMetadata = {
-                        ISO: exif.tags.ISO,
-                        model: exif.tags.Modeol,
-                        maker: exif.tags.Make,
-                        fStop: exif.tags.FNumber,
-                        exposure: exif.tags.ExposureTime,
-                        focalLength: exif.tags.FocalLength,
-                        lens: exif.tags.LensModel,
-                    };
-                    let GPS: GPSMetadata = {
-                        latitude: exif.tags.GPSLatitude,
-                        longitude: exif.tags.GPSLongitude,
-                        altitude: exif.tags.GPSAltitude
-
-                    };
-
-                    let positionData: PositionMetaData = {
-                        GPSData: GPS,
-                        country: iptcData.country_or_primary_location_name,
-                        state: iptcData.province_or_state,
-                        city: iptcData.city
-                    };
-
-                    //Decode characters to UTF8
-                    let decode = (s: any) => {
-                        for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
-                             ((a = s[i][c](0)) & 0x80) &&
-                             (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
-                                 o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
-                        );
-                        return s.join("");
-                    };
-
-                    let keywords: [string] = iptcData.keywords.map((s: string) => decode(s));
-                    let creationDate: number = iptcData.date_time.getTime();
-
-
-                    let metadata: PhotoMetadata = <PhotoMetadata>{
-                        keywords: keywords,
-                        cameraData: cameraData,
-                        positionData: positionData,
-                        size: imageSize,
-                        creationDate: creationDate
-                    };
-                    return resolve(metadata);
-                }
             });
+            return cb(error, result);
+        }).on('error', (job, error) => {
+            return cb(error, null);
         });
-
-
     }
+
 }

frontend/app/gallery/grid/thumnailLoader.service.ts

@@ -76,7 +76,7 @@ export class ThumbnailLoaderService {
 
         for (let i = 0; i < this.que.length; i++) {
             for (let j = 0; j < this.que[i].taskEntities.length; j++) {
-                if (this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.high) {
+                if (this.que[i].inProgress == false && this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.high) {
                     return this.que[i];
                 }
             }
@@ -84,13 +84,21 @@ export class ThumbnailLoaderService {
 
         for (let i = 0; i < this.que.length; i++) {
             for (let j = 0; j < this.que[i].taskEntities.length; j++) {
-                if (this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.medium) {
+                if (this.que[i].inProgress == false && this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.medium) {
                     return this.que[i];
                 }
             }
         }
 
-        return this.que[0];
+
+        for (let i = 0; i < this.que.length; i++) {
+            if (this.que[i].inProgress == false) {
+                return this.que[i];
+            }
+
+        }
+
+        return null;
     }
 
     private taskReady(task: ThumbnailTask) {
@@ -119,10 +127,10 @@ export class ThumbnailLoaderService {
         task.taskEntities.forEach(te => te.listener.onStartedLoading());
         task.inProgress = true;
 
+        console.log("loading", task.gridPhoto.getThumbnailPath());
         let curImg = new Image();
-        curImg.src = task.gridPhoto.getThumbnailPath();
-
         curImg.onload = () => {
+            console.log("loaded", task.gridPhoto.getThumbnailPath());
 
             task.gridPhoto.thumbnailLoaded();
             this.galleryChacheService.photoUpdated(task.gridPhoto.photo);
@@ -134,12 +142,15 @@ export class ThumbnailLoaderService {
         };
 
         curImg.onerror = (error) => {
+            console.log("error", task.gridPhoto.getThumbnailPath());
             task.taskEntities.forEach((te: ThumbnailTaskEntity) => te.listener.onError(error));
 
             this.taskReady(task);
             this.runningRequests--;
             this.run();
         };
+
+        curImg.src = task.gridPhoto.getThumbnailPath();
     };
 }