Implementing cluster based threading

removing threads dependency

backend/index.ts

@@ -0,0 +1,10 @@
+import * as cluster from "cluster";
+import {Worker} from "./model/threading/Worker";
+import {Server} from "./server";
+
+
+if (cluster.isMaster) {
+  new Server();
+} else {
+  Worker.process();
+}

backend/middlewares/GalleryMWs.ts

@@ -84,7 +84,7 @@ export class GalleryMWs {
 
     //check if thumbnail already exist
     if (fs.existsSync(fullImagePath) === false) {
-      return next(new Error(ErrorCodes.GENERAL_ERROR, "no such file :" + fullImagePath));
+      return next(new Error(ErrorCodes.GENERAL_ERROR, "no such file:" + fullImagePath));
     }
 
     req.resultPipe = fullImagePath;

backend/middlewares/thumbnail/THRenderers.ts

@@ -1,170 +0,0 @@
-import {Metadata, SharpInstance} from "sharp";
-import {Dimensions, State} from "gm";
-import {Logger} from "../../Logger";
-import {Error, ErrorCodes} from "../../../common/entities/Error";
-
-export module ThumbnailRenderers {
-
-  export interface RendererInput {
-    imagePath: string;
-    size: number;
-    makeSquare: boolean;
-    thPath: string;
-    qualityPriority: boolean,
-    __dirname: string;
-  }
-
-  export const jimp = (input: RendererInput, done) => {
-    //generate thumbnail
-    const Jimp = require("jimp");
-    Jimp.read(input.imagePath).then((image) => {
-
-      const Logger = require(input.__dirname + "/../../Logger").Logger;
-      Logger.silly("[JimpThRenderer] rendering thumbnail:", input.imagePath);
-      /**
-       * newWidth * newHeight = size*size
-       * newHeight/newWidth = height/width
-       *
-       * newHeight = (height/width)*newWidth
-       * newWidth * newWidth = (size*size) / (height/width)
-       *
-       * @type {number}
-       */
-      const ratio = image.bitmap.height / image.bitmap.width;
-      const algo = input.qualityPriority == true ? Jimp.RESIZE_BEZIER : Jimp.RESIZE_NEAREST_NEIGHBOR;
-      if (input.makeSquare == false) {
-        let newWidth = Math.sqrt((input.size * input.size) / ratio);
-
-        image.resize(newWidth, Jimp.AUTO, algo);
-      } else {
-        image.resize(input.size / Math.min(ratio, 1), Jimp.AUTO, algo);
-        image.crop(0, 0, input.size, input.size);
-      }
-      image.quality(60);        // set JPEG quality
-      image.write(input.thPath, () => { // save
-        return done();
-      });
-    }).catch(function (err) {
-      const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-      const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-      return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-    });
-  };
-
-  export const sharp = (input: RendererInput, done) => {
-    //generate thumbnail
-    const sharp = require("sharp");
-
-    const image: SharpInstance = sharp(input.imagePath);
-    image
-      .metadata()
-      .then((metadata: Metadata) => {
-
-        //   const Logger = require(input.__dirname + "/../../Logger").Logger;
-        Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath);
-        /**
-         * newWidth * newHeight = size*size
-         * newHeight/newWidth = height/width
-         *
-         * newHeight = (height/width)*newWidth
-         * newWidth * newWidth = (size*size) / (height/width)
-         *
-         * @type {number}
-         */
-        try {
-          const ratio = metadata.height / metadata.width;
-          const kernel = input.qualityPriority == true ? sharp.kernel.lanczos3 : sharp.kernel.nearest;
-          const interpolator = input.qualityPriority == true ? sharp.interpolator.bicubic : sharp.interpolator.nearest;
-          if (input.makeSquare == false) {
-            const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
-            image.resize(newWidth, null, {
-              kernel: kernel,
-              interpolator: interpolator
-            });
-
-          } else {
-            image
-              .resize(input.size, input.size, {
-                kernel: kernel,
-                interpolator: interpolator
-              })
-              .crop(sharp.strategy.center);
-          }
-          image
-            .jpeg()
-            .toFile(input.thPath).then(() => {
-            return done();
-          }).catch(function (err) {
-            //      const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-            //       const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-            console.error(err);
-            return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-          });
-        } catch (err) {
-          //  const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-          // const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-          console.error(err);
-          return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-        }
-      });
-
-  };
-
-
-  export const gm = (input: RendererInput, done) => {
-    //generate thumbnail
-    const gm = require("gm");
-
-    let image: State = gm(input.imagePath);
-    image
-      .size((err, value: Dimensions) => {
-        if (err) {
-          const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-          const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-          return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-        }
-        const Logger = require(input.__dirname + "/../../Logger").Logger;
-        Logger.silly("[GMThRenderer] rendering thumbnail:", input.imagePath);
-
-
-        /**
-         * newWidth * newHeight = size*size
-         * newHeight/newWidth = height/width
-         *
-         * newHeight = (height/width)*newWidth
-         * newWidth * newWidth = (size*size) / (height/width)
-         *
-         * @type {number}
-         */
-        try {
-          const ratio = value.height / value.width;
-          const filter = input.qualityPriority == true ? 'Lanczos' : 'Point';
-          image.filter(filter);
-
-          if (input.makeSquare == false) {
-            const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
-            image = image.resize(newWidth);
-          } else {
-            image = image.resize(input.size, input.size)
-              .crop(input.size, input.size);
-          }
-          image.write(input.thPath, (err) => {
-            if (err) {
-
-              const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-              const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-              return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-            }
-            return done();
-          });
-        } catch (err) {
-          const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-          const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
-          return done(new Error(ErrorCodes.GENERAL_ERROR, err));
-        }
-
-      });
-
-
-  };
-}

backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts

@@ -9,16 +9,16 @@ import {ContentWrapper} from "../../../common/entities/ConentWrapper";
 import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
 import {ProjectPath} from "../../ProjectPath";
 import {PhotoDTO} from "../../../common/entities/PhotoDTO";
-import {ThumbnailRenderers} from "./THRenderers";
 import {Config} from "../../../common/config/private/Config";
 import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
-import RendererInput = ThumbnailRenderers.RendererInput;
+import {ThumbnailTH} from "../../model/threading/ThreadPool";
+import {RendererFactory, RendererInput} from "../../model/threading/ThumbnailWoker";
 
 
 export class ThumbnailGeneratorMWs {
   private static initDone = false;
-  private static ThumbnailFunction = null;
+  private static ThumbnailFunction: (input: RendererInput) => Promise<void> = null;
-  private static thPool = null;
+  private static threadPool: ThumbnailTH = null;
 
   public static init() {
     if (this.initDone == true) {
@@ -26,28 +26,18 @@ export class ThumbnailGeneratorMWs {
     }
 
 
-    Config.Client.concurrentThumbnailGenerations = 1;
+    if (Config.Server.enableThreading == true ||
-    switch (Config.Server.thumbnail.processingLibrary) {
+      Config.Server.thumbnail.processingLibrary != ThumbnailProcessingLib.Jimp) {
-      case ThumbnailProcessingLib.Jimp:
+      Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
-        this.ThumbnailFunction = ThumbnailRenderers.jimp;
+    } else {
-        break;
+      Config.Client.concurrentThumbnailGenerations = 1;
-      case ThumbnailProcessingLib.gm:
-        this.ThumbnailFunction = ThumbnailRenderers.gm;
-        break;
-      case ThumbnailProcessingLib.sharp:
-        this.ThumbnailFunction = ThumbnailRenderers.sharp;
-
-        break;
-      default:
-        throw "Unknown thumbnail processing lib";
     }
 
+    this.ThumbnailFunction = RendererFactory.build(Config.Server.thumbnail.processingLibrary);
+
     if (Config.Server.enableThreading == true &&
       Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.Jimp) {
-      Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
+      this.threadPool = new ThumbnailTH(Config.Client.concurrentThumbnailGenerations);
-      const Pool = require('threads').Pool;
-      this.thPool = new Pool(Config.Client.concurrentThumbnailGenerations);
-      this.thPool.run(this.ThumbnailFunction);
     }
 
     this.initDone = true;
@@ -138,8 +128,7 @@ export class ThumbnailGeneratorMWs {
   }
 
 
-  private static generateImage(imagePath: string, size: number, makeSquare: boolean, req: Request, res: Response, next: NextFunction) {
+  private static async generateImage(imagePath: string, size: number, makeSquare: boolean, req: Request, res: Response, next: NextFunction) {
-    ThumbnailGeneratorMWs.init();
     //generate thumbnail path
     let thPath = path.join(ProjectPath.ThumbnailFolder, ThumbnailGeneratorMWs.generateThumbnailName(imagePath, size));
 
@@ -157,30 +146,24 @@ export class ThumbnailGeneratorMWs {
     }
 
     //run on other thread
-
     let input = <RendererInput>{
       imagePath: imagePath,
       size: size,
       thPath: thPath,
       makeSquare: makeSquare,
-      qualityPriority: Config.Server.thumbnail.qualityPriority,
+      qualityPriority: Config.Server.thumbnail.qualityPriority
-      __dirname: __dirname,
     };
-    if (this.thPool !== null) {
+    try {
-      this.thPool.send(input)
+      if (this.threadPool !== null) {
-        .on('done', (out) => {
+        await this.threadPool.execute(input);
-          return next(out);
+        return next();
-        }).on('error', (error) => {
+      } else {
-        console.log(error);
+        await ThumbnailGeneratorMWs.ThumbnailFunction(input);
-        return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
+        return next();
-      });
-    } else {
-      try {
-        ThumbnailGeneratorMWs.ThumbnailFunction(input, out => next(out));
-      } catch (error) {
-        console.log(error);
-        return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
       }
+    } catch (error) {
+      console.log(error);
+      return next(new Error(ErrorCodes.THUMBNAIL_GENERATION_ERROR, error));
     }
   }
 

backend/model/DiskManger.ts

@@ -1,66 +1,44 @@
 ///<reference path="exif.d.ts"/>
-import * as path from "path";
 import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
-import {ProjectPath} from "../ProjectPath";
 import {Logger} from "../Logger";
-import {diskManagerTask, DiskManagerTask} from "./DiskMangerTask";
 import {Config} from "../../common/config/private/Config";
+import {DiskManagerTH} from "./threading/ThreadPool";
+import {DiskMangerWorker} from "./threading/DiskMangerWorker";
 
-const Pool = require('threads').Pool;
-const pool = new Pool(1);
 
 const LOG_TAG = "[DiskManager]";
 
 
-pool.run(diskManagerTask);
-
 export class DiskManager {
-  public static scanDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
+  static threadPool: DiskManagerTH = null;
-    return new Promise((resolve, reject) => {
-      Logger.silly(LOG_TAG, "scanning directory:", relativeDirectoryName);
-      let directoryName = path.basename(relativeDirectoryName);
-      let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
-      let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
 
-      let input = <DiskManagerTask.PoolInput>{
+  public static init() {
-        relativeDirectoryName,
+    if (Config.Server.enableThreading == true) {
-        directoryName,
+      DiskManager.threadPool = new DiskManagerTH(1);
-        directoryParent,
+    }
-        absoluteDirectoryName
+  }
-      };
 
-      let done = (error: any, result: DirectoryDTO) => {
+  public static async scanDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
-        if (error || !result) {
+    Logger.silly(LOG_TAG, "scanning directory:", relativeDirectoryName);
-          return reject(error);
-        }
-
-        let addDirs = (dir: DirectoryDTO) => {
-          dir.photos.forEach((ph) => {
-            ph.directory = dir;
-          });
-          dir.directories.forEach((d) => {
-            addDirs(d);
-          });
-        };
-        addDirs(result);
-        return resolve(result);
-      };
-
-      let error = (error) => {
-        return reject(error);
-      };
 
 
-      if (Config.Server.enableThreading == true) {
+    let directory: DirectoryDTO = null;
-        pool.send(input).on('done', done).on('error', error);
+
-      } else {
+    if (Config.Server.enableThreading == true) {
-        try {
+      directory = await DiskManager.threadPool.execute(relativeDirectoryName);
-          diskManagerTask(input, done);
+    } else {
-        } catch (err) {
+      directory = await DiskMangerWorker.scanDirectory(relativeDirectoryName);
-          error(err);
+    }
-        }
+    let addDirs = (dir: DirectoryDTO) => {
-      }
+      dir.photos.forEach((ph) => {
-    });
+        ph.directory = dir;
+      });
+      dir.directories.forEach((d) => {
+        addDirs(d);
+      });
+    };
+    addDirs(directory);
+    return directory;
   }
 
 }

backend/model/DiskMangerTask.ts

@@ -1,202 +0,0 @@
-///<reference path="exif.d.ts"/>
-import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
-import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../common/entities/PhotoDTO";
-import {Logger} from "../Logger";
-
-const LOG_TAG = "[DiskManagerTask]";
-
-export const diskManagerTask = (input: DiskManagerTask.PoolInput, done) => {
-    const fs = require("fs");
-    const path = require("path");
-    const mime = require("mime");
-    const iptc = require("node-iptc");
-    const exif_parser = require("exif-parser");
-
-
-    let isImage = (fullPath: string) => {
-      let imageMimeTypes = [
-        'image/bmp',
-        'image/gif',
-        'image/jpeg',
-        'image/png',
-        'image/pjpeg',
-        'image/tiff',
-        'image/webp',
-        'image/x-tiff',
-        'image/x-windows-bmp'
-      ];
-
-      let extension = mime.lookup(fullPath);
-
-      return imageMimeTypes.indexOf(extension) !== -1;
-    };
-
-    let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
-        return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
-            fs.readFile(fullPath, function (err, data) {
-              if (err) {
-                return reject({file: fullPath, error: err});
-              }
-              const metadata: PhotoMetadata = <PhotoMetadata>{
-                keywords: {},
-                cameraData: {},
-                positionData: null,
-                size: {},
-                creationDate: {}
-              };
-
-              try {
-
-                try {
-                  const exif = exif_parser.create(data).parse();
-                  metadata.cameraData = <CameraMetadata> {
-                    ISO: exif.tags.ISO,
-                    model: exif.tags.Modeol,
-                    maker: exif.tags.Make,
-                    fStop: exif.tags.FNumber,
-                    exposure: exif.tags.ExposureTime,
-                    focalLength: exif.tags.FocalLength,
-                    lens: exif.tags.LensModel,
-                  };
-                  if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
-                    metadata.positionData = metadata.positionData || {};
-                    metadata.positionData.GPSData = <GPSMetadata> {
-                      latitude: exif.tags.GPSLatitude,
-                      longitude: exif.tags.GPSLongitude,
-                      altitude: exif.tags.GPSAltitude
-                    };
-                  }
-
-                  metadata.size = <ImageSize> {width: exif.imageSize.width, height: exif.imageSize.height};
-                } catch (err) {
-                  Logger.info(LOG_TAG, "Error parsing exif", fullPath);
-                  metadata.size = <ImageSize> {width: 1, height: 1};
-                }
-
-                try {
-
-                  const iptcData = iptc(data);
-                  //Decode characters to UTF8
-                  const decode = (s: any) => {
-                    for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
-                         ((a = s[i][c](0)) & 0x80) &&
-                         (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
-                           o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
-                    );
-                    return s.join("");
-                  };
-
-                  if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) {
-                    metadata.positionData = metadata.positionData || {};
-                    metadata.positionData.country = iptcData.country_or_primary_location_name;
-                    metadata.positionData.state = iptcData.province_or_state;
-                    metadata.positionData.city = iptcData.city;
-                  }
-
-
-                  metadata.keywords = <string[]> (iptcData.keywords || []).map((s: string) => decode(s));
-                  metadata.creationDate = <number> iptcData.date_time ? iptcData.date_time.getTime() : 0;
-                } catch (err) {
-                  Logger.info(LOG_TAG, "Error parsing iptc data", fullPath);
-                }
-
-
-                return resolve(metadata);
-              } catch (err) {
-                return reject({file: fullPath, error: err});
-              }
-            });
-          }
-        );
-      }
-    ;
-
-    let parseDir = (directoryInfo: {
-      relativeDirectoryName: string,
-      directoryName: string,
-      directoryParent: string,
-      absoluteDirectoryName: string
-    }, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> => {
-      return new Promise<DirectoryDTO>((resolve, reject) => {
-        let promises: Array<Promise<any>> = [];
-        let directory = <DirectoryDTO>{
-          name: directoryInfo.directoryName,
-          path: directoryInfo.directoryParent,
-          lastUpdate: Date.now(),
-          directories: [],
-          photos: []
-        };
-        fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => {
-
-          if (err) {
-            return reject(err);
-          }
-
-          try {
-            for (let i = 0; i < list.length; i++) {
-              let file = list[i];
-              let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file));
-              if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
-                let promise = parseDir({
-                    relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep),
-                    directoryName: file,
-                    directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep),
-                    absoluteDirectoryName: fullFilePath
-                  },
-                  5, true
-                ).then((dir) => {
-                  directory.directories.push(dir);
-                });
-                promises.push(promise);
-              } else if (isImage(fullFilePath)) {
-
-                let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
-                  directory.photos.push(<PhotoDTO>{
-                    name: file,
-                    directory: null,
-                    metadata: photoMetadata
-                  });
-                });
-                promises.push(promise);
-
-                if (maxPhotos != null && promises.length > maxPhotos) {
-                  break;
-                }
-              }
-            }
-
-            Promise.all(promises).then(() => {
-              return resolve(directory);
-            }).catch((err) => {
-              return reject({directoryInfo: directoryInfo, error: err});
-            });
-          } catch (err) {
-            return reject({directoryInfo: directoryInfo, error: err});
-          }
-
-        });
-
-      });
-    };
-
-
-    parseDir(input).then((dir) => {
-      done(null, dir);
-    }).catch((err) => {
-      done(err, null);
-    });
-
-  }
-;
-
-
-export module DiskManagerTask {
-
-  export  interface PoolInput {
-    relativeDirectoryName: string;
-    directoryName: string;
-    directoryParent: string;
-    absoluteDirectoryName: string;
-  }
-
-}

backend/model/threading/DiskMangerWorker.ts

@@ -0,0 +1,162 @@
+///<reference path="../exif.d.ts"/>
+import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../../common/entities/PhotoDTO";
+import {Logger} from "../../Logger";
+import * as fs from "fs";
+import * as path from "path";
+import * as  mime from "mime";
+import * as iptc from "node-iptc";
+import * as exif_parser from "exif-parser";
+import {ProjectPath} from "../../ProjectPath";
+
+const LOG_TAG = "[DiskManagerTask]";
+
+export class DiskMangerWorker {
+  private static isImage(fullPath: string) {
+    let imageMimeTypes = [
+      'image/bmp',
+      'image/gif',
+      'image/jpeg',
+      'image/png',
+      'image/pjpeg',
+      'image/tiff',
+      'image/webp',
+      'image/x-tiff',
+      'image/x-windows-bmp'
+    ];
+
+    let extension = mime.lookup(fullPath);
+
+    return imageMimeTypes.indexOf(extension) !== -1;
+  }
+
+  private static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
+    return new Promise<PhotoMetadata>((resolve, reject) => {
+        fs.readFile(fullPath, function (err, data) {
+          if (err) {
+            return reject({file: fullPath, error: err});
+          }
+          const metadata: PhotoMetadata = <PhotoMetadata>{
+            keywords: {},
+            cameraData: {},
+            positionData: null,
+            size: {},
+            creationDate: 0
+          };
+
+          try {
+
+            try {
+              const exif = exif_parser.create(data).parse();
+              metadata.cameraData = <CameraMetadata> {
+                ISO: exif.tags.ISO,
+                model: exif.tags.Modeol,
+                maker: exif.tags.Make,
+                fStop: exif.tags.FNumber,
+                exposure: exif.tags.ExposureTime,
+                focalLength: exif.tags.FocalLength,
+                lens: exif.tags.LensModel,
+              };
+              if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
+                metadata.positionData = metadata.positionData || {};
+                metadata.positionData.GPSData = <GPSMetadata> {
+                  latitude: exif.tags.GPSLatitude,
+                  longitude: exif.tags.GPSLongitude,
+                  altitude: exif.tags.GPSAltitude
+                };
+              }
+
+              metadata.size = <ImageSize> {width: exif.imageSize.width, height: exif.imageSize.height};
+            } catch (err) {
+              Logger.info(LOG_TAG, "Error parsing exif", fullPath);
+              metadata.size = <ImageSize> {width: 1, height: 1};
+            }
+
+            try {
+
+              const iptcData = iptc(data);
+              //Decode characters to UTF8
+              const decode = (s: any) => {
+                for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
+                     ((a = s[i][c](0)) & 0x80) &&
+                     (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
+                       o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
+                );
+                return s.join("");
+              };
+
+              if (iptcData.country_or_primary_location_name || iptcData.province_or_state || iptcData.city) {
+                metadata.positionData = metadata.positionData || {};
+                metadata.positionData.country = iptcData.country_or_primary_location_name;
+                metadata.positionData.state = iptcData.province_or_state;
+                metadata.positionData.city = iptcData.city;
+              }
+
+
+              metadata.keywords = <string[]> (iptcData.keywords || []).map((s: string) => decode(s));
+              metadata.creationDate = <number> iptcData.date_time ? iptcData.date_time.getTime() : 0;
+            } catch (err) {
+              Logger.info(LOG_TAG, "Error parsing iptc data", fullPath);
+            }
+
+
+            return resolve(metadata);
+          } catch (err) {
+            return reject({file: fullPath, error: err});
+          }
+        });
+      }
+    );
+  }
+
+  public static  scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> {
+    return new Promise<DirectoryDTO>((resolve, reject) => {
+
+      const directoryName = path.basename(relativeDirectoryName);
+      const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
+      const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
+
+      // let promises: Array<Promise<any>> = [];
+      let directory = <DirectoryDTO>{
+        name: directoryName,
+        path: directoryParent,
+        lastUpdate: Date.now(),
+        directories: [],
+        photos: []
+      };
+      fs.readdir(absoluteDirectoryName, async (err, list) => {
+        if (err) {
+          return reject(err);
+        }
+
+        try {
+          for (let i = 0; i < list.length; i++) {
+            let file = list[i];
+            let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
+            if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
+              directory.directories.push(await DiskMangerWorker.scanDirectory(path.join(relativeDirectoryName, file),
+                5, true
+              ));
+            } else if (DiskMangerWorker.isImage(fullFilePath)) {
+              directory.photos.push(<PhotoDTO>{
+                name: file,
+                directory: null,
+                metadata: await DiskMangerWorker.loadPhotoMetadata(fullFilePath)
+              });
+
+              if (maxPhotos != null && directory.photos.length > maxPhotos) {
+                break;
+              }
+            }
+          }
+
+          return resolve(directory);
+        } catch (err) {
+          return reject({error: err});
+        }
+
+      });
+    });
+
+  }
+}

backend/model/threading/ThreadPool.ts

@@ -0,0 +1,110 @@
+import * as cluster from "cluster";
+import {Logger} from "../../Logger";
+import {DiskManagerTask, ThumbnailTask, WorkerMessage, WorkerTask, WorkerTaskTypes} from "./Worker";
+import {DirectoryDTO} from "../../../common/entities/DirectoryDTO";
+import {RendererInput} from "./ThumbnailWoker";
+import {Config} from "../../../common/config/private/Config";
+
+
+interface PoolTask {
+  task: WorkerTask;
+  promise: { resolve: Function, reject: Function };
+}
+
+interface WorkerWrapper {
+  worker: cluster.Worker;
+  poolTask: PoolTask;
+}
+
+export class ThreadPool {
+
+  public static WorkerCount = 0;
+  private workers: WorkerWrapper[] = [];
+  private tasks: PoolTask[] = [];
+
+  constructor(private size: number) {
+    Logger.silly("Creating thread pool with", size, "workers");
+    for (let i = 0; i < size; i++) {
+      this.startWorker();
+
+    }
+  }
+
+  private startWorker() {
+    const worker = <WorkerWrapper>{poolTask: null, worker: cluster.fork()};
+    this.workers.push(worker);
+    worker.worker.on('online', () => {
+      ThreadPool.WorkerCount++;
+      Logger.debug('Worker ' + worker.worker.process.pid + ' is online, worker count:', ThreadPool.WorkerCount);
+    });
+    worker.worker.on('exit', (code, signal) => {
+      ThreadPool.WorkerCount--;
+      Logger.warn('Worker ' + worker.worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal + ", worker count:", ThreadPool.WorkerCount);
+      Logger.debug('Starting a new worker');
+      this.startWorker();
+    });
+
+    worker.worker.on("message", (msg: WorkerMessage) => {
+      if (worker.poolTask == null) {
+        throw "No worker task after worker task is completed"
+      }
+      if (msg.error) {
+        worker.poolTask.promise.reject(msg.error);
+      } else {
+        worker.poolTask.promise.resolve(msg.result);
+      }
+      worker.poolTask = null;
+      this.run();
+    });
+  }
+
+  protected executeTask<T>(task: WorkerTask): Promise<T> {
+    return new Promise((resolve: Function, reject: Function) => {
+      this.tasks.push({task: task, promise: {resolve: resolve, reject: reject}});
+      this.run();
+    });
+  }
+
+  private getFreeWorker() {
+    for (let i = 0; i < this.workers.length; i++) {
+      if (this.workers[i].poolTask == null) {
+        return this.workers[i];
+      }
+    }
+    return null;
+  }
+
+  private run = () => {
+    if (this.tasks.length == 0) {
+      return;
+    }
+    const worker = this.getFreeWorker();
+    if (worker == null) {
+      return;
+    }
+
+    const poolTask = this.tasks.pop();
+    worker.poolTask = poolTask;
+    worker.worker.send(poolTask.task);
+  };
+
+}
+
+export class DiskManagerTH extends ThreadPool {
+  execute(relativeDirectoryName: string): Promise<DirectoryDTO> {
+    return super.executeTask(<DiskManagerTask>{
+      type: WorkerTaskTypes.diskManager,
+      relativeDirectoryName: relativeDirectoryName
+    });
+  }
+}
+
+export class ThumbnailTH extends ThreadPool {
+  execute(input: RendererInput): Promise<void> {
+    return super.executeTask(<ThumbnailTask>{
+      type: WorkerTaskTypes.thumbnail,
+      input: input,
+      renderer: Config.Server.thumbnail.processingLibrary
+    });
+  }
+}

backend/model/threading/ThumbnailWoker.ts

@@ -0,0 +1,170 @@
+import {Metadata, SharpInstance} from "sharp";
+import {Dimensions, State} from "gm";
+import {Logger} from "../../Logger";
+import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
+
+export class ThumbnailWoker {
+
+  private static renderer: (input: RendererInput) => Promise<void> = null;
+  private static rendererType = null;
+
+  public static render(input: RendererInput, renderer: ThumbnailProcessingLib): Promise<void> {
+    if (ThumbnailWoker.rendererType != renderer) {
+      ThumbnailWoker.renderer = RendererFactory.build(renderer);
+      ThumbnailWoker.rendererType = renderer;
+    }
+    return ThumbnailWoker.renderer(input);
+  }
+
+
+}
+
+
+export interface RendererInput {
+  imagePath: string;
+  size: number;
+  makeSquare: boolean;
+  thPath: string;
+  qualityPriority: boolean
+}
+
+export class RendererFactory {
+
+  public static build(renderer: ThumbnailProcessingLib): (input: RendererInput) => Promise<void> {
+    switch (renderer) {
+      case ThumbnailProcessingLib.Jimp:
+        return RendererFactory.Jimp();
+      case ThumbnailProcessingLib.gm:
+        return RendererFactory.Gm();
+      case ThumbnailProcessingLib.sharp:
+        return RendererFactory.Sharp();
+    }
+    throw "unknown renderer"
+  }
+
+  public static Jimp() {
+    const Jimp = require("jimp");
+    return async (input: RendererInput): Promise<void> => {
+      //generate thumbnail
+      Logger.silly("[JimpThRenderer] rendering thumbnail:", input.imagePath);
+      const image = await Jimp.read(input.imagePath);
+      /**
+       * newWidth * newHeight = size*size
+       * newHeight/newWidth = height/width
+       *
+       * newHeight = (height/width)*newWidth
+       * newWidth * newWidth = (size*size) / (height/width)
+       *
+       * @type {number}
+       */
+      const ratio = image.bitmap.height / image.bitmap.width;
+      const algo = input.qualityPriority == true ? Jimp.RESIZE_BEZIER : Jimp.RESIZE_NEAREST_NEIGHBOR;
+      if (input.makeSquare == false) {
+        let newWidth = Math.sqrt((input.size * input.size) / ratio);
+
+        image.resize(newWidth, Jimp.AUTO, algo);
+      } else {
+        image.resize(input.size / Math.min(ratio, 1), Jimp.AUTO, algo);
+        image.crop(0, 0, input.size, input.size);
+      }
+      image.quality(60);        // set JPEG quality
+
+      await new Promise((resolve, reject) => {
+        image.write(input.thPath, (err) => { // save
+          if (err) {
+            return reject(err);
+          }
+          resolve();
+        });
+      });
+
+    };
+  }
+
+
+  public static Sharp() {
+    const sharp = require("sharp");
+    return async (input: RendererInput): Promise<void> => {
+
+      Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath);
+      const image: SharpInstance = sharp(input.imagePath);
+      const metadata: Metadata = await image.metadata();
+
+      /**
+       * newWidth * newHeight = size*size
+       * newHeight/newWidth = height/width
+       *
+       * newHeight = (height/width)*newWidth
+       * newWidth * newWidth = (size*size) / (height/width)
+       *
+       * @type {number}
+       */
+      const ratio = metadata.height / metadata.width;
+      const kernel = input.qualityPriority == true ? sharp.kernel.lanczos3 : sharp.kernel.nearest;
+      const interpolator = input.qualityPriority == true ? sharp.interpolator.bicubic : sharp.interpolator.nearest;
+      if (input.makeSquare == false) {
+        const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
+        image.resize(newWidth, null, {
+          kernel: kernel,
+          interpolator: interpolator
+        });
+
+      } else {
+        image
+          .resize(input.size, input.size, {
+            kernel: kernel,
+            interpolator: interpolator
+          })
+          .crop(sharp.strategy.center);
+      }
+      await image.jpeg().toFile(input.thPath);
+    };
+  }
+
+
+  public static Gm() {
+    const gm = require("gm");
+    return (input: RendererInput): Promise<void> => {
+      return new Promise((resolve, reject) => {
+        Logger.silly("[GMThRenderer] rendering thumbnail:", input.imagePath);
+        let image: State = gm(input.imagePath);
+        image.size((err, value: Dimensions) => {
+          if (err) {
+            return reject(err);
+          }
+
+          /**
+           * newWidth * newHeight = size*size
+           * newHeight/newWidth = height/width
+           *
+           * newHeight = (height/width)*newWidth
+           * newWidth * newWidth = (size*size) / (height/width)
+           *
+           * @type {number}
+           */
+          try {
+            const ratio = value.height / value.width;
+            const filter = input.qualityPriority == true ? 'Lanczos' : 'Point';
+            image.filter(filter);
+
+            if (input.makeSquare == false) {
+              const newWidth = Math.round(Math.sqrt((input.size * input.size) / ratio));
+              image = image.resize(newWidth);
+            } else {
+              image = image.resize(input.size, input.size)
+                .crop(input.size, input.size);
+            }
+            image.write(input.thPath, (err) => {
+              if (err) {
+                return reject(err);
+              }
+              return resolve();
+            });
+          } catch (err) {
+            return reject(err);
+          }
+        });
+      });
+    };
+  }
+}

backend/model/threading/Worker.ts

@@ -0,0 +1,57 @@
+import {DiskMangerWorker} from "./DiskMangerWorker";
+import {Logger} from "../../Logger";
+import {RendererInput, ThumbnailWoker} from "./ThumbnailWoker";
+import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateConfig";
+export class Worker {
+
+
+  public static process() {
+    Logger.debug("Worker is waiting for tasks");
+    process.on('message', async (task: WorkerTask) => {
+      try {
+        let result = null;
+        switch (task.type) {
+          case WorkerTaskTypes.diskManager:
+            result = await DiskMangerWorker.scanDirectory((<DiskManagerTask>task).relativeDirectoryName);
+            break;
+          case WorkerTaskTypes.thumbnail:
+            result = await ThumbnailWoker.render((<ThumbnailTask>task).input, (<ThumbnailTask>task).renderer);
+            break;
+          default:
+            Logger.error("Unknown worker task type");
+            throw "Unknown worker task type";
+        }
+        process.send(<WorkerMessage>{
+          error: null,
+          result: result
+        });
+      } catch (err) {
+        process.send({error: err, result: null});
+      }
+
+    });
+  }
+}
+
+
+export enum WorkerTaskTypes{
+  thumbnail, diskManager
+}
+
+export interface WorkerTask {
+  type: WorkerTaskTypes;
+}
+
+export interface DiskManagerTask extends WorkerTask {
+  relativeDirectoryName: string;
+}
+
+export interface ThumbnailTask extends WorkerTask {
+  input: RendererInput;
+  renderer: ThumbnailProcessingLib;
+}
+
+export interface WorkerMessage {
+  error: any;
+  result: any;
+}

backend/server.ts

@@ -14,20 +14,26 @@ import {Config} from "../common/config/private/Config";
 import {DatabaseType, ThumbnailProcessingLib} from "../common/config/private/IPrivateConfig";
 import {LoggerRouter} from "./routes/LoggerRouter";
 import {ProjectPath} from "./ProjectPath";
+import {ThumbnailGeneratorMWs} from "./middlewares/thumbnail/ThumbnailGeneratorMWs";
+import {DiskManager} from "./model/DiskManger";
 
 const LOG_TAG = "[server]";
 export class Server {
 
-  private debug: any;
   private app: any;
   private server: any;
 
   constructor() {
+    if (process.env.DEBUG) {
+      Logger.debug(LOG_TAG, "Running in DEBUG mode");
+    }
     this.init();
   }
 
   async init() {
-    Logger.info(LOG_TAG, "config:");
+    Logger.info(LOG_TAG, "running diagnostics...");
+    await this.runDiagnostics();
+    Logger.info(LOG_TAG, "using config:");
     Logger.info(LOG_TAG, JSON.stringify(Config, null, '\t'));
 
     this.app = _express();
@@ -57,59 +63,8 @@ export class Server {
     // for parsing application/json
     this.app.use(_bodyParser.json());
 
-    if (Config.Server.database.type == DatabaseType.mysql) {
+    DiskManager.init();
-      try {
+    ThumbnailGeneratorMWs.init();
-        await ObjectManagerRepository.InitMySQLManagers();
-      } catch (err) {
-        Logger.warn(LOG_TAG, "[MYSQL error]", err);
-        Logger.warn(LOG_TAG, "Error during initializing mysql falling back to memory DB");
-        Config.setDatabaseType(DatabaseType.memory);
-        await ObjectManagerRepository.InitMemoryManagers();
-      }
-    } else {
-      await ObjectManagerRepository.InitMemoryManagers();
-    }
-
-    if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.sharp) {
-      try {
-        const sharp = require("sharp");
-        sharp();
-
-      } catch (err) {
-        Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] sharp module error: ", err);
-
-        Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
-          " 'Sharp' node module is not found." +
-          " Falling back to JS based thumbnail generation");
-        Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
-      }
-    }
-
-
-    if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.gm) {
-      try {
-        const gm = require("gm");
-        gm(ProjectPath.FrontendFolder + "/assets/icon.png").size((err, value) => {
-          console.log(err, value);
-          if (!err) {
-            return;
-          }
-          Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
-          Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
-            " 'gm' node module is not found." +
-            " Falling back to JS based thumbnail generation");
-          Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
-        });
-
-      } catch (err) {
-        Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
-
-        Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
-          " 'gm' node module is not found." +
-          " Falling back to JS based thumbnail generation");
-        Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
-      }
-    }
 
     PublicRouter.route(this.app);
 
@@ -135,6 +90,61 @@ export class Server {
 
   }
 
+  async runDiagnostics() {
+
+
+    if (Config.Server.database.type == DatabaseType.mysql) {
+      try {
+        await ObjectManagerRepository.InitMySQLManagers();
+      } catch (err) {
+        Logger.warn(LOG_TAG, "[MYSQL error]", err);
+        Logger.warn(LOG_TAG, "Error during initializing mysql falling back to memory DB");
+        Config.setDatabaseType(DatabaseType.memory);
+        await ObjectManagerRepository.InitMemoryManagers();
+      }
+    } else {
+      await ObjectManagerRepository.InitMemoryManagers();
+    }
+
+    if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.sharp) {
+      try {
+        const sharp = require("sharp");
+        sharp();
+
+      } catch (err) {
+        Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] sharp module error: ", err);
+        Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
+          " 'Sharp' node module is not found." +
+          " Falling back to JS based thumbnail generation");
+        Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
+      }
+    }
+
+
+    if (Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.gm) {
+      try {
+        const gm = require("gm");
+        gm(ProjectPath.FrontendFolder + "/assets/icon.png").size((err, value) => {
+          if (!err) {
+            return;
+          }
+          Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
+          Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
+            " 'gm' node module is not found." +
+            " Falling back to JS based thumbnail generation");
+          Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
+        });
+
+      } catch (err) {
+        Logger.warn(LOG_TAG, "[Thumbnail hardware acceleration] gm module error: ", err);
+        Logger.warn(LOG_TAG, "Thumbnail hardware acceleration is not possible." +
+          " 'gm' node module is not found." +
+          " Falling back to JS based thumbnail generation");
+        Config.Server.thumbnail.processingLibrary = ThumbnailProcessingLib.Jimp;
+      }
+    }
+  }
+
 
   /**
    * Event listener for HTTP server "error" event.
@@ -178,8 +188,7 @@ export class Server {
 }
 
 
-if (process.env.DEBUG) {
-  Logger.debug(LOG_TAG, "Running in DEBUG mode");
-}
 
-new Server();
+
+
+

frontend/app/gallery/thumnailLoader.service.ts

@@ -96,7 +96,7 @@ export class ThumbnailLoaderService {
 
     let thumbnailTaskEntity = {priority: priority, listener: listener, parentTask: thTask};
 
-    //add to task
+    //add to poolTask
     thTask.taskEntities.push(thumbnailTaskEntity);
     if (thTask.inProgress == true) {
       listener.onStartedLoading();
@@ -144,7 +144,7 @@ export class ThumbnailLoaderService {
     let i = this.que.indexOf(task);
     if (i == -1) {
       if (task.taskEntities.length !== 0) {
-        console.error("ThumbnailLoader: can't find task to remove");
+        console.error("ThumbnailLoader: can't find poolTask to remove");
       }
       return;
     }

package.json

@@ -5,13 +5,13 @@
   "author": "Patrik J. Braun",
   "homepage": "https://github.com/bpatrik/PiGallery2",
   "license": "MIT",
-  "main": "./backend/server.js",
+  "main": "./backend/index.js",
-  "bin": "./backend/server.js",
+  "bin": "./backend/index.js",
   "scripts": {
     "build": "ng build",
     "pretest": "tsc",
     "test": "ng test --single-run && mocha --recursive test/backend/unit",
-    "start": "node ./backend/server",
+    "start": "node ./backend/index",
     "ng": "ng",
     "lint": "ng lint",
     "e2e": "ng e2e"
@@ -35,43 +35,39 @@
     "mysql": "^2.13.0",
     "node-iptc": "^1.0.4",
     "reflect-metadata": "^0.1.10",
-    "threads": "^0.8.1",
     "typeconfig": "^1.0.1",
     "typeorm": "0.0.11",
     "winston": "^2.3.1"
   },
   "devDependencies": {
+    "@agm/core": "^1.0.0-beta.0",
     "@angular/cli": "1.2.0",
+    "@angular/common": "~4.2.5",
+    "@angular/compiler": "~4.2.5",
     "@angular/compiler-cli": "^4.2.5",
+    "@angular/core": "~4.2.5",
+    "@angular/forms": "~4.2.5",
+    "@angular/http": "~4.2.5",
     "@angular/language-service": "^4.2.5",
+    "@angular/platform-browser": "~4.2.5",
+    "@angular/platform-browser-dynamic": "~4.2.5",
+    "@angular/router": "~4.2.5",
     "@types/express": "^4.0.36",
     "@types/express-session": "1.15.0",
     "@types/gm": "^1.17.31",
     "@types/jasmine": "^2.5.53",
+    "@types/jimp": "^0.2.1",
     "@types/node": "^8.0.7",
     "@types/sharp": "^0.17.2",
-    "ng2-cookies": "^1.0.12",
-    "ng2-slim-loading-bar": "^4.0.0",
-    "intl": "^1.2.5",
-    "core-js": "^2.4.1",
-    "zone.js": "^0.8.12",
-    "rxjs": "^5.4.1",
     "@types/winston": "^2.3.3",
-    "@agm/core": "^1.0.0-beta.0",
-    "@angular/common": "~4.2.5",
-    "@angular/compiler": "~4.2.5",
-    "@angular/core": "~4.2.5",
-    "@angular/forms": "~4.2.5",
-    "@angular/http": "~4.2.5",
-    "@angular/platform-browser": "~4.2.5",
-    "@angular/platform-browser-dynamic": "~4.2.5",
-    "@angular/router": "~4.2.5",
     "chai": "^4.0.2",
     "codelyzer": "~3.1.1",
+    "core-js": "^2.4.1",
     "ejs-loader": "^0.3.0",
     "gulp": "^3.9.1",
     "gulp-typescript": "^3.1.7",
     "gulp-zip": "^4.0.0",
+    "intl": "^1.2.5",
     "jasmine-core": "^2.6.4",
     "jasmine-spec-reporter": "~4.1.1",
     "karma": "^1.7.0",
@@ -84,15 +80,19 @@
     "karma-systemjs": "^0.16.0",
     "merge2": "^1.1.0",
     "mocha": "^3.4.2",
+    "ng2-cookies": "^1.0.12",
+    "ng2-slim-loading-bar": "^4.0.0",
     "phantomjs-prebuilt": "^2.1.14",
     "protractor": "^5.1.2",
     "remap-istanbul": "^0.9.5",
     "rimraf": "^2.6.1",
     "run-sequence": "^2.0.0",
+    "rxjs": "^5.4.1",
     "ts-helpers": "^1.1.2",
     "ts-node": "~3.1.0",
     "tslint": "^5.4.3",
-    "typescript": "^2.4.1"
+    "typescript": "^2.4.1",
+    "zone.js": "^0.8.12"
   },
   "optionalDependencies": {
     "gm": "^1.23.0",