improving thumbnail generation and directory parsing

backend/middlewares/thumbnail/THRenderers.ts

@@ -1,5 +1,7 @@
 import {Metadata, SharpInstance} from "@types/sharp";
 import {Dimensions, State} from "@types/gm";
+import {Logger} from "../../Logger";
+import {Error, ErrorCodes} from "../../../common/entities/Error";
 
 export module ThumbnailRenderers {
 
@@ -60,7 +62,7 @@ export module ThumbnailRenderers {
       .metadata()
       .then((metadata: Metadata) => {
 
-        const Logger = require(input.__dirname + "/../../Logger").Logger;
+        //   const Logger = require(input.__dirname + "/../../Logger").Logger;
         Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath);
         /**
          * newWidth * newHeight = size*size
@@ -95,13 +97,13 @@ export module ThumbnailRenderers {
             .toFile(input.thPath).then(() => {
             return done();
           }).catch(function (err) {
-            const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
+            //      const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-            const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
+            //       const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
             return done(new Error(ErrorCodes.GENERAL_ERROR, err));
           });
         } catch (err) {
-          const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
+          //  const Error = require(input.__dirname + "/../../../common/entities/Error").Error;
-          const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
+          // const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes;
           return done(new Error(ErrorCodes.GENERAL_ERROR, err));
         }
       });

backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts

@@ -15,20 +15,19 @@ import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateCon
 import RendererInput = ThumbnailRenderers.RendererInput;
 
 
-Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
-
-const Pool = require('threads').Pool;
-const pool = new Pool(Config.Client.concurrentThumbnailGenerations);
-
-
 export class ThumbnailGeneratorMWs {
-  private static poolsInited = false;
+  private static initDone = false;
-  private static ThumbnailFunction = null
+  private static ThumbnailFunction = null;
+  private static thPool = null;
 
-  private static initPools() {
+  public static init() {
-    if (this.poolsInited == true) {
+    if (this.initDone == true) {
       return
     }
+
+    Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
+
+
     switch (Config.Server.thumbnail.processingLibrary) {
       case ThumbnailProcessingLib.Jimp:
         this.ThumbnailFunction = ThumbnailRenderers.jimp;
@@ -43,9 +42,15 @@ export class ThumbnailGeneratorMWs {
       default:
         throw "Unknown thumbnail processing lib";
     }
-    pool.run(this.ThumbnailFunction);
 
-    this.poolsInited = true;
+    if (Config.Server.enableThreading == true &&
+      Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.Jimp) {
+      const Pool = require('threads').Pool;
+      this.thPool = new Pool(Config.Client.concurrentThumbnailGenerations);
+      this.thPool.run(this.ThumbnailFunction);
+    }
+
+    this.initDone = true;
   }
 
   private static addThInfoTODir(directory: DirectoryDTO) {
@@ -151,7 +156,6 @@ export class ThumbnailGeneratorMWs {
       fs.mkdirSync(ProjectPath.ThumbnailFolder);
     }
 
-    this.initPools();
     //run on other thread
 
     let input = <RendererInput>{
@@ -162,8 +166,8 @@ export class ThumbnailGeneratorMWs {
       qualityPriority: Config.Server.thumbnail.qualityPriority,
       __dirname: __dirname,
     };
-    if (Config.Server.enableThreading == true) {
+    if (this.thPool !== null) {
-      pool.send(input)
+      this.thPool.send(input)
         .on('done', (out) => {
           return next(out);
         }).on('error', (error) => {
@@ -184,3 +188,5 @@ export class ThumbnailGeneratorMWs {
     return crypto.createHash('md5').update(imagePath).digest('hex') + "_" + size + ".jpg";
   }
 }
+
+ThumbnailGeneratorMWs.init();

backend/model/DiskManger.ts

@@ -7,7 +7,7 @@ import {diskManagerTask, DiskManagerTask} from "./DiskMangerTask";
 import {Config} from "../../common/config/private/Config";
 
 const Pool = require('threads').Pool;
-const pool = new Pool();
+const pool = new Pool(1);
 
 const LOG_TAG = "[DiskManager]";
 

backend/model/DiskMangerTask.ts

@@ -1,183 +1,193 @@
 ///<reference path="exif.d.ts"/>
 import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
-import {
+import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../common/entities/PhotoDTO";
-  CameraMetadata,
+import {Logger} from "../Logger";
-  GPSMetadata,
-  ImageSize,
-  PhotoDTO,
-  PhotoMetadata,
-  PositionMetaData
-} from "../../common/entities/PhotoDTO";
 
 const LOG_TAG = "[DiskManagerTask]";
 
 export const diskManagerTask = (input: DiskManagerTask.PoolInput, done) => {
-  const fs = require("fs");
+    const fs = require("fs");
-  const path = require("path");
+    const path = require("path");
-  const mime = require("mime");
+    const mime = require("mime");
-  const iptc = require("node-iptc");
+    const iptc = require("node-iptc");
-  const exif_parser = require("exif-parser");
+    const exif_parser = require("exif-parser");
 
 
-  let isImage = (fullPath: string) => {
+    let isImage = (fullPath: string) => {
-    let imageMimeTypes = [
+      let imageMimeTypes = [
-      'image/bmp',
+        'image/bmp',
-      'image/gif',
+        'image/gif',
-      'image/jpeg',
+        'image/jpeg',
-      'image/png',
+        'image/png',
-      'image/pjpeg',
+        'image/pjpeg',
-      'image/tiff',
+        'image/tiff',
-      'image/webp',
+        'image/webp',
-      'image/x-tiff',
+        'image/x-tiff',
-      'image/x-windows-bmp'
+        'image/x-windows-bmp'
-    ];
+      ];
 
-    let extension = mime.lookup(fullPath);
+      let extension = mime.lookup(fullPath);
 
-    return imageMimeTypes.indexOf(extension) !== -1;
+      return imageMimeTypes.indexOf(extension) !== -1;
-  };
+    };
 
-  let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
+    let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
-    return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
+        return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
-      fs.readFile(fullPath, function (err, data) {
+            fs.readFile(fullPath, function (err, data) {
-        if (err) {
+              if (err) {
-          return reject({file: fullPath, error: err});
+                return reject({file: fullPath, error: err});
-        }
-        try {
-
-          const exif = exif_parser.create(data).parse();
-          const iptcData = iptc(data);
-
-          const imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height};
-          const cameraData: CameraMetadata = {
-            ISO: exif.tags.ISO,
-            model: exif.tags.Modeol,
-            maker: exif.tags.Make,
-            fStop: exif.tags.FNumber,
-            exposure: exif.tags.ExposureTime,
-            focalLength: exif.tags.FocalLength,
-            lens: exif.tags.LensModel,
-          };
-          const GPS: GPSMetadata = {
-            latitude: exif.tags.GPSLatitude,
-            longitude: exif.tags.GPSLongitude,
-            altitude: exif.tags.GPSAltitude
-
-          };
-
-          const positionData: PositionMetaData = {
-            GPSData: GPS,
-            country: iptcData.country_or_primary_location_name,
-            state: iptcData.province_or_state,
-            city: iptcData.city
-          };
-
-          //Decode characters to UTF8
-          const decode = (s: any) => {
-            for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
-                 ((a = s[i][c](0)) & 0x80) &&
-                 (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
-                   o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
-            );
-            return s.join("");
-          };
-
-
-          const keywords: string[] = (iptcData.keywords || []).map((s: string) => decode(s));
-          const creationDate: number = iptcData.date_time ? iptcData.date_time.getTime() : 0;
-
-
-          const metadata: PhotoMetadata = <PhotoMetadata>{
-            keywords: keywords,
-            cameraData: cameraData,
-            positionData: positionData,
-            size: imageSize,
-            creationDate: creationDate
-          };
-          return resolve(metadata);
-        } catch (err) {
-          return reject({file: fullPath, error: err});
-        }
-      });
-    });
-  };
-
-  let parseDir = (directoryInfo: {
-    relativeDirectoryName: string,
-    directoryName: string,
-    directoryParent: string,
-    absoluteDirectoryName: string
-  }, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> => {
-    return new Promise<DirectoryDTO>((resolve, reject) => {
-      let promises: Array<Promise<any>> = [];
-      let directory = <DirectoryDTO>{
-        name: directoryInfo.directoryName,
-        path: directoryInfo.directoryParent,
-        lastUpdate: Date.now(),
-        directories: [],
-        photos: []
-      };
-      fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => {
-
-        if (err) {
-          return reject(err);
-        }
-
-        try {
-          for (let i = 0; i < list.length; i++) {
-            let file = list[i];
-            let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file));
-            if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
-              let promise = parseDir({
-                  relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep),
-                  directoryName: file,
-                  directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep),
-                  absoluteDirectoryName: fullFilePath
-                },
-                5, true
-              ).then((dir) => {
-                directory.directories.push(dir);
-              });
-              promises.push(promise);
-            } else if (isImage(fullFilePath)) {
-
-              let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
-                directory.photos.push(<PhotoDTO>{
-                  name: file,
-                  directory: null,
-                  metadata: photoMetadata
-                });
-              });
-              promises.push(promise);
-
-              if (maxPhotos != null && promises.length > maxPhotos) {
-                break;
               }
-            }
+              const metadata: PhotoMetadata = <PhotoMetadata>{
+                keywords: {},
+                cameraData: {},
+                positionData: {
+                  GPSData: {},
+                  country: null,
+                  state: null,
+                  city: null
+                },
+                size: {},
+                creationDate: {}
+              };
+
+              try {
+
+                try {
+                  const exif = exif_parser.create(data).parse();
+                  metadata.cameraData = <CameraMetadata> {
+                    ISO: exif.tags.ISO,
+                    model: exif.tags.Modeol,
+                    maker: exif.tags.Make,
+                    fStop: exif.tags.FNumber,
+                    exposure: exif.tags.ExposureTime,
+                    focalLength: exif.tags.FocalLength,
+                    lens: exif.tags.LensModel,
+                  };
+                  metadata.positionData.GPSData = <GPSMetadata> {
+                    latitude: exif.tags.GPSLatitude,
+                    longitude: exif.tags.GPSLongitude,
+                    altitude: exif.tags.GPSAltitude
+
+                  };
+
+                  metadata.size = <ImageSize> {width: exif.imageSize.width, height: exif.imageSize.height};
+                } catch (err) {
+                  Logger.info(LOG_TAG, "Error parsing exif", fullPath);
+                  metadata.size = <ImageSize> {width: 1, height: 1};
+                }
+
+                try {
+
+                  const iptcData = iptc(data);
+                  //Decode characters to UTF8
+                  const decode = (s: any) => {
+                    for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
+                         ((a = s[i][c](0)) & 0x80) &&
+                         (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
+                           o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
+                    );
+                    return s.join("");
+                  };
+
+                  metadata.positionData.country = iptcData.country_or_primary_location_name;
+                  metadata.positionData.state = iptcData.province_or_state;
+                  metadata.positionData.city = iptcData.city;
+
+
+                  metadata.keywords = <string[]> (iptcData.keywords || []).map((s: string) => decode(s));
+                  metadata.creationDate = <number> iptcData.date_time ? iptcData.date_time.getTime() : 0;
+                } catch (err) {
+                  Logger.info(LOG_TAG, "Error parsing iptc data", fullPath);
+                }
+
+
+                return resolve(metadata);
+              } catch (err) {
+                return reject({file: fullPath, error: err});
+              }
+            });
+          }
+        );
+      }
+    ;
+
+    let parseDir = (directoryInfo: {
+      relativeDirectoryName: string,
+      directoryName: string,
+      directoryParent: string,
+      absoluteDirectoryName: string
+    }, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> => {
+      return new Promise<DirectoryDTO>((resolve, reject) => {
+        let promises: Array<Promise<any>> = [];
+        let directory = <DirectoryDTO>{
+          name: directoryInfo.directoryName,
+          path: directoryInfo.directoryParent,
+          lastUpdate: Date.now(),
+          directories: [],
+          photos: []
+        };
+        fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => {
+
+          if (err) {
+            return reject(err);
           }
 
-          Promise.all(promises).then(() => {
+          try {
-            return resolve(directory);
+            for (let i = 0; i < list.length; i++) {
-          }).catch((err) => {
+              let file = list[i];
+              let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file));
+              if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) {
+                let promise = parseDir({
+                    relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep),
+                    directoryName: file,
+                    directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep),
+                    absoluteDirectoryName: fullFilePath
+                  },
+                  5, true
+                ).then((dir) => {
+                  directory.directories.push(dir);
+                });
+                promises.push(promise);
+              } else if (isImage(fullFilePath)) {
+
+                let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
+                  directory.photos.push(<PhotoDTO>{
+                    name: file,
+                    directory: null,
+                    metadata: photoMetadata
+                  });
+                });
+                promises.push(promise);
+
+                if (maxPhotos != null && promises.length > maxPhotos) {
+                  break;
+                }
+              }
+            }
+
+            Promise.all(promises).then(() => {
+              return resolve(directory);
+            }).catch((err) => {
+              return reject({directoryInfo: directoryInfo, error: err});
+            });
+          } catch (err) {
             return reject({directoryInfo: directoryInfo, error: err});
-          });
+          }
-        } catch (err) {
+
-          return reject({directoryInfo: directoryInfo, error: err});
+        });
-        }
 
       });
+    };
 
+
+    parseDir(input).then((dir) => {
+      done(null, dir);
+    }).catch((err) => {
+      done(err, null);
     });
-  };
 
-
+  }
-  parseDir(input).then((dir) => {
+;
-    done(null, dir);
-  }).catch((err) => {
-    done(err, null);
-  });
-
-};
 
 
 export module DiskManagerTask {