diff --git a/backend/middlewares/thumbnail/THRenderers.ts b/backend/middlewares/thumbnail/THRenderers.ts index d2e317b9..24b1df6c 100644 --- a/backend/middlewares/thumbnail/THRenderers.ts +++ b/backend/middlewares/thumbnail/THRenderers.ts @@ -1,5 +1,7 @@ import {Metadata, SharpInstance} from "@types/sharp"; import {Dimensions, State} from "@types/gm"; +import {Logger} from "../../Logger"; +import {Error, ErrorCodes} from "../../../common/entities/Error"; export module ThumbnailRenderers { @@ -60,7 +62,7 @@ export module ThumbnailRenderers { .metadata() .then((metadata: Metadata) => { - const Logger = require(input.__dirname + "/../../Logger").Logger; + // const Logger = require(input.__dirname + "/../../Logger").Logger; Logger.silly("[SharpThRenderer] rendering thumbnail:", input.imagePath); /** * newWidth * newHeight = size*size @@ -95,13 +97,13 @@ export module ThumbnailRenderers { .toFile(input.thPath).then(() => { return done(); }).catch(function (err) { - const Error = require(input.__dirname + "/../../../common/entities/Error").Error; - const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes; + // const Error = require(input.__dirname + "/../../../common/entities/Error").Error; + // const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes; return done(new Error(ErrorCodes.GENERAL_ERROR, err)); }); } catch (err) { - const Error = require(input.__dirname + "/../../../common/entities/Error").Error; - const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes; + // const Error = require(input.__dirname + "/../../../common/entities/Error").Error; + // const ErrorCodes = require(input.__dirname + "/../../../common/entities/Error").ErrorCodes; return done(new Error(ErrorCodes.GENERAL_ERROR, err)); } }); diff --git a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts index 7c2245fd..11ea0b04 100644 --- a/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts +++ b/backend/middlewares/thumbnail/ThumbnailGeneratorMWs.ts @@ -15,20 +15,19 @@ import {ThumbnailProcessingLib} from "../../../common/config/private/IPrivateCon import RendererInput = ThumbnailRenderers.RendererInput; -Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1); - -const Pool = require('threads').Pool; -const pool = new Pool(Config.Client.concurrentThumbnailGenerations); - - export class ThumbnailGeneratorMWs { - private static poolsInited = false; - private static ThumbnailFunction = null + private static initDone = false; + private static ThumbnailFunction = null; + private static thPool = null; - private static initPools() { - if (this.poolsInited == true) { + public static init() { + if (this.initDone == true) { return } + + Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1); + + switch (Config.Server.thumbnail.processingLibrary) { case ThumbnailProcessingLib.Jimp: this.ThumbnailFunction = ThumbnailRenderers.jimp; @@ -43,9 +42,15 @@ export class ThumbnailGeneratorMWs { default: throw "Unknown thumbnail processing lib"; } - pool.run(this.ThumbnailFunction); - this.poolsInited = true; + if (Config.Server.enableThreading == true && + Config.Server.thumbnail.processingLibrary == ThumbnailProcessingLib.Jimp) { + const Pool = require('threads').Pool; + this.thPool = new Pool(Config.Client.concurrentThumbnailGenerations); + this.thPool.run(this.ThumbnailFunction); + } + + this.initDone = true; } private static addThInfoTODir(directory: DirectoryDTO) { @@ -151,7 +156,6 @@ export class ThumbnailGeneratorMWs { fs.mkdirSync(ProjectPath.ThumbnailFolder); } - this.initPools(); //run on other thread let input = { @@ -162,8 +166,8 @@ export class ThumbnailGeneratorMWs { qualityPriority: Config.Server.thumbnail.qualityPriority, __dirname: __dirname, }; - if (Config.Server.enableThreading == true) { - pool.send(input) + if (this.thPool !== null) { + this.thPool.send(input) .on('done', (out) => { return next(out); }).on('error', (error) => { @@ -184,3 +188,5 @@ export class ThumbnailGeneratorMWs { return crypto.createHash('md5').update(imagePath).digest('hex') + "_" + size + ".jpg"; } } + +ThumbnailGeneratorMWs.init(); diff --git a/backend/model/DiskManger.ts b/backend/model/DiskManger.ts index 665e4a6c..2fb6a5ca 100644 --- a/backend/model/DiskManger.ts +++ b/backend/model/DiskManger.ts @@ -7,7 +7,7 @@ import {diskManagerTask, DiskManagerTask} from "./DiskMangerTask"; import {Config} from "../../common/config/private/Config"; const Pool = require('threads').Pool; -const pool = new Pool(); +const pool = new Pool(1); const LOG_TAG = "[DiskManager]"; diff --git a/backend/model/DiskMangerTask.ts b/backend/model/DiskMangerTask.ts index 7e4f17ca..e6f9dc93 100644 --- a/backend/model/DiskMangerTask.ts +++ b/backend/model/DiskMangerTask.ts @@ -1,183 +1,193 @@ /// import {DirectoryDTO} from "../../common/entities/DirectoryDTO"; -import { - CameraMetadata, - GPSMetadata, - ImageSize, - PhotoDTO, - PhotoMetadata, - PositionMetaData -} from "../../common/entities/PhotoDTO"; +import {CameraMetadata, GPSMetadata, ImageSize, PhotoDTO, PhotoMetadata} from "../../common/entities/PhotoDTO"; +import {Logger} from "../Logger"; const LOG_TAG = "[DiskManagerTask]"; export const diskManagerTask = (input: DiskManagerTask.PoolInput, done) => { - const fs = require("fs"); - const path = require("path"); - const mime = require("mime"); - const iptc = require("node-iptc"); - const exif_parser = require("exif-parser"); + const fs = require("fs"); + const path = require("path"); + const mime = require("mime"); + const iptc = require("node-iptc"); + const exif_parser = require("exif-parser"); - let isImage = (fullPath: string) => { - let imageMimeTypes = [ - 'image/bmp', - 'image/gif', - 'image/jpeg', - 'image/png', - 'image/pjpeg', - 'image/tiff', - 'image/webp', - 'image/x-tiff', - 'image/x-windows-bmp' - ]; + let isImage = (fullPath: string) => { + let imageMimeTypes = [ + 'image/bmp', + 'image/gif', + 'image/jpeg', + 'image/png', + 'image/pjpeg', + 'image/tiff', + 'image/webp', + 'image/x-tiff', + 'image/x-windows-bmp' + ]; - let extension = mime.lookup(fullPath); + let extension = mime.lookup(fullPath); - return imageMimeTypes.indexOf(extension) !== -1; - }; + return imageMimeTypes.indexOf(extension) !== -1; + }; - let loadPhotoMetadata = (fullPath: string): Promise => { - return new Promise((resolve: (metadata: PhotoMetadata) => void, reject) => { - fs.readFile(fullPath, function (err, data) { - if (err) { - return reject({file: fullPath, error: err}); - } - try { - - const exif = exif_parser.create(data).parse(); - const iptcData = iptc(data); - - const imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height}; - const cameraData: CameraMetadata = { - ISO: exif.tags.ISO, - model: exif.tags.Modeol, - maker: exif.tags.Make, - fStop: exif.tags.FNumber, - exposure: exif.tags.ExposureTime, - focalLength: exif.tags.FocalLength, - lens: exif.tags.LensModel, - }; - const GPS: GPSMetadata = { - latitude: exif.tags.GPSLatitude, - longitude: exif.tags.GPSLongitude, - altitude: exif.tags.GPSAltitude - - }; - - const positionData: PositionMetaData = { - GPSData: GPS, - country: iptcData.country_or_primary_location_name, - state: iptcData.province_or_state, - city: iptcData.city - }; - - //Decode characters to UTF8 - const decode = (s: any) => { - for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l; - ((a = s[i][c](0)) & 0x80) && - (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ? - o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "") - ); - return s.join(""); - }; - - - const keywords: string[] = (iptcData.keywords || []).map((s: string) => decode(s)); - const creationDate: number = iptcData.date_time ? iptcData.date_time.getTime() : 0; - - - const metadata: PhotoMetadata = { - keywords: keywords, - cameraData: cameraData, - positionData: positionData, - size: imageSize, - creationDate: creationDate - }; - return resolve(metadata); - } catch (err) { - return reject({file: fullPath, error: err}); - } - }); - }); - }; - - let parseDir = (directoryInfo: { - relativeDirectoryName: string, - directoryName: string, - directoryParent: string, - absoluteDirectoryName: string - }, maxPhotos: number = null, photosOnly: boolean = false): Promise => { - return new Promise((resolve, reject) => { - let promises: Array> = []; - let directory = { - name: directoryInfo.directoryName, - path: directoryInfo.directoryParent, - lastUpdate: Date.now(), - directories: [], - photos: [] - }; - fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => { - - if (err) { - return reject(err); - } - - try { - for (let i = 0; i < list.length; i++) { - let file = list[i]; - let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file)); - if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { - let promise = parseDir({ - relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep), - directoryName: file, - directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep), - absoluteDirectoryName: fullFilePath - }, - 5, true - ).then((dir) => { - directory.directories.push(dir); - }); - promises.push(promise); - } else if (isImage(fullFilePath)) { - - let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => { - directory.photos.push({ - name: file, - directory: null, - metadata: photoMetadata - }); - }); - promises.push(promise); - - if (maxPhotos != null && promises.length > maxPhotos) { - break; + let loadPhotoMetadata = (fullPath: string): Promise => { + return new Promise((resolve: (metadata: PhotoMetadata) => void, reject) => { + fs.readFile(fullPath, function (err, data) { + if (err) { + return reject({file: fullPath, error: err}); } - } + const metadata: PhotoMetadata = { + keywords: {}, + cameraData: {}, + positionData: { + GPSData: {}, + country: null, + state: null, + city: null + }, + size: {}, + creationDate: {} + }; + + try { + + try { + const exif = exif_parser.create(data).parse(); + metadata.cameraData = { + ISO: exif.tags.ISO, + model: exif.tags.Modeol, + maker: exif.tags.Make, + fStop: exif.tags.FNumber, + exposure: exif.tags.ExposureTime, + focalLength: exif.tags.FocalLength, + lens: exif.tags.LensModel, + }; + metadata.positionData.GPSData = { + latitude: exif.tags.GPSLatitude, + longitude: exif.tags.GPSLongitude, + altitude: exif.tags.GPSAltitude + + }; + + metadata.size = {width: exif.imageSize.width, height: exif.imageSize.height}; + } catch (err) { + Logger.info(LOG_TAG, "Error parsing exif", fullPath); + metadata.size = {width: 1, height: 1}; + } + + try { + + const iptcData = iptc(data); + //Decode characters to UTF8 + const decode = (s: any) => { + for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l; + ((a = s[i][c](0)) & 0x80) && + (s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ? + o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "") + ); + return s.join(""); + }; + + metadata.positionData.country = iptcData.country_or_primary_location_name; + metadata.positionData.state = iptcData.province_or_state; + metadata.positionData.city = iptcData.city; + + + metadata.keywords = (iptcData.keywords || []).map((s: string) => decode(s)); + metadata.creationDate = iptcData.date_time ? iptcData.date_time.getTime() : 0; + } catch (err) { + Logger.info(LOG_TAG, "Error parsing iptc data", fullPath); + } + + + return resolve(metadata); + } catch (err) { + return reject({file: fullPath, error: err}); + } + }); + } + ); + } + ; + + let parseDir = (directoryInfo: { + relativeDirectoryName: string, + directoryName: string, + directoryParent: string, + absoluteDirectoryName: string + }, maxPhotos: number = null, photosOnly: boolean = false): Promise => { + return new Promise((resolve, reject) => { + let promises: Array> = []; + let directory = { + name: directoryInfo.directoryName, + path: directoryInfo.directoryParent, + lastUpdate: Date.now(), + directories: [], + photos: [] + }; + fs.readdir(directoryInfo.absoluteDirectoryName, (err, list) => { + + if (err) { + return reject(err); } - Promise.all(promises).then(() => { - return resolve(directory); - }).catch((err) => { + try { + for (let i = 0; i < list.length; i++) { + let file = list[i]; + let fullFilePath = path.normalize(path.resolve(directoryInfo.absoluteDirectoryName, file)); + if (photosOnly == false && fs.statSync(fullFilePath).isDirectory()) { + let promise = parseDir({ + relativeDirectoryName: path.join(directoryInfo.relativeDirectoryName, path.sep), + directoryName: file, + directoryParent: path.join(directoryInfo.relativeDirectoryName, path.sep), + absoluteDirectoryName: fullFilePath + }, + 5, true + ).then((dir) => { + directory.directories.push(dir); + }); + promises.push(promise); + } else if (isImage(fullFilePath)) { + + let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => { + directory.photos.push({ + name: file, + directory: null, + metadata: photoMetadata + }); + }); + promises.push(promise); + + if (maxPhotos != null && promises.length > maxPhotos) { + break; + } + } + } + + Promise.all(promises).then(() => { + return resolve(directory); + }).catch((err) => { + return reject({directoryInfo: directoryInfo, error: err}); + }); + } catch (err) { return reject({directoryInfo: directoryInfo, error: err}); - }); - } catch (err) { - return reject({directoryInfo: directoryInfo, error: err}); - } + } + + }); }); + }; + + parseDir(input).then((dir) => { + done(null, dir); + }).catch((err) => { + done(err, null); }); - }; - - parseDir(input).then((dir) => { - done(null, dir); - }).catch((err) => { - done(err, null); - }); - -}; + } +; export module DiskManagerTask {