mirror of
https://github.com/xuthus83/pigallery2.git
synced 2025-01-14 14:43:17 +08:00
improving mutithreading at thumbnail generation
implementing mutithread directory scanning
This commit is contained in:
Braun Patrik
parent
f134d74b13
commit
88c8d53d49
@ -2,6 +2,7 @@
|
||||
import * as path from "path";
|
||||
import * as crypto from "crypto";
|
||||
import * as fs from "fs";
|
||||
import * as os from "os";
|
||||
import {NextFunction, Request, Response} from "express";
|
||||
import {Error, ErrorCodes} from "../../common/entities/Error";
|
||||
import {Config} from "../config/Config";
|
||||
@ -10,8 +11,11 @@ import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
|
||||
import {ProjectPath} from "../ProjectPath";
|
||||
import {PhotoDTO} from "../../common/entities/PhotoDTO";
|
||||
|
||||
|
||||
Config.Client.concurrentThumbnailGenerations = Math.max(1, os.cpus().length - 1);
|
||||
|
||||
const Pool = require('threads').Pool;
|
||||
const pool = new Pool();
|
||||
const pool = new Pool(Config.Client.concurrentThumbnailGenerations);
|
||||
|
||||
pool.run(
|
||||
(input: {imagePath: string, size: number, thPath: string}, done) => {
|
||||
@ -43,6 +47,7 @@ pool.run(
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
export class ThumbnailGeneratorMWs {
|
||||
|
||||
|
||||
@ -117,6 +122,7 @@ export class ThumbnailGeneratorMWs {
|
||||
//check if thumbnail already exist
|
||||
if (fs.existsSync(thPath) === true) {
|
||||
return next();
|
||||
//return setTimeout(()=>{next();},2000);
|
||||
}
|
||||
|
||||
//create thumbnail folder if not exist
|
||||
@ -124,7 +130,7 @@ export class ThumbnailGeneratorMWs {
|
||||
fs.mkdirSync(ProjectPath.ThumbnailFolder);
|
||||
}
|
||||
|
||||
|
||||
console.log("generating thumbnail", imagePath, size);
|
||||
//run on other thread
|
||||
pool.send({imagePath: imagePath, size: size, thPath: thPath})
|
||||
.on('done', (out) => {
|
||||
@ -134,8 +140,6 @@ export class ThumbnailGeneratorMWs {
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static generateThumbnailName(imagePath: string, size: number): string {
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
///<reference path="exif.d.ts"/>
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as mime from "mime";
|
||||
import * as iptc from "node-iptc";
|
||||
import * as exif_parser from "exif-parser";
|
||||
import {DirectoryDTO} from "../../common/entities/DirectoryDTO";
|
||||
import {
|
||||
PhotoDTO,
|
||||
@ -15,47 +11,135 @@ import {
|
||||
} from "../../common/entities/PhotoDTO";
|
||||
import {ProjectPath} from "../ProjectPath";
|
||||
|
||||
export class DiskManager {
|
||||
public static scanDirectory(relativeDirectoryName: string, cb: (error: any, result: DirectoryDTO) => void) {
|
||||
console.log("DiskManager: scanDirectory");
|
||||
let directoryName = path.basename(relativeDirectoryName);
|
||||
let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
|
||||
let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
|
||||
const Pool = require('threads').Pool;
|
||||
const pool = new Pool();
|
||||
pool.run(
|
||||
(input: {
|
||||
relativeDirectoryName: string,
|
||||
directoryName: string,
|
||||
directoryParent: string,
|
||||
absoluteDirectoryName: string
|
||||
}, done) => {
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const mime = require("mime");
|
||||
const iptc = require("node-iptc");
|
||||
const exif_parser = require("exif-parser");
|
||||
|
||||
|
||||
let isImage = (fullPath: string) => {
|
||||
let imageMimeTypes = [
|
||||
'image/bmp',
|
||||
'image/gif',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/pjpeg',
|
||||
'image/tiff',
|
||||
'image/webp',
|
||||
'image/x-tiff',
|
||||
'image/x-windows-bmp'
|
||||
];
|
||||
|
||||
let extension = mime.lookup(fullPath);
|
||||
|
||||
return imageMimeTypes.indexOf(extension) !== -1;
|
||||
};
|
||||
|
||||
let loadPhotoMetadata = (fullPath: string): Promise<PhotoMetadata> => {
|
||||
return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
|
||||
fs.readFile(fullPath, function (err, data) {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
} else {
|
||||
let exif = exif_parser.create(data).parse();
|
||||
let iptcData = iptc(data);
|
||||
|
||||
let imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height};
|
||||
let cameraData: CameraMetadata = {
|
||||
ISO: exif.tags.ISO,
|
||||
model: exif.tags.Modeol,
|
||||
maker: exif.tags.Make,
|
||||
fStop: exif.tags.FNumber,
|
||||
exposure: exif.tags.ExposureTime,
|
||||
focalLength: exif.tags.FocalLength,
|
||||
lens: exif.tags.LensModel,
|
||||
};
|
||||
let GPS: GPSMetadata = {
|
||||
latitude: exif.tags.GPSLatitude,
|
||||
longitude: exif.tags.GPSLongitude,
|
||||
altitude: exif.tags.GPSAltitude
|
||||
|
||||
};
|
||||
|
||||
let positionData: PositionMetaData = {
|
||||
GPSData: GPS,
|
||||
country: iptcData.country_or_primary_location_name,
|
||||
state: iptcData.province_or_state,
|
||||
city: iptcData.city
|
||||
};
|
||||
|
||||
//Decode characters to UTF8
|
||||
let decode = (s: any) => {
|
||||
for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
|
||||
((a = s[i][c](0)) & 0x80) &&
|
||||
(s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
|
||||
o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
|
||||
);
|
||||
return s.join("");
|
||||
};
|
||||
|
||||
let keywords: [string] = iptcData.keywords.map((s: string) => decode(s));
|
||||
let creationDate: number = iptcData.date_time.getTime();
|
||||
|
||||
|
||||
let metadata: PhotoMetadata = <PhotoMetadata>{
|
||||
keywords: keywords,
|
||||
cameraData: cameraData,
|
||||
positionData: positionData,
|
||||
size: imageSize,
|
||||
creationDate: creationDate
|
||||
};
|
||||
return resolve(metadata);
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
let directory = <DirectoryDTO>{
|
||||
name: directoryName,
|
||||
path: directoryParent,
|
||||
name: input.directoryName,
|
||||
path: input.directoryParent,
|
||||
lastUpdate: Date.now(),
|
||||
directories: [],
|
||||
photos: []
|
||||
};
|
||||
|
||||
let promises: Array< Promise<any> > = [];
|
||||
fs.readdir(absoluteDirectoryName, (err, list) => {
|
||||
fs.readdir(input.absoluteDirectoryName, (err, list) => {
|
||||
|
||||
if (err) {
|
||||
return cb(err, null);
|
||||
return done(err, null);
|
||||
}
|
||||
|
||||
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
let file = list[i];
|
||||
let fullFilePath = path.normalize(path.resolve(absoluteDirectoryName, file));
|
||||
let fullFilePath = path.normalize(path.resolve(input.absoluteDirectoryName, file));
|
||||
if (fs.statSync(fullFilePath).isDirectory()) {
|
||||
directory.directories.push(<DirectoryDTO>{
|
||||
name: file,
|
||||
path: path.join(relativeDirectoryName, path.sep),
|
||||
path: path.join(input.relativeDirectoryName, path.sep),
|
||||
lastUpdate: Date.now(),
|
||||
directories: [],
|
||||
photos: []
|
||||
});
|
||||
}
|
||||
|
||||
if (DiskManager.isImage(fullFilePath)) {
|
||||
if (isImage(fullFilePath)) {
|
||||
|
||||
|
||||
let promise = DiskManager.loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
|
||||
directory.photos.push(<PhotoDTO>{name: file, directory: directory, metadata: photoMetadata});
|
||||
let promise = loadPhotoMetadata(fullFilePath).then((photoMetadata) => {
|
||||
directory.photos.push(<PhotoDTO>{name: file, directory: null, metadata: photoMetadata});
|
||||
});
|
||||
|
||||
promises.push(promise);
|
||||
@ -64,123 +148,34 @@ export class DiskManager {
|
||||
|
||||
Promise.all(promises).then(() => {
|
||||
console.log("DiskManager: scanDirectory finished");
|
||||
return cb(err, directory);
|
||||
return done(err, directory);
|
||||
}).catch((err) => {
|
||||
console.error(err);
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
private static isImage(fullPath: string) {
|
||||
let imageMimeTypes = [
|
||||
'image/bmp',
|
||||
'image/gif',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/pjpeg',
|
||||
'image/tiff',
|
||||
'image/webp',
|
||||
'image/x-tiff',
|
||||
'image/x-windows-bmp'
|
||||
];
|
||||
export class DiskManager {
|
||||
public static scanDirectory(relativeDirectoryName: string, cb: (error: any, result: DirectoryDTO) => void) {
|
||||
console.log("DiskManager: scanDirectory");
|
||||
let directoryName = path.basename(relativeDirectoryName);
|
||||
let directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
|
||||
let absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
|
||||
|
||||
let extension = mime.lookup(fullPath);
|
||||
|
||||
return imageMimeTypes.indexOf(extension) !== -1;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
UTF8 = {
|
||||
|
||||
encode: function(s){
|
||||
|
||||
for(var c, i = -1, l = (s = s.split("")).length, o = String.fromCharCode; ++i < l;
|
||||
|
||||
s[i] = (c = s[i].charCodeAt(0)) >= 127 ? o(0xc0 | (c >>> 6)) + o(0x80 | (c & 0x3f)) : s[i]
|
||||
|
||||
);
|
||||
|
||||
return s.join("");
|
||||
|
||||
},
|
||||
|
||||
decode: function(s){
|
||||
|
||||
for(var a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
|
||||
|
||||
((a = s[i][c](0)) & 0x80) &&
|
||||
|
||||
(s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
|
||||
|
||||
o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
|
||||
|
||||
);
|
||||
|
||||
return s.join("");
|
||||
|
||||
}
|
||||
|
||||
};*/
|
||||
private static loadPhotoMetadata(fullPath: string): Promise<PhotoMetadata> {
|
||||
return new Promise<PhotoMetadata>((resolve: (metadata: PhotoMetadata) => void, reject) => {
|
||||
fs.readFile(fullPath, function (err, data) {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
} else {
|
||||
let exif = exif_parser.create(data).parse();
|
||||
let iptcData = iptc(data);
|
||||
|
||||
let imageSize: ImageSize = {width: exif.imageSize.width, height: exif.imageSize.height};
|
||||
let cameraData: CameraMetadata = {
|
||||
ISO: exif.tags.ISO,
|
||||
model: exif.tags.Modeol,
|
||||
maker: exif.tags.Make,
|
||||
fStop: exif.tags.FNumber,
|
||||
exposure: exif.tags.ExposureTime,
|
||||
focalLength: exif.tags.FocalLength,
|
||||
lens: exif.tags.LensModel,
|
||||
};
|
||||
let GPS: GPSMetadata = {
|
||||
latitude: exif.tags.GPSLatitude,
|
||||
longitude: exif.tags.GPSLongitude,
|
||||
altitude: exif.tags.GPSAltitude
|
||||
|
||||
};
|
||||
|
||||
let positionData: PositionMetaData = {
|
||||
GPSData: GPS,
|
||||
country: iptcData.country_or_primary_location_name,
|
||||
state: iptcData.province_or_state,
|
||||
city: iptcData.city
|
||||
};
|
||||
|
||||
//Decode characters to UTF8
|
||||
let decode = (s: any) => {
|
||||
for (let a, b, i = -1, l = (s = s.split("")).length, o = String.fromCharCode, c = "charCodeAt"; ++i < l;
|
||||
((a = s[i][c](0)) & 0x80) &&
|
||||
(s[i] = (a & 0xfc) == 0xc0 && ((b = s[i + 1][c](0)) & 0xc0) == 0x80 ?
|
||||
o(((a & 0x03) << 6) + (b & 0x3f)) : o(128), s[++i] = "")
|
||||
);
|
||||
return s.join("");
|
||||
};
|
||||
|
||||
let keywords: [string] = iptcData.keywords.map((s: string) => decode(s));
|
||||
let creationDate: number = iptcData.date_time.getTime();
|
||||
|
||||
|
||||
let metadata: PhotoMetadata = <PhotoMetadata>{
|
||||
keywords: keywords,
|
||||
cameraData: cameraData,
|
||||
positionData: positionData,
|
||||
size: imageSize,
|
||||
creationDate: creationDate
|
||||
};
|
||||
return resolve(metadata);
|
||||
}
|
||||
pool.send({
|
||||
relativeDirectoryName,
|
||||
directoryName,
|
||||
directoryParent,
|
||||
absoluteDirectoryName
|
||||
}) .on('done', (error: any, result: DirectoryDTO) => {
|
||||
result.photos.forEach((ph) => {
|
||||
ph.directory = result;
|
||||
});
|
||||
return cb(error, result);
|
||||
}).on('error', (job, error) => {
|
||||
return cb(error, null);
|
||||
});
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -76,7 +76,7 @@ export class ThumbnailLoaderService {
|
||||
|
||||
for (let i = 0; i < this.que.length; i++) {
|
||||
for (let j = 0; j < this.que[i].taskEntities.length; j++) {
|
||||
if (this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.high) {
|
||||
if (this.que[i].inProgress == false && this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.high) {
|
||||
return this.que[i];
|
||||
}
|
||||
}
|
||||
@ -84,13 +84,21 @@ export class ThumbnailLoaderService {
|
||||
|
||||
for (let i = 0; i < this.que.length; i++) {
|
||||
for (let j = 0; j < this.que[i].taskEntities.length; j++) {
|
||||
if (this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.medium) {
|
||||
if (this.que[i].inProgress == false && this.que[i].taskEntities[j].priority === ThumbnailLoadingPriority.medium) {
|
||||
return this.que[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this.que[0];
|
||||
|
||||
for (let i = 0; i < this.que.length; i++) {
|
||||
if (this.que[i].inProgress == false) {
|
||||
return this.que[i];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private taskReady(task: ThumbnailTask) {
|
||||
@ -119,10 +127,10 @@ export class ThumbnailLoaderService {
|
||||
task.taskEntities.forEach(te => te.listener.onStartedLoading());
|
||||
task.inProgress = true;
|
||||
|
||||
console.log("loading", task.gridPhoto.getThumbnailPath());
|
||||
let curImg = new Image();
|
||||
curImg.src = task.gridPhoto.getThumbnailPath();
|
||||
|
||||
curImg.onload = () => {
|
||||
console.log("loaded", task.gridPhoto.getThumbnailPath());
|
||||
|
||||
task.gridPhoto.thumbnailLoaded();
|
||||
this.galleryChacheService.photoUpdated(task.gridPhoto.photo);
|
||||
@ -134,12 +142,15 @@ export class ThumbnailLoaderService {
|
||||
};
|
||||
|
||||
curImg.onerror = (error) => {
|
||||
console.log("error", task.gridPhoto.getThumbnailPath());
|
||||
task.taskEntities.forEach((te: ThumbnailTaskEntity) => te.listener.onError(error));
|
||||
|
||||
this.taskReady(task);
|
||||
this.runningRequests--;
|
||||
this.run();
|
||||
};
|
||||
|
||||
curImg.src = task.gridPhoto.getThumbnailPath();
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user