diff --git a/src/backend/model/jobs/jobs/IndexingJob.ts b/src/backend/model/jobs/jobs/IndexingJob.ts index 6709af43..395dec4d 100644 --- a/src/backend/model/jobs/jobs/IndexingJob.ts +++ b/src/backend/model/jobs/jobs/IndexingJob.ts @@ -11,6 +11,8 @@ import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {Logger} from '../../../Logger'; import {FileDTO} from '../../../../common/entities/FileDTO'; +const LOG_TAG = '[IndexingJob]'; + export class IndexingJob< S extends { indexChangesOnly: boolean } = { indexChangesOnly: boolean } > extends Job { @@ -48,33 +50,50 @@ export class IndexingJob< let scanned: ParentDirectoryDTO; let dirChanged = true; - // check if the folder got modified if only changes need to be indexed - if (this.config.indexChangesOnly) { - const stat = fs.statSync(path.join(ProjectPath.ImageFolder, directory)); - const lastModified = DiskMangerWorker.calcLastModified(stat); - scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory); - // If not modified and it was scanned before, dir is up-to-date - if ( - scanned && - scanned.lastModified === lastModified && - scanned.lastScanned != null - ) { - dirChanged = false; - } - } + try { - // reindex - if (dirChanged || !this.config.indexChangesOnly) { - this.Progress.log('Indexing: ' + directory); - this.Progress.Processed++; - scanned = - await ObjectManagers.getInstance().IndexingManager.indexDirectory( - directory - ); - } else { - this.Progress.log('Skipped: ' + directory); + const absDirPath = path.join(ProjectPath.ImageFolder, directory); + if (!fs.existsSync(absDirPath)) { + this.Progress.log('Skipping. Directory does not exist: ' + directory); + this.Progress.Skipped++; + } else { // dir should exist now + + // check if the folder got modified if only changes need to be indexed + if (this.config.indexChangesOnly) { + + const stat = fs.statSync(absDirPath); + const lastModified = DiskMangerWorker.calcLastModified(stat); + scanned = await ObjectManagers.getInstance().GalleryManager.selectDirStructure(directory); + // If not modified and it was scanned before, dir is up-to-date + if ( + scanned && + scanned.lastModified === lastModified && + scanned.lastScanned != null + ) { + dirChanged = false; + } + } + + + // reindex + if (dirChanged || !this.config.indexChangesOnly) { + this.Progress.log('Indexing: ' + directory); + this.Progress.Processed++; + scanned = + await ObjectManagers.getInstance().IndexingManager.indexDirectory( + directory + ); + } else { + this.Progress.log('Skipping. No change for: ' + directory); + this.Progress.Skipped++; + Logger.silly(LOG_TAG, 'Skipping reindexing, no change for: ' + directory); + } + } + } catch (e) { + this.Progress.log('Skipping. Indexing failed for: ' + directory); this.Progress.Skipped++; - Logger.silly('Skipping reindexing, no change for: ' + directory); + Logger.warn(LOG_TAG, 'Skipping. Indexing failed for: ' + directory); + console.error(e); } if (this.Progress.State !== JobProgressStates.running) { return false; diff --git a/src/backend/model/jobs/jobs/Job.ts b/src/backend/model/jobs/jobs/Job.ts index c69b4c15..632fc0bd 100644 --- a/src/backend/model/jobs/jobs/Job.ts +++ b/src/backend/model/jobs/jobs/Job.ts @@ -153,6 +153,7 @@ export abstract class Job = Record> i await new Promise(setImmediate); this.run(); } catch (e) { + Logger.error(LOG_TAG, 'Job failed with:'); Logger.error(LOG_TAG, e); this.Progress.log('Failed with: ' + (typeof e.toString === 'function') ? e.toString() : JSON.stringify(e)); this.Progress.State = JobProgressStates.failed; diff --git a/src/backend/model/threading/MetadataLoader.ts b/src/backend/model/threading/MetadataLoader.ts index a0be144a..7fc48770 100644 --- a/src/backend/model/threading/MetadataLoader.ts +++ b/src/backend/model/threading/MetadataLoader.ts @@ -92,334 +92,349 @@ export class MetadataLoader { }); } + private static readonly EMPTY_METADATA: PhotoMetadata = { + size: {width: 1, height: 1}, + creationDate: 0, + fileSize: 0, + }; + public static loadPhotoMetadata(fullPath: string): Promise { return new Promise((resolve, reject) => { - const fd = fs.openSync(fullPath, 'r'); + try { + const fd = fs.openSync(fullPath, 'r'); - const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize); - fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => { - fs.closeSync(fd); - if (err) { - return reject({file: fullPath, error: err}); - } - const metadata: PhotoMetadata = { - size: {width: 1, height: 1}, - creationDate: 0, - fileSize: 0, - }; - try { - try { - const stat = fs.statSync(fullPath); - metadata.fileSize = stat.size; - metadata.creationDate = stat.mtime.getTime(); - } catch (err) { - // ignoring errors + const data = Buffer.allocUnsafe(Config.Media.photoMetadataSize); + fs.read(fd, data, 0, Config.Media.photoMetadataSize, 0, (err) => { + fs.closeSync(fd); + const metadata: PhotoMetadata = { + size: {width: 1, height: 1}, + creationDate: 0, + fileSize: 0, + }; + if (err) { + Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath); + console.error(err); + return resolve(MetadataLoader.EMPTY_METADATA); } - try { - const exif = ExifParserFactory.create(data).parse(); - if ( - exif.tags.ISO || - exif.tags.Model || - exif.tags.Make || - exif.tags.FNumber || - exif.tags.ExposureTime || - exif.tags.FocalLength || - exif.tags.LensModel - ) { - if (exif.tags.Model && exif.tags.Model !== '') { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.model = '' + exif.tags.Model; - } - if (exif.tags.Make && exif.tags.Make !== '') { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.make = '' + exif.tags.Make; - } - if (exif.tags.LensModel && exif.tags.LensModel !== '') { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.lens = '' + exif.tags.LensModel; - } - if (Utils.isUInt32(exif.tags.ISO)) { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10); - } - if (Utils.isFloat32(exif.tags.FocalLength)) { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.focalLength = parseFloat( - '' + exif.tags.FocalLength - ); - } - if (Utils.isFloat32(exif.tags.ExposureTime)) { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.exposure = parseFloat( - parseFloat('' + exif.tags.ExposureTime).toFixed(6) - ); - } - if (Utils.isFloat32(exif.tags.FNumber)) { - metadata.cameraData = metadata.cameraData || {}; - metadata.cameraData.fStop = parseFloat( - parseFloat('' + exif.tags.FNumber).toFixed(2) - ); - } - } - if ( - !isNaN(exif.tags.GPSLatitude) || - exif.tags.GPSLongitude || - exif.tags.GPSAltitude - ) { - metadata.positionData = metadata.positionData || {}; - metadata.positionData.GPSData = {}; - - if (Utils.isFloat32(exif.tags.GPSLongitude)) { - metadata.positionData.GPSData.longitude = parseFloat( - exif.tags.GPSLongitude.toFixed(6) - ); - } - if (Utils.isFloat32(exif.tags.GPSLatitude)) { - metadata.positionData.GPSData.latitude = parseFloat( - exif.tags.GPSLatitude.toFixed(6) - ); - } - } - if ( - exif.tags.CreateDate || - exif.tags.DateTimeOriginal || - exif.tags.ModifyDate - ) { - metadata.creationDate = - (exif.tags.DateTimeOriginal || - exif.tags.CreateDate || - exif.tags.ModifyDate) * 1000; - } - if (exif.imageSize) { - metadata.size = { - width: exif.imageSize.width, - height: exif.imageSize.height, - }; - } else if ( - exif.tags.RelatedImageWidth && - exif.tags.RelatedImageHeight - ) { - metadata.size = { - width: exif.tags.RelatedImageWidth, - height: exif.tags.RelatedImageHeight, - }; - } else if ( - exif.tags.ImageWidth && - exif.tags.ImageHeight - ) { - metadata.size = { - width: exif.tags.ImageWidth, - height: exif.tags.ImageHeight, - }; - } else { - const info = imageSize(fullPath); - metadata.size = {width: info.width, height: info.height}; - } - } catch (err) { - Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err); try { - const info = imageSize(fullPath); - metadata.size = {width: info.width, height: info.height}; - } catch (e) { - metadata.size = {width: 1, height: 1}; + const stat = fs.statSync(fullPath); + metadata.fileSize = stat.size; + metadata.creationDate = stat.mtime.getTime(); + } catch (err) { + // ignoring errors } - } - try { - const iptcData = IptcParser.parse(data); - if (iptcData.country_or_primary_location_name) { - metadata.positionData = metadata.positionData || {}; - metadata.positionData.country = - iptcData.country_or_primary_location_name + try { + const exif = ExifParserFactory.create(data).parse(); + if ( + exif.tags.ISO || + exif.tags.Model || + exif.tags.Make || + exif.tags.FNumber || + exif.tags.ExposureTime || + exif.tags.FocalLength || + exif.tags.LensModel + ) { + if (exif.tags.Model && exif.tags.Model !== '') { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.model = '' + exif.tags.Model; + } + if (exif.tags.Make && exif.tags.Make !== '') { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.make = '' + exif.tags.Make; + } + if (exif.tags.LensModel && exif.tags.LensModel !== '') { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.lens = '' + exif.tags.LensModel; + } + if (Utils.isUInt32(exif.tags.ISO)) { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.ISO = parseInt('' + exif.tags.ISO, 10); + } + if (Utils.isFloat32(exif.tags.FocalLength)) { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.focalLength = parseFloat( + '' + exif.tags.FocalLength + ); + } + if (Utils.isFloat32(exif.tags.ExposureTime)) { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.exposure = parseFloat( + parseFloat('' + exif.tags.ExposureTime).toFixed(6) + ); + } + if (Utils.isFloat32(exif.tags.FNumber)) { + metadata.cameraData = metadata.cameraData || {}; + metadata.cameraData.fStop = parseFloat( + parseFloat('' + exif.tags.FNumber).toFixed(2) + ); + } + } + if ( + !isNaN(exif.tags.GPSLatitude) || + exif.tags.GPSLongitude || + exif.tags.GPSAltitude + ) { + metadata.positionData = metadata.positionData || {}; + metadata.positionData.GPSData = {}; + + if (Utils.isFloat32(exif.tags.GPSLongitude)) { + metadata.positionData.GPSData.longitude = parseFloat( + exif.tags.GPSLongitude.toFixed(6) + ); + } + if (Utils.isFloat32(exif.tags.GPSLatitude)) { + metadata.positionData.GPSData.latitude = parseFloat( + exif.tags.GPSLatitude.toFixed(6) + ); + } + } + if ( + exif.tags.CreateDate || + exif.tags.DateTimeOriginal || + exif.tags.ModifyDate + ) { + metadata.creationDate = + (exif.tags.DateTimeOriginal || + exif.tags.CreateDate || + exif.tags.ModifyDate) * 1000; + } + if (exif.imageSize) { + metadata.size = { + width: exif.imageSize.width, + height: exif.imageSize.height, + }; + } else if ( + exif.tags.RelatedImageWidth && + exif.tags.RelatedImageHeight + ) { + metadata.size = { + width: exif.tags.RelatedImageWidth, + height: exif.tags.RelatedImageHeight, + }; + } else if ( + exif.tags.ImageWidth && + exif.tags.ImageHeight + ) { + metadata.size = { + width: exif.tags.ImageWidth, + height: exif.tags.ImageHeight, + }; + } else { + const info = imageSize(fullPath); + metadata.size = {width: info.width, height: info.height}; + } + } catch (err) { + Logger.debug(LOG_TAG, 'Error parsing exif', fullPath, err); + try { + const info = imageSize(fullPath); + metadata.size = {width: info.width, height: info.height}; + } catch (e) { + metadata.size = {width: 1, height: 1}; + } + } + + try { + const iptcData = IptcParser.parse(data); + if (iptcData.country_or_primary_location_name) { + metadata.positionData = metadata.positionData || {}; + metadata.positionData.country = + iptcData.country_or_primary_location_name + .replace(/\0/g, '') + .trim(); + } + if (iptcData.province_or_state) { + metadata.positionData = metadata.positionData || {}; + metadata.positionData.state = iptcData.province_or_state .replace(/\0/g, '') .trim(); - } - if (iptcData.province_or_state) { - metadata.positionData = metadata.positionData || {}; - metadata.positionData.state = iptcData.province_or_state - .replace(/\0/g, '') - .trim(); - } - if (iptcData.city) { - metadata.positionData = metadata.positionData || {}; - metadata.positionData.city = iptcData.city - .replace(/\0/g, '') - .trim(); - } - if (iptcData.caption) { - metadata.caption = iptcData.caption.replace(/\0/g, '').trim(); - } - if (Array.isArray(iptcData.keywords)) { - metadata.keywords = iptcData.keywords; - } - - if (iptcData.date_time) { - metadata.creationDate = iptcData.date_time.getTime(); - } - } catch (err) { - // Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err); - } - - if (!metadata.creationDate) { - // creationDate can be negative, when it was created before epoch (1970) - metadata.creationDate = 0; - } - - try { - // TODO: clean up the three different exif readers, - // and keep the minimum amount only - const exif: ExifReader.Tags &ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data); - if (exif.Rating) { - metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5; - if (metadata.rating < 0) { - metadata.rating = 0; } - } - if ( - exif.subject && - exif.subject.value && - exif.subject.value.length > 0 - ) { - if (metadata.keywords === undefined) { - metadata.keywords = []; + if (iptcData.city) { + metadata.positionData = metadata.positionData || {}; + metadata.positionData.city = iptcData.city + .replace(/\0/g, '') + .trim(); } - for (const kw of exif.subject.value as ExifReader.XmpTag[]) { - if (metadata.keywords.indexOf(kw.description) === -1) { - metadata.keywords.push(kw.description); + if (iptcData.caption) { + metadata.caption = iptcData.caption.replace(/\0/g, '').trim(); + } + if (Array.isArray(iptcData.keywords)) { + metadata.keywords = iptcData.keywords; + } + + if (iptcData.date_time) { + metadata.creationDate = iptcData.date_time.getTime(); + } + } catch (err) { + // Logger.debug(LOG_TAG, 'Error parsing iptc data', fullPath, err); + } + + if (!metadata.creationDate) { + // creationDate can be negative, when it was created before epoch (1970) + metadata.creationDate = 0; + } + + try { + // TODO: clean up the three different exif readers, + // and keep the minimum amount only + const exif: ExifReader.Tags & ExifReader.XmpTags & ExifReader.IccTags = ExifReader.load(data); + if (exif.Rating) { + metadata.rating = parseInt(exif.Rating.value as string, 10) as 0 | 1 | 2 | 3 | 4 | 5; + if (metadata.rating < 0) { + metadata.rating = 0; } } - } - let orientation = OrientationTypes.TOP_LEFT; - if (exif.Orientation) { - orientation = parseInt( - exif.Orientation.value as any, - 10 - ) as number; - } - if (OrientationTypes.BOTTOM_LEFT < orientation) { - // noinspection JSSuspiciousNameCombination - const height = metadata.size.width; - // noinspection JSSuspiciousNameCombination - metadata.size.width = metadata.size.height; - metadata.size.height = height; - } - - if (Config.Faces.enabled) { - const faces: FaceRegion[] = []; if ( - ((exif.Regions as any)?.value?.RegionList)?.value + exif.subject && + exif.subject.value && + exif.subject.value.length > 0 ) { - for (const regionRoot of (exif.Regions as any).value.RegionList - .value as any[]) { - let type; - let name; - let box; - const createFaceBox = ( - w: string, - h: string, - x: string, - y: string - ) => { - if (OrientationTypes.BOTTOM_LEFT < orientation) { - [x, y] = [y, x]; - [w, h] = [h, w]; - } - let swapX = 0; - let swapY = 0; - switch (orientation) { - case OrientationTypes.TOP_RIGHT: - case OrientationTypes.RIGHT_TOP: - swapX = 1; - break; - case OrientationTypes.BOTTOM_RIGHT: - case OrientationTypes.RIGHT_BOTTOM: - swapX = 1; - swapY = 1; - break; - case OrientationTypes.BOTTOM_LEFT: - case OrientationTypes.LEFT_BOTTOM: - swapY = 1; - break; - } - // converting ratio to px - return { - width: Math.round(parseFloat(w) * metadata.size.width), - height: Math.round(parseFloat(h) * metadata.size.height), - left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width), - top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height), + if (metadata.keywords === undefined) { + metadata.keywords = []; + } + for (const kw of exif.subject.value as ExifReader.XmpTag[]) { + if (metadata.keywords.indexOf(kw.description) === -1) { + metadata.keywords.push(kw.description); + } + } + } + let orientation = OrientationTypes.TOP_LEFT; + if (exif.Orientation) { + orientation = parseInt( + exif.Orientation.value as any, + 10 + ) as number; + } + if (OrientationTypes.BOTTOM_LEFT < orientation) { + // noinspection JSSuspiciousNameCombination + const height = metadata.size.width; + // noinspection JSSuspiciousNameCombination + metadata.size.width = metadata.size.height; + metadata.size.height = height; + } + + if (Config.Faces.enabled) { + const faces: FaceRegion[] = []; + if ( + ((exif.Regions as any)?.value?.RegionList)?.value + ) { + for (const regionRoot of (exif.Regions as any).value.RegionList + .value as any[]) { + let type; + let name; + let box; + const createFaceBox = ( + w: string, + h: string, + x: string, + y: string + ) => { + if (OrientationTypes.BOTTOM_LEFT < orientation) { + [x, y] = [y, x]; + [w, h] = [h, w]; + } + let swapX = 0; + let swapY = 0; + switch (orientation) { + case OrientationTypes.TOP_RIGHT: + case OrientationTypes.RIGHT_TOP: + swapX = 1; + break; + case OrientationTypes.BOTTOM_RIGHT: + case OrientationTypes.RIGHT_BOTTOM: + swapX = 1; + swapY = 1; + break; + case OrientationTypes.BOTTOM_LEFT: + case OrientationTypes.LEFT_BOTTOM: + swapY = 1; + break; + } + // converting ratio to px + return { + width: Math.round(parseFloat(w) * metadata.size.width), + height: Math.round(parseFloat(h) * metadata.size.height), + left: Math.round(Math.abs(parseFloat(x) - swapX) * metadata.size.width), + top: Math.round(Math.abs(parseFloat(y) - swapY) * metadata.size.height), + }; }; - }; - /* Adobe Lightroom based face region structure */ - if ( - regionRoot.value && - regionRoot.value['rdf:Description'] && - regionRoot.value['rdf:Description'].value && - regionRoot.value['rdf:Description'].value['mwg-rs:Area'] - ) { - const region = regionRoot.value['rdf:Description']; - const regionBox = region.value['mwg-rs:Area'].attributes; + /* Adobe Lightroom based face region structure */ + if ( + regionRoot.value && + regionRoot.value['rdf:Description'] && + regionRoot.value['rdf:Description'].value && + regionRoot.value['rdf:Description'].value['mwg-rs:Area'] + ) { + const region = regionRoot.value['rdf:Description']; + const regionBox = region.value['mwg-rs:Area'].attributes; - name = region.attributes['mwg-rs:Name']; - type = region.attributes['mwg-rs:Type']; - box = createFaceBox( - regionBox['stArea:w'], - regionBox['stArea:h'], - regionBox['stArea:x'], - regionBox['stArea:y'] - ); - /* Load exiftool edited face region structure, see github issue #191 */ - } else if ( - regionRoot.Area && - regionRoot.Name && - regionRoot.Type - ) { - const regionBox = regionRoot.Area.value; - name = regionRoot.Name.value; - type = regionRoot.Type.value; - box = createFaceBox( - regionBox.w.value, - regionBox.h.value, - regionBox.x.value, - regionBox.y.value - ); - } - - if (type !== 'Face' || !name) { - continue; - } - - // convert center base box to corner based box - box.left = Math.round(Math.max(0, box.left - box.width / 2)); - box.top = Math.round(Math.max(0, box.top - box.height / 2)); - - - faces.push({name, box}); - } - } - if (faces.length > 0) { - metadata.faces = faces; // save faces - if (Config.Faces.keywordsToPersons) { - // remove faces from keywords - metadata.faces.forEach((f) => { - const index = metadata.keywords.indexOf(f.name); - if (index !== -1) { - metadata.keywords.splice(index, 1); + name = region.attributes['mwg-rs:Name']; + type = region.attributes['mwg-rs:Type']; + box = createFaceBox( + regionBox['stArea:w'], + regionBox['stArea:h'], + regionBox['stArea:x'], + regionBox['stArea:y'] + ); + /* Load exiftool edited face region structure, see github issue #191 */ + } else if ( + regionRoot.Area && + regionRoot.Name && + regionRoot.Type + ) { + const regionBox = regionRoot.Area.value; + name = regionRoot.Name.value; + type = regionRoot.Type.value; + box = createFaceBox( + regionBox.w.value, + regionBox.h.value, + regionBox.x.value, + regionBox.y.value + ); } - }); + + if (type !== 'Face' || !name) { + continue; + } + + // convert center base box to corner based box + box.left = Math.round(Math.max(0, box.left - box.width / 2)); + box.top = Math.round(Math.max(0, box.top - box.height / 2)); + + + faces.push({name, box}); + } + } + if (faces.length > 0) { + metadata.faces = faces; // save faces + if (Config.Faces.keywordsToPersons) { + // remove faces from keywords + metadata.faces.forEach((f) => { + const index = metadata.keywords.indexOf(f.name); + if (index !== -1) { + metadata.keywords.splice(index, 1); + } + }); + } } } + } catch (err) { + // ignoring errors } - } catch (err) { - // ignoring errors - } - return resolve(metadata); - } catch (err) { - return reject({file: fullPath, error: err}); - } - }); + return resolve(metadata); + } catch (err) { + return reject({file: fullPath, error: err}); + } + }); + } catch (err) { + Logger.error(LOG_TAG, 'Error during reading photo: ' + fullPath); + console.error(err); + return resolve(MetadataLoader.EMPTY_METADATA); + } }); + } }