xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00
Code Actions Projects Activity

new date and offset loading. More generic easier to add new elements

Browse Source
This commit is contained in:
gras 2024-04-18 20:45:08 +02:00
parent 5dc1d62863
commit 7dd09100d4
13 changed files with 227 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/backend/model/fileaccess/MetadataCreationDate.ts
View File

@ -13,13 +13,14 @@ export const DateTags: [string, string, string][] = [
["ifd0.ModifyDate", undefined, undefined], //The date and time of image creation. In Exif standard, it is the date and time the file was changed.
["ihdr.Creation Time", undefined, undefined], //Time of original image creation for PNG files
["photoshop.DateCreated", undefined, undefined], //The date the intellectual content of the document was created. Used and set by LightRoom among others
["xmp:CreateDate", undefined, undefined], //Date and time when the image was created (XMP standard)
["xmp.CreateDate", undefined, undefined], //Date and time when the image was created (XMP standard)
["iptc.DateCreated", "iptc.TimeCreated", 'T'], //Designates the date and optionally the time the content of the image was created rather than the date of the creation of the digital representation
["quicktime.CreationDate", undefined, undefined], //Date and time when the QuickTime movie was created"],
["quicktime.CreateDate", undefined, undefined], //Date and time when the QuickTime movie was created in UTC"],
["heic.ContentCreateDate", undefined, undefined], //Date and time when the HEIC image content was created"],
["heic.CreationDate", undefined, undefined], //Date and time when the HEIC image was created"],
["tiff.DateTime", undefined, undefined], //Date and time of image creation. This property is stored in XMP as xmp:ModifyDate.
["xmp:ModifyDate", "exif.OffsetTime", 'O'], //Date and time when the image was last modified (XMP standard)"]
["tiff.DateTime", undefined, undefined], //Date and time of image creation.
["exif.ModifyDate", "exif.OffsetTime", 'O'], //Modification date
["xmp.ModifyDate", undefined, undefined], //Date and time when the image was last modified (XMP standard)"]
["xmp.MetadataDate", undefined, undefined], //The date and time that any metadata for this resource was last changed. It should be the same as or more recent than xmp:ModifyDate.
];

122
src/backend/model/fileaccess/MetadataLoader.ts
View File

@ -250,6 +250,7 @@ export class MetadataLoader {
const sidecarData: any = await exifr.sidecar(sidecarPath, exifrOptions);
if (sidecarData !== undefined) {
MetadataLoader.mapMetadata(metadata, sidecarData);
break;
}
}
}
@ -285,7 +286,6 @@ export class MetadataLoader {
MetadataLoader.mapKeywords(metadata, exif);
MetadataLoader.mapTitle(metadata, exif);
MetadataLoader.mapCaption(metadata, exif);
MetadataLoader.mapTimestampAndOffset2(metadata, exif);
MetadataLoader.mapTimestampAndOffset(metadata, exif);
MetadataLoader.mapCameraData(metadata, exif);
MetadataLoader.mapGPS(metadata, exif);
@ -362,73 +362,69 @@ export class MetadataLoader {
metadata.caption = exif.dc?.description?.value || Utils.asciiToUTF8(exif.iptc?.Caption) || metadata.caption || exif.ifd0?.ImageDescription || exif.exif?.UserComment?.value || exif.Iptc4xmpCore?.ExtDescrAccessibility?.value ||exif.acdsee?.notes;
}
private static getValue(exif: any, path: string): any {
const pathElements = path.split('.');
let currentObject: any = exif;
for (const pathElm of pathElements) {
const tmp = currentObject[pathElm];
if (tmp === undefined) {
return undefined;
}
currentObject = tmp;
}
return currentObject;
}
private static mapTimestampAndOffset2(metadata: PhotoMetadata, exif: any) {
let ts;
for (const [mainpath, extrapath, extratype] of DateTags) {
ts = MetadataLoader.getValue(exif, mainpath);
}
}
private static mapTimestampAndOffset(metadata: PhotoMetadata, exif: any) {
metadata.creationDate = Utils.timestampToMS(exif?.photoshop?.DateCreated, null) ||
Utils.timestampToMS(exif?.xmp?.CreateDate, null) ||
Utils.timestampToMS(exif?.xmp?.ModifyDate, null) ||
Utils.timestampToMS(Utils.toIsoTimestampString(exif?.iptc?.DateCreated, exif?.iptc?.TimeCreated), null) ||
metadata.creationDate;
metadata.creationDateOffset = Utils.timestampToOffsetString(exif?.photoshop?.DateCreated) ||
Utils.timestampToOffsetString(exif?.xmp?.CreateDate) ||
metadata.creationDateOffset;
if (exif.exif) {
let offset = undefined;
//Preceedence of dates: exif.DateTimeOriginal, exif.CreateDate, ifd0.ModifyDate, ihdr["Creation Time"], xmp.MetadataDate, file system date
//Filesystem is the absolute last resort, and it's hard to write tests for, since file system dates are changed on e.g. git clone.
if (exif.exif.DateTimeOriginal) {
//DateTimeOriginal is when the camera shutter closed
offset = exif.exif.OffsetTimeOriginal; //OffsetTimeOriginal is the corresponding offset
if (!offset) { //Find offset among other options if possible
offset = exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || Utils.getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
let ts: string, offset: string;
for (let i = 0; i < DateTags.length; i++) {
const [mainpath, extrapath, extratype] = DateTags[i];
[ts, offset] = extractTSAndOffset(mainpath, extrapath, extratype);
if (ts) {
if (!offset) { //We don't have the offset from the timestamp or from extra tag, let's see if we can find it in another way
//Check the explicit offset tags. Otherwise calculate from GPS
offset = exif.exif?.OffsetTimeOriginal || exif.exif?.OffsetTimeDigitized || exif.exif?.OffsetTime || Utils.getTimeOffsetByGPSStamp(ts, exif.exif?.GPSTimeStamp, exif.gps);
}
metadata.creationDate = Utils.timestampToMS(exif.exif.DateTimeOriginal, offset) || metadata.creationDate;
} else if (exif.exif.CreateDate) { //using else if here, because DateTimeOriginal has preceedence
//Create is when the camera wrote the file (typically within the same ms as shutter close)
offset = exif.exif.OffsetTimeDigitized; //OffsetTimeDigitized is the corresponding offset
if (!offset) { //Find offset among other options if possible
offset = exif.exif.OffsetTimeOriginal || exif.exif.OffsetTime || Utils.getTimeOffsetByGPSStamp(exif.exif.DateTimeOriginal, exif.exif.GPSTimeStamp, exif.gps);
if (!offset) { //still no offset? let's look for a timestamp with offset in the rest of the DateTags list
const [tsonly, tsoffset] = Utils.splitTimestampAndOffset(ts);
for (let j = i+1; j < DateTags.length; j++) {
const [exts, exOffset] = extractTSAndOffset(DateTags[j][0], DateTags[j][1], DateTags[j][2]);
if (exts && exOffset && Math.abs(Utils.timestampToMS(tsonly, null) - Utils.timestampToMS(exts, null)) < 30000) {
//if there is an offset and the found timestamp is within 30 seconds of the extra timestamp, we will use the offset from the found timestamp
offset = exOffset;
break;
}
}
}
metadata.creationDate = Utils.timestampToMS(exif.exif.CreateDate, offset) || metadata.creationDate;
} else if (exif.ifd0?.ModifyDate) { //using else if here, because DateTimeOriginal and CreatDate have preceedence
offset = exif.exif.OffsetTime; //exif.Offsettime is the offset corresponding to ifd0.ModifyDate
if (!offset) { //Find offset among other options if possible
offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || Utils.getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
}
metadata.creationDate = Utils.timestampToMS(exif.ifd0.ModifyDate, offset) || metadata.creationDate;
} else if (exif.ihdr && exif.ihdr["Creation Time"]) {// again else if (another fallback date if the good ones aren't there) {
const any_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || Utils.getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = Utils.timestampToMS(exif.ihdr["Creation Time"], any_offset);
offset = any_offset;
} else if (exif.xmp?.MetadataDate) {// again else if (another fallback date if the good ones aren't there - metadata date is probably later than actual creation date, but much better than file time) {
const any_offset = exif.exif.DateTimeOriginal || exif.exif.OffsetTimeDigitized || exif.exif.OffsetTime || Utils.getTimeOffsetByGPSStamp(exif.ifd0.ModifyDate, exif.exif.GPSTimeStamp, exif.gps);
metadata.creationDate = Utils.timestampToMS(exif.xmp.MetadataDate, any_offset) || metadata.creationDate;
offset = any_offset;
break; //timestamp is found, look no further
}
metadata.creationDateOffset = offset || metadata.creationDateOffset;
}
metadata.creationDate = Utils.timestampToMS(ts, offset) || metadata.creationDate;
metadata.creationDateOffset = offset || metadata.creationDateOffset;
//---- End of mapTimestampAndOffset logic ----
//---- Helper functions for mapTimestampAndOffset ----
function getValue(exif: any, path: string): any {
const pathElements = path.split('.');
let currentObject: any = exif;
for (const pathElm of pathElements) {
const tmp = currentObject[pathElm];
if (tmp === undefined) {
return undefined;
}
currentObject = tmp;
}
return currentObject;
}
function extractTSAndOffset(mainpath: string, extrapath: string, extratype: string) {
let ts: string | undefined = undefined;
let offset: string | undefined = undefined;
//line below is programmatic way of finding a timestamp in the exif object. For example "xmp.CreateDate", from the DateTags list
//ts = exif.xmp?.CreateDate
ts = getValue(exif, mainpath);
if (ts) {
if (!extratype || extratype == 'O') { //offset can be in the timestamp itself
[ts, offset] = Utils.splitTimestampAndOffset(ts);
if (extratype == 'O' && !offset) { //offset in the extra tag and not already extracted from main tag
offset = getValue(exif, extrapath);
}
} else if (extratype == 'T') { //date only in main tag, time in the extra tag
ts = Utils.toIsoTimestampString(ts, getValue(exif, extrapath));
[ts, offset] = Utils.splitTimestampAndOffset(ts);
}
}
return [ts, offset];
}
}
private static mapCameraData(metadata: PhotoMetadata, exif: any) {

28
src/common/Utils.ts
View File

@ -56,6 +56,7 @@ export class Utils {
return c;
}
//TODO: use zeroPad which works for longer values?
static zeroPrefix(value: string | number, length: number): string {
const ret = '00000' + value;
return ret.substr(ret.length - length);
@ -118,7 +119,6 @@ export class Utils {
}
}
static makeUTCMidnight(d: number | Date) {
if (!(d instanceof Date)) {
d = new Date(d);
@ -162,22 +162,22 @@ export class Utils {
return Date.parse(formattedTimestamp);
}
//function to extract offset string from timestamp string, returns undefined if timestamp does not contain offset
static timestampToOffsetString(timestamp: string) {
try {
const offsetRegex = /[+-]\d{2}:\d{2}$/;
const match = timestamp.match(offsetRegex);
if (match) {
return match[0];
} else if (timestamp.indexOf("Z") > 0) {
return '+00:00';
}
return undefined;
} catch (err) {
return undefined;
static splitTimestampAndOffset(timestamp: string): [string|undefined, string|undefined] {
if (!timestamp) {
return [undefined, undefined];
}
// |---------------------TIMESTAMP WITH OPTIONAL MILLISECONDS--------------------||-OPTIONAL TZONE--|
// |YYYY MM DD HH MM SS (MS optio)||(timezone offset)|
const timestampWithOffsetRegex = /^(\d{4}[-.: ]\d{2}[-.: ]\d{2}[-.: T]\d{2}[-.: ]\d{2}[-.: ]\d{2}(?:\.\d+)?)([+-]\d{2}:\d{2})?$/;
const match = timestamp.match(timestampWithOffsetRegex);
if (match) {
return [match[1], match[2]]; //match[0] is the full string, not interested in that.
} else {
return [undefined, undefined];
}
}
//function to calculate offset from exif.exif.gpsTimeStamp or exif.gps.GPSDateStamp + exif.gps.GPSTimestamp
static getTimeOffsetByGPSStamp(timestamp: string, gpsTimeStamp: string, gps: any) {
let UTCTimestamp = gpsTimeStamp;

18
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203430-Auckland.json Normal file
View File

@ -0,0 +1,18 @@
{
"creationDate": 1713040470000,
"creationDateOffset": "+12:00",
"fileSize": 62150,
"positionData": {
"GPSData": {
"latitude": -36.854351,
"longitude": 174.755402
},
"city": "Arch Hill",
"country": "New Zealand",
"state": "Auckland"
},
"size": {
"height": 1080,
"width": 1920
}
}

18
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203530-Montevideo.json Normal file
View File

@ -0,0 +1,18 @@
{
"creationDate": 1713040530000,
"creationDateOffset": "-03:00",
"fileSize": 64033,
"positionData": {
"GPSData": {
"latitude": -34.862412,
"longitude": -56.154556
},
"city": "Jacinto Vera",
"country": "Uruguay",
"state": "Montevideo"
},
"size": {
"height": 1080,
"width": 1920
}
}

17
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203531-Kathmandu.json Normal file
View File

@ -0,0 +1,17 @@
{
"creationDate": 1713040531000,
"creationDateOffset": "+05:45",
"fileSize": 62443,
"positionData": {
"GPSData": {
"latitude": 27.693999,
"longitude": 85.285262
},
"country": "Nepal",
"state": "Bagmati Province"
},
"size": {
"height": 1080,
"width": 1920
}
}

18
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203630-London.json Normal file
View File

@ -0,0 +1,18 @@
{
"creationDate": 1713040590000,
"creationDateOffset": "+01:00",
"fileSize": 60679,
"positionData": {
"GPSData": {
"latitude": 51.515618,
"longitude": -0.091998
},
"city": "Bassishaw",
"country": "Storbritannien",
"state": "England"
},
"size": {
"height": 1080,
"width": 1920
}
}

18
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203720-Tokyo.json Normal file
View File

@ -0,0 +1,18 @@
{
"creationDate": 1713040640000,
"creationDateOffset": "+09:00",
"fileSize": 60451,
"positionData": {
"GPSData": {
"latitude": 35.682194,
"longitude": 139.762221
},
"city": "Kokyogaien",
"country": "Japan",
"state": "Tokyo To"
},
"size": {
"height": 1080,
"width": 1920
}
}

17
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203730-Reykjavik.json Normal file
View File

@ -0,0 +1,17 @@
{
"creationDate": 1713040650000,
"creationDateOffset": "+00:00",
"fileSize": 59743,
"positionData": {
"GPSData": {
"latitude": 64.141166,
"longitude": -21.94313
},
"city": "Reykjavík - 1",
"country": "Island"
},
"size": {
"height": 1080,
"width": 1920
}
}

18
test/backend/assets/4MinsAroundTheWorld/UTC-20240413-203830-NewYork.json Normal file
View File

@ -0,0 +1,18 @@
{
"creationDate": 1713040710000,
"creationDateOffset": "-04:00",
"fileSize": 62918,
"positionData": {
"GPSData": {
"latitude": 40.712728,
"longitude": -74.006015
},
"city": "New York",
"country": "USA",
"state": "New York"
},
"size": {
"height": 1080,
"width": 1920
}
}

3
test/backend/assets/edge_case_exif_data/no_metadata.json
View File

@ -1,6 +1,7 @@
{
"skip": [
"creationDate"
"creationDate",
"creationDateOffset"
],
"fileSize": 5860,
"size": {

2
test/backend/assets/sidecar/flatxmp.json
View File

@ -4,7 +4,7 @@
"height": 5
},
"caption": "Description of image",
"creationDate": 328817998000,
"creationDate": 328839598000,
"creationDateOffset": "-06:00",
"faces": [
{

23
test/backend/unit/model/threading/MetaDataLoader.spec.ts
View File

@ -268,6 +268,7 @@ describe('MetadataLoader', () => {
const expected = require(path.join(__dirname, '/../../../assets/orientation/Landscape.json'));
delete data.fileSize;
delete data.creationDate;
delete data.creationDateOffset;
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
it('Portrait ' + i, async () => {
@ -275,6 +276,7 @@ describe('MetadataLoader', () => {
const expected = require(path.join(__dirname, '/../../../assets/orientation/Portrait.json'));
delete data.fileSize;
delete data.creationDate;
delete data.creationDateOffset;
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
}
@ -334,4 +336,25 @@ describe('MetadataLoader', () => {
}
}
});
describe('should load metadata from files with times and coordinates in different parts of the world', () => {
const root = path.join(__dirname, '/../../../assets/4MinsAroundTheWorld');
const files = fs.readdirSync(root);
for (const item of files) {
const fullFilePath = path.join(root, item);
if (PhotoProcessing.isPhoto(fullFilePath)) {
it(item, async () => {
const data = await MetadataLoader.loadPhotoMetadata(fullFilePath);
const expected = require(fullFilePath.split('.').slice(0, -1).join('.') + '.json');
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
} else if (VideoProcessing.isVideo(fullFilePath)) {
it(item, async () => {
const data = await MetadataLoader.loadVideoMetadata(fullFilePath);
const expected = require(fullFilePath.split('.').slice(0, -1).join('.') + '.json');
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
}
}
});
});