Implementing XMP face region parsing

backend/model/sql/GalleryManager.ts

@@ -5,13 +5,13 @@ import * as fs from 'fs';
 import {DirectoryEntity} from './enitites/DirectoryEntity';
 import {SQLConnection} from './SQLConnection';
 import {DiskManager} from '../DiskManger';
-import {PhotoEntity} from './enitites/PhotoEntity';
+import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
 import {Utils} from '../../../common/Utils';
 import {ProjectPath} from '../../ProjectPath';
 import {Config} from '../../../common/config/private/Config';
 import {ISQLGalleryManager} from './IGalleryManager';
 import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig';
-import {PhotoDTO} from '../../../common/entities/PhotoDTO';
+import {FaceRegion, PhotoDTO, PhotoMetadata} from '../../../common/entities/PhotoDTO';
 import {OrientationType} from '../../../common/entities/RandomQueryDTO';
 import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm';
 import {MediaEntity} from './enitites/MediaEntity';
@@ -22,6 +22,8 @@ import {FileDTO} from '../../../common/entities/FileDTO';
 import {NotificationManager} from '../NotifocationManager';
 import {DiskMangerWorker} from '../threading/DiskMangerWorker';
 import {Logger} from '../../Logger';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {PersonEntry} from './enitites/PersonEntry';
 
 const LOG_TAG = '[GalleryManager]';
 
@@ -50,11 +52,23 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
 
   protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
     if (dir.media) {
+      const indexedFaces = await connection.getRepository(FaceRegionEntry)
+        .createQueryBuilder('face')
+        .leftJoinAndSelect('face.media', 'media')
+        .where('media.directory = :directory', {
+          directory: dir.id
+        })
+        .leftJoinAndSelect('face.person', 'person')
+        .getMany();
       for (let i = 0; i < dir.media.length; i++) {
         dir.media[i].directory = dir;
         dir.media[i].readyThumbnails = [];
         dir.media[i].readyIcon = false;
+        (<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
+          .filter(fe => fe.media.id === dir.media[i].id)
+          .map(f => ({box: f.box, name: f.person.name}));
       }
+
     }
     if (dir.directories) {
       for (let i = 0; i < dir.directories.length; i++) {
@@ -235,6 +249,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
   }
 
   protected async saveToDB(scannedDirectory: DirectoryDTO) {
+    console.log('saving');
     this.isSaving = true;
     try {
       const connection = await SQLConnection.getConnection();
@@ -259,7 +274,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
         currentDir = await directoryRepository.save(<DirectoryEntity>scannedDirectory);
       }
 
-
+      // TODO: fix when first opened directory is not root
       // save subdirectories
       const childDirectories = await directoryRepository.createQueryBuilder('directory')
         .where('directory.parent = :dir', {
@@ -299,10 +314,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
       await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
 
       // save media
-      const indexedMedia = await mediaRepository.createQueryBuilder('media')
+      const indexedMedia = (await mediaRepository.createQueryBuilder('media')
         .where('media.directory = :dir', {
           dir: currentDir.id
-        }).getMany();
+        })
+        .getMany());
 
 
       const mediaToSave = [];
@@ -315,18 +331,30 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
             break;
           }
         }
+
+
         if (media == null) { // not in DB yet
           scannedDirectory.media[i].directory = null;
           media = Utils.clone(scannedDirectory.media[i]);
           scannedDirectory.media[i].directory = scannedDirectory;
           media.directory = currentDir;
           mediaToSave.push(media);
-        } else if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) {
+        } else {
-          media.metadata = scannedDirectory.media[i].metadata;
+          delete (<PhotoMetadata>media.metadata).faces;
-          mediaToSave.push(media);
+
+          if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) {
+            media.metadata = scannedDirectory.media[i].metadata;
+            mediaToSave.push(media);
+          }
         }
+        const scannedFaces = (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
+        delete (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
+
+        const mediaEntry = await this.saveAMedia(connection, media);
+
+        await this.saveFaces(connection, mediaEntry, scannedFaces);
       }
-      await this.saveMedia(connection, mediaToSave);
+      // await this.saveMedia(connection, mediaToSave);
       await mediaRepository.remove(indexedMedia);
 
 
@@ -364,6 +392,64 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
     }
   }
 
+  protected async saveFaces(connection: Connection, media: MediaEntity, scannedFaces: FaceRegion[]) {
+
+    const faceRepository = connection.getRepository(FaceRegionEntry);
+    const personRepository = connection.getRepository(PersonEntry);
+    const indexedPersons = await personRepository.createQueryBuilder('person').getMany();
+
+    const indexedFaces = await faceRepository.createQueryBuilder('face')
+      .where('face.media = :media', {
+        media: media.id
+      })
+      .leftJoinAndSelect('face.person', 'person')
+      .getMany();
+
+
+    const getPerson = async (name: string) => {
+      let person = indexedPersons.find(p => p.name === name);
+      if (!person) {
+        person = <any>await personRepository.save({name: name});
+        indexedPersons.push(person);
+      }
+      return person;
+    };
+
+    const faceToSave = [];
+    for (let i = 0; i < scannedFaces.length; i++) {
+      let face: FaceRegionEntry = null;
+      for (let j = 0; j < indexedFaces.length; j++) {
+        if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
+          indexedFaces[j].box.width === scannedFaces[i].box.width &&
+          indexedFaces[j].box.x === scannedFaces[i].box.x &&
+          indexedFaces[j].box.y === scannedFaces[i].box.y &&
+          indexedFaces[j].person.name === scannedFaces[i].name) {
+          face = indexedFaces[j];
+          indexedFaces.splice(j, 1);
+          break;
+        }
+      }
+
+      if (face == null) {
+        (<FaceRegionEntry>scannedFaces[i]).person = await getPerson(scannedFaces[i].name);
+        (<FaceRegionEntry>scannedFaces[i]).media = media;
+        //   console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person, (<FaceRegionEntry>scannedFaces[i]).media);
+        //  console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person.id, (<FaceRegionEntry>scannedFaces[i]).media.id);
+        faceToSave.push(scannedFaces[i]);
+      }
+    }
+    await faceRepository.save(faceToSave, {chunk: Math.max(Math.ceil(faceToSave.length / 500), 1)});
+    await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
+
+  }
+
+  protected async saveAMedia(connection: Connection, media: MediaDTO): Promise<MediaEntity> {
+    if (MediaDTO.isPhoto(media)) {
+      return await <any>connection.getRepository(PhotoEntity).save(media);
+    }
+    return await <any>connection.getRepository(VideoEntity).save(media);
+  }
+
   protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise<MediaEntity[]> {
     const chunked = Utils.chunkArrays(mediaList, 100);
     let list: MediaEntity[] = [];

backend/model/sql/SQLConnection.ts

@@ -15,6 +15,8 @@ import {MediaEntity} from './enitites/MediaEntity';
 import {VideoEntity} from './enitites/VideoEntity';
 import {DataStructureVersion} from '../../../common/DataStructureVersion';
 import {FileEntity} from './enitites/FileEntity';
+import {FaceRegionEntry} from './enitites/FaceRegionEntry';
+import {PersonEntry} from './enitites/PersonEntry';
 
 
 export class SQLConnection {
@@ -32,6 +34,8 @@ export class SQLConnection {
       options.entities = [
         UserEntity,
         FileEntity,
+        FaceRegionEntry,
+        PersonEntry,
         MediaEntity,
         PhotoEntity,
         VideoEntity,
@@ -40,7 +44,7 @@ export class SQLConnection {
         VersionEntity
       ];
       options.synchronize = false;
-    //  options.logging = 'all';
+      options.logging = 'all';
       this.connection = await createConnection(options);
       await SQLConnection.schemeSync(this.connection);
     }
@@ -57,6 +61,8 @@ export class SQLConnection {
     options.entities = [
       UserEntity,
       FileEntity,
+      FaceRegionEntry,
+      PersonEntry,
       MediaEntity,
       PhotoEntity,
       VideoEntity,

backend/model/sql/enitites/FaceRegionEntry.ts

@@ -0,0 +1,38 @@
+import {FaceRegionBox} from '../../../../common/entities/PhotoDTO';
+import {Column, ManyToOne, Entity, PrimaryGeneratedColumn} from 'typeorm';
+import {PersonEntry} from './PersonEntry';
+import {MediaEntity, MediaMetadataEntity} from './MediaEntity';
+
+export class FaceRegionBoxEntry implements FaceRegionBox {
+  @Column('int')
+  height: number;
+  @Column('int')
+  width: number;
+  @Column('int')
+  x: number;
+  @Column('int')
+  y: number;
+}
+
+/**
+ * This is a switching table between media and persons
+ */
+@Entity()
+export class FaceRegionEntry {
+
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column(type => FaceRegionBoxEntry)
+  box: FaceRegionBoxEntry;
+
+  // @PrimaryColumn('int')
+  @ManyToOne(type => MediaEntity, media => media.metadata.faces, {onDelete: 'CASCADE', nullable: false})
+  media: MediaEntity;
+
+  // @PrimaryColumn('int')
+  @ManyToOne(type => PersonEntry, person => person.faces, {onDelete: 'CASCADE', nullable: false})
+  person: PersonEntry;
+
+  name: string;
+}

backend/model/sql/enitites/MediaEntity.ts

@@ -1,8 +1,9 @@
-import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm';
+import {Column, Entity, OneToMany, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm';
 import {DirectoryEntity} from './DirectoryEntity';
 import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entities/MediaDTO';
 import {OrientationTypes} from 'ts-exif-parser';
 import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity';
+import {FaceRegionEntry} from './FaceRegionEntry';
 
 export class MediaDimensionEntity implements MediaDimension {
 
@@ -27,7 +28,6 @@ export class MediaMetadataEntity implements MediaMetadata {
   @Column('int')
   fileSize: number;
 
-
   @Column('simple-array')
   keywords: string[];
 
@@ -40,6 +40,9 @@ export class MediaMetadataEntity implements MediaMetadata {
   @Column('tinyint', {default: OrientationTypes.TOP_LEFT})
   orientation: OrientationTypes;
 
+  @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media)
+  faces: FaceRegionEntry[];
+
   @Column('int')
   bitRate: number;
 

backend/model/sql/enitites/PersonEntry.ts

@@ -0,0 +1,17 @@
+import {Column, Entity,  OneToMany, PrimaryGeneratedColumn,  Unique, Index} from 'typeorm';
+import {FaceRegionEntry} from './FaceRegionEntry';
+
+
+@Entity()
+@Unique(['name'])
+export class PersonEntry {
+  @Index()
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column()
+  name: string;
+
+  @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.person)
+  public faces: FaceRegionEntry[];
+}

backend/model/sql/enitites/PhotoEntity.ts

@@ -1,6 +1,13 @@
 import {Column, Entity, ChildEntity, Unique} from 'typeorm';
-import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../common/entities/PhotoDTO';
+import {
-import {OrientationTypes} from 'ts-exif-parser';
+  CameraMetadata,
+  FaceRegion,
+  FaceRegionBox,
+  GPSMetadata,
+  PhotoDTO,
+  PhotoMetadata,
+  PositionMetaData
+} from '../../../../common/entities/PhotoDTO';
 import {MediaEntity, MediaMetadataEntity} from './MediaEntity';
 
 export class CameraMetadataEntity implements CameraMetadata {
@@ -38,6 +45,7 @@ export class GPSMetadataEntity implements GPSMetadata {
   altitude: number;
 }
 
+
 export class PositionMetaDataEntity implements PositionMetaData {
 
   @Column(type => GPSMetadataEntity)
@@ -75,5 +83,4 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet
 export class PhotoEntity extends MediaEntity implements PhotoDTO {
   @Column(type => PhotoMetadataEntity)
   metadata: PhotoMetadataEntity;
-
 }

backend/model/threading/MetadataLoader.ts

@@ -1,5 +1,5 @@
 import {VideoMetadata} from '../../../common/entities/VideoDTO';
-import {PhotoMetadata} from '../../../common/entities/PhotoDTO';
+import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO';
 import {Config} from '../../../common/config/private/Config';
 import {Logger} from '../../Logger';
 import * as fs from 'fs';
@@ -9,6 +9,14 @@ import {IptcParser} from 'ts-node-iptc';
 import {FFmpegFactory} from '../FFmpegFactory';
 import {FfprobeData} from 'fluent-ffmpeg';
 
+// TODO: fix up different metadata loaders
+// @ts-ignore
+global.DataView = require('jdataview');
+// @ts-ignore
+global.DOMParser = require('xmldom').DOMParser;
+// @ts-ignore
+const ExifReader = require('exifreader');
+
 const LOG_TAG = '[MetadataLoader]';
 const ffmpeg = FFmpegFactory.get();
 
@@ -157,6 +165,47 @@ export class MetadataLoader {
 
             metadata.creationDate = metadata.creationDate || 0;
 
+
+            try {
+              const ret = ExifReader.load(data);
+              const faces: FaceRegion[] = [];
+              if (ret.Regions && ret.Regions.value.RegionList && ret.Regions.value.RegionList.value) {
+                for (let i = 0; i < ret.Regions.value.RegionList.value.length; i++) {
+                  if (!ret.Regions.value.RegionList.value[i].value ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'] ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value ||
+                    !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area']) {
+                    continue;
+                  }
+                  const region = ret.Regions.value.RegionList.value[i].value['rdf:Description'];
+                  const regionBox = ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area'].attributes;
+                  if (region.attributes['mwg-rs:Type'] !== 'Face' ||
+                    !region.attributes['mwg-rs:Name']) {
+                    continue;
+                  }
+                  const name = region.attributes['mwg-rs:Name'];
+                  const box = {
+                    width: Math.round(regionBox['stArea:w'] * metadata.size.width),
+                    height: Math.round(regionBox['stArea:h'] * metadata.size.height),
+                    x: Math.round(regionBox['stArea:x'] * metadata.size.width),
+                    y: Math.round(regionBox['stArea:y'] * metadata.size.height)
+                  };
+                  faces.push({name: name, box: box});
+                }
+              }
+              if (faces.length > 0) {
+                metadata.faces = faces; // save faces
+                // remove faces from keywords
+                metadata.faces.forEach(f => {
+                  const index = metadata.keywords.indexOf(f.name);
+                  if (index !== -1) {
+                    metadata.keywords.splice(index, 1);
+                  }
+                });
+              }
+            } catch (err) {
+            }
+
             return resolve(metadata);
           } catch (err) {
             return reject({file: fullPath, error: err});

common/DataStructureVersion.ts

@@ -1 +1 @@
-export const DataStructureVersion = 7;
+export const DataStructureVersion = 8;

common/entities/PhotoDTO.ts

@@ -11,6 +11,18 @@ export interface PhotoDTO extends MediaDTO {
   readyIcon: boolean;
 }
 
+export interface FaceRegionBox {
+  width: number;
+  height: number;
+  x: number;
+  y: number;
+}
+
+export interface FaceRegion {
+  name: string;
+  box: FaceRegionBox;
+}
+
 export interface PhotoMetadata extends MediaMetadata {
   caption?: string;
   keywords?: string[];
@@ -20,6 +32,7 @@ export interface PhotoMetadata extends MediaMetadata {
   size: MediaDimension;
   creationDate: number;
   fileSize: number;
+  faces?: FaceRegion[];
 }
 
 

package.json

@@ -30,9 +30,11 @@
     "cookie-parser": "1.4.3",
     "cookie-session": "2.0.0-beta.3",
     "ejs": "2.6.1",
+    "exifreader": "2.5.0",
     "express": "4.16.4",
     "fluent-ffmpeg": "2.1.2",
     "image-size": "0.6.3",
+    "jdataview": "2.5.0",
     "jimp": "0.6.0",
     "locale": "0.1.0",
     "reflect-metadata": "0.1.12",
@@ -41,7 +43,8 @@
     "ts-node-iptc": "1.0.11",
     "typeconfig": "1.0.7",
     "typeorm": "0.2.9",
-    "winston": "2.4.2"
+    "winston": "2.4.2",
+    "xmldom": "0.1.27"
   },
   "devDependencies": {
     "@angular-devkit/build-angular": "0.11.4",