diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 752eb13d..45e948a2 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -5,13 +5,13 @@ import * as fs from 'fs'; import {DirectoryEntity} from './enitites/DirectoryEntity'; import {SQLConnection} from './SQLConnection'; import {DiskManager} from '../DiskManger'; -import {PhotoEntity} from './enitites/PhotoEntity'; +import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity'; import {Utils} from '../../../common/Utils'; import {ProjectPath} from '../../ProjectPath'; import {Config} from '../../../common/config/private/Config'; import {ISQLGalleryManager} from './IGalleryManager'; import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig'; -import {PhotoDTO} from '../../../common/entities/PhotoDTO'; +import {FaceRegion, PhotoDTO, PhotoMetadata} from '../../../common/entities/PhotoDTO'; import {OrientationType} from '../../../common/entities/RandomQueryDTO'; import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm'; import {MediaEntity} from './enitites/MediaEntity'; @@ -22,6 +22,8 @@ import {FileDTO} from '../../../common/entities/FileDTO'; import {NotificationManager} from '../NotifocationManager'; import {DiskMangerWorker} from '../threading/DiskMangerWorker'; import {Logger} from '../../Logger'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {PersonEntry} from './enitites/PersonEntry'; const LOG_TAG = '[GalleryManager]'; @@ -50,11 +52,23 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { if (dir.media) { + const indexedFaces = await connection.getRepository(FaceRegionEntry) + .createQueryBuilder('face') + .leftJoinAndSelect('face.media', 'media') + .where('media.directory = :directory', { + directory: dir.id + }) + .leftJoinAndSelect('face.person', 'person') + .getMany(); for (let i = 0; i < dir.media.length; i++) { dir.media[i].directory = dir; dir.media[i].readyThumbnails = []; dir.media[i].readyIcon = false; + (dir.media[i]).metadata.faces = indexedFaces + .filter(fe => fe.media.id === dir.media[i].id) + .map(f => ({box: f.box, name: f.person.name})); } + } if (dir.directories) { for (let i = 0; i < dir.directories.length; i++) { @@ -235,6 +249,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { } protected async saveToDB(scannedDirectory: DirectoryDTO) { + console.log('saving'); this.isSaving = true; try { const connection = await SQLConnection.getConnection(); @@ -259,7 +274,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { currentDir = await directoryRepository.save(scannedDirectory); } - + // TODO: fix when first opened directory is not root // save subdirectories const childDirectories = await directoryRepository.createQueryBuilder('directory') .where('directory.parent = :dir', { @@ -299,10 +314,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)}); // save media - const indexedMedia = await mediaRepository.createQueryBuilder('media') + const indexedMedia = (await mediaRepository.createQueryBuilder('media') .where('media.directory = :dir', { dir: currentDir.id - }).getMany(); + }) + .getMany()); const mediaToSave = []; @@ -315,18 +331,30 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { break; } } + + if (media == null) { // not in DB yet scannedDirectory.media[i].directory = null; media = Utils.clone(scannedDirectory.media[i]); scannedDirectory.media[i].directory = scannedDirectory; media.directory = currentDir; mediaToSave.push(media); - } else if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) { - media.metadata = scannedDirectory.media[i].metadata; - mediaToSave.push(media); + } else { + delete (media.metadata).faces; + + if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) { + media.metadata = scannedDirectory.media[i].metadata; + mediaToSave.push(media); + } } + const scannedFaces = (scannedDirectory.media[i].metadata).faces; + delete (scannedDirectory.media[i].metadata).faces; + + const mediaEntry = await this.saveAMedia(connection, media); + + await this.saveFaces(connection, mediaEntry, scannedFaces); } - await this.saveMedia(connection, mediaToSave); + // await this.saveMedia(connection, mediaToSave); await mediaRepository.remove(indexedMedia); @@ -364,6 +392,64 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { } } + protected async saveFaces(connection: Connection, media: MediaEntity, scannedFaces: FaceRegion[]) { + + const faceRepository = connection.getRepository(FaceRegionEntry); + const personRepository = connection.getRepository(PersonEntry); + const indexedPersons = await personRepository.createQueryBuilder('person').getMany(); + + const indexedFaces = await faceRepository.createQueryBuilder('face') + .where('face.media = :media', { + media: media.id + }) + .leftJoinAndSelect('face.person', 'person') + .getMany(); + + + const getPerson = async (name: string) => { + let person = indexedPersons.find(p => p.name === name); + if (!person) { + person = await personRepository.save({name: name}); + indexedPersons.push(person); + } + return person; + }; + + const faceToSave = []; + for (let i = 0; i < scannedFaces.length; i++) { + let face: FaceRegionEntry = null; + for (let j = 0; j < indexedFaces.length; j++) { + if (indexedFaces[j].box.height === scannedFaces[i].box.height && + indexedFaces[j].box.width === scannedFaces[i].box.width && + indexedFaces[j].box.x === scannedFaces[i].box.x && + indexedFaces[j].box.y === scannedFaces[i].box.y && + indexedFaces[j].person.name === scannedFaces[i].name) { + face = indexedFaces[j]; + indexedFaces.splice(j, 1); + break; + } + } + + if (face == null) { + (scannedFaces[i]).person = await getPerson(scannedFaces[i].name); + (scannedFaces[i]).media = media; + // console.log('inserting', (scannedFaces[i]).person, (scannedFaces[i]).media); + // console.log('inserting', (scannedFaces[i]).person.id, (scannedFaces[i]).media.id); + faceToSave.push(scannedFaces[i]); + } + } + await faceRepository.save(faceToSave, {chunk: Math.max(Math.ceil(faceToSave.length / 500), 1)}); + await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)}); + + } + + protected async saveAMedia(connection: Connection, media: MediaDTO): Promise { + if (MediaDTO.isPhoto(media)) { + return await connection.getRepository(PhotoEntity).save(media); + } + return await connection.getRepository(VideoEntity).save(media); + } + protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise { const chunked = Utils.chunkArrays(mediaList, 100); let list: MediaEntity[] = []; diff --git a/backend/model/sql/SQLConnection.ts b/backend/model/sql/SQLConnection.ts index 0afc79f9..658e863a 100644 --- a/backend/model/sql/SQLConnection.ts +++ b/backend/model/sql/SQLConnection.ts @@ -15,6 +15,8 @@ import {MediaEntity} from './enitites/MediaEntity'; import {VideoEntity} from './enitites/VideoEntity'; import {DataStructureVersion} from '../../../common/DataStructureVersion'; import {FileEntity} from './enitites/FileEntity'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {PersonEntry} from './enitites/PersonEntry'; export class SQLConnection { @@ -32,6 +34,8 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + FaceRegionEntry, + PersonEntry, MediaEntity, PhotoEntity, VideoEntity, @@ -40,7 +44,7 @@ export class SQLConnection { VersionEntity ]; options.synchronize = false; - // options.logging = 'all'; + options.logging = 'all'; this.connection = await createConnection(options); await SQLConnection.schemeSync(this.connection); } @@ -57,6 +61,8 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + FaceRegionEntry, + PersonEntry, MediaEntity, PhotoEntity, VideoEntity, diff --git a/backend/model/sql/enitites/FaceRegionEntry.ts b/backend/model/sql/enitites/FaceRegionEntry.ts new file mode 100644 index 00000000..f9d1afd1 --- /dev/null +++ b/backend/model/sql/enitites/FaceRegionEntry.ts @@ -0,0 +1,38 @@ +import {FaceRegionBox} from '../../../../common/entities/PhotoDTO'; +import {Column, ManyToOne, Entity, PrimaryGeneratedColumn} from 'typeorm'; +import {PersonEntry} from './PersonEntry'; +import {MediaEntity, MediaMetadataEntity} from './MediaEntity'; + +export class FaceRegionBoxEntry implements FaceRegionBox { + @Column('int') + height: number; + @Column('int') + width: number; + @Column('int') + x: number; + @Column('int') + y: number; +} + +/** + * This is a switching table between media and persons + */ +@Entity() +export class FaceRegionEntry { + + @PrimaryGeneratedColumn() + id: number; + + @Column(type => FaceRegionBoxEntry) + box: FaceRegionBoxEntry; + + // @PrimaryColumn('int') + @ManyToOne(type => MediaEntity, media => media.metadata.faces, {onDelete: 'CASCADE', nullable: false}) + media: MediaEntity; + + // @PrimaryColumn('int') + @ManyToOne(type => PersonEntry, person => person.faces, {onDelete: 'CASCADE', nullable: false}) + person: PersonEntry; + + name: string; +} diff --git a/backend/model/sql/enitites/MediaEntity.ts b/backend/model/sql/enitites/MediaEntity.ts index 2a1aa00e..026290d3 100644 --- a/backend/model/sql/enitites/MediaEntity.ts +++ b/backend/model/sql/enitites/MediaEntity.ts @@ -1,8 +1,9 @@ -import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm'; +import {Column, Entity, OneToMany, ManyToOne, PrimaryGeneratedColumn, TableInheritance, Unique, Index} from 'typeorm'; import {DirectoryEntity} from './DirectoryEntity'; import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entities/MediaDTO'; import {OrientationTypes} from 'ts-exif-parser'; import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity'; +import {FaceRegionEntry} from './FaceRegionEntry'; export class MediaDimensionEntity implements MediaDimension { @@ -27,7 +28,6 @@ export class MediaMetadataEntity implements MediaMetadata { @Column('int') fileSize: number; - @Column('simple-array') keywords: string[]; @@ -40,6 +40,9 @@ export class MediaMetadataEntity implements MediaMetadata { @Column('tinyint', {default: OrientationTypes.TOP_LEFT}) orientation: OrientationTypes; + @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media) + faces: FaceRegionEntry[]; + @Column('int') bitRate: number; diff --git a/backend/model/sql/enitites/PersonEntry.ts b/backend/model/sql/enitites/PersonEntry.ts new file mode 100644 index 00000000..d539bd5e --- /dev/null +++ b/backend/model/sql/enitites/PersonEntry.ts @@ -0,0 +1,17 @@ +import {Column, Entity, OneToMany, PrimaryGeneratedColumn, Unique, Index} from 'typeorm'; +import {FaceRegionEntry} from './FaceRegionEntry'; + + +@Entity() +@Unique(['name']) +export class PersonEntry { + @Index() + @PrimaryGeneratedColumn() + id: number; + + @Column() + name: string; + + @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.person) + public faces: FaceRegionEntry[]; +} diff --git a/backend/model/sql/enitites/PhotoEntity.ts b/backend/model/sql/enitites/PhotoEntity.ts index fa9be6f1..4fe49156 100644 --- a/backend/model/sql/enitites/PhotoEntity.ts +++ b/backend/model/sql/enitites/PhotoEntity.ts @@ -1,6 +1,13 @@ import {Column, Entity, ChildEntity, Unique} from 'typeorm'; -import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../common/entities/PhotoDTO'; -import {OrientationTypes} from 'ts-exif-parser'; +import { + CameraMetadata, + FaceRegion, + FaceRegionBox, + GPSMetadata, + PhotoDTO, + PhotoMetadata, + PositionMetaData +} from '../../../../common/entities/PhotoDTO'; import {MediaEntity, MediaMetadataEntity} from './MediaEntity'; export class CameraMetadataEntity implements CameraMetadata { @@ -38,6 +45,7 @@ export class GPSMetadataEntity implements GPSMetadata { altitude: number; } + export class PositionMetaDataEntity implements PositionMetaData { @Column(type => GPSMetadataEntity) @@ -75,5 +83,4 @@ export class PhotoMetadataEntity extends MediaMetadataEntity implements PhotoMet export class PhotoEntity extends MediaEntity implements PhotoDTO { @Column(type => PhotoMetadataEntity) metadata: PhotoMetadataEntity; - } diff --git a/backend/model/threading/MetadataLoader.ts b/backend/model/threading/MetadataLoader.ts index 16e506e0..27123fc8 100644 --- a/backend/model/threading/MetadataLoader.ts +++ b/backend/model/threading/MetadataLoader.ts @@ -1,5 +1,5 @@ import {VideoMetadata} from '../../../common/entities/VideoDTO'; -import {PhotoMetadata} from '../../../common/entities/PhotoDTO'; +import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO'; import {Config} from '../../../common/config/private/Config'; import {Logger} from '../../Logger'; import * as fs from 'fs'; @@ -9,6 +9,14 @@ import {IptcParser} from 'ts-node-iptc'; import {FFmpegFactory} from '../FFmpegFactory'; import {FfprobeData} from 'fluent-ffmpeg'; +// TODO: fix up different metadata loaders +// @ts-ignore +global.DataView = require('jdataview'); +// @ts-ignore +global.DOMParser = require('xmldom').DOMParser; +// @ts-ignore +const ExifReader = require('exifreader'); + const LOG_TAG = '[MetadataLoader]'; const ffmpeg = FFmpegFactory.get(); @@ -157,6 +165,47 @@ export class MetadataLoader { metadata.creationDate = metadata.creationDate || 0; + + try { + const ret = ExifReader.load(data); + const faces: FaceRegion[] = []; + if (ret.Regions && ret.Regions.value.RegionList && ret.Regions.value.RegionList.value) { + for (let i = 0; i < ret.Regions.value.RegionList.value.length; i++) { + if (!ret.Regions.value.RegionList.value[i].value || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'] || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value || + !ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area']) { + continue; + } + const region = ret.Regions.value.RegionList.value[i].value['rdf:Description']; + const regionBox = ret.Regions.value.RegionList.value[i].value['rdf:Description'].value['mwg-rs:Area'].attributes; + if (region.attributes['mwg-rs:Type'] !== 'Face' || + !region.attributes['mwg-rs:Name']) { + continue; + } + const name = region.attributes['mwg-rs:Name']; + const box = { + width: Math.round(regionBox['stArea:w'] * metadata.size.width), + height: Math.round(regionBox['stArea:h'] * metadata.size.height), + x: Math.round(regionBox['stArea:x'] * metadata.size.width), + y: Math.round(regionBox['stArea:y'] * metadata.size.height) + }; + faces.push({name: name, box: box}); + } + } + if (faces.length > 0) { + metadata.faces = faces; // save faces + // remove faces from keywords + metadata.faces.forEach(f => { + const index = metadata.keywords.indexOf(f.name); + if (index !== -1) { + metadata.keywords.splice(index, 1); + } + }); + } + } catch (err) { + } + return resolve(metadata); } catch (err) { return reject({file: fullPath, error: err}); diff --git a/common/DataStructureVersion.ts b/common/DataStructureVersion.ts index 9313616e..5611455a 100644 --- a/common/DataStructureVersion.ts +++ b/common/DataStructureVersion.ts @@ -1 +1 @@ -export const DataStructureVersion = 7; +export const DataStructureVersion = 8; diff --git a/common/entities/PhotoDTO.ts b/common/entities/PhotoDTO.ts index 0eb994b5..23e06a71 100644 --- a/common/entities/PhotoDTO.ts +++ b/common/entities/PhotoDTO.ts @@ -11,6 +11,18 @@ export interface PhotoDTO extends MediaDTO { readyIcon: boolean; } +export interface FaceRegionBox { + width: number; + height: number; + x: number; + y: number; +} + +export interface FaceRegion { + name: string; + box: FaceRegionBox; +} + export interface PhotoMetadata extends MediaMetadata { caption?: string; keywords?: string[]; @@ -20,6 +32,7 @@ export interface PhotoMetadata extends MediaMetadata { size: MediaDimension; creationDate: number; fileSize: number; + faces?: FaceRegion[]; } diff --git a/package.json b/package.json index 269a7cfb..79fa2594 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,11 @@ "cookie-parser": "1.4.3", "cookie-session": "2.0.0-beta.3", "ejs": "2.6.1", + "exifreader": "2.5.0", "express": "4.16.4", "fluent-ffmpeg": "2.1.2", "image-size": "0.6.3", + "jdataview": "2.5.0", "jimp": "0.6.0", "locale": "0.1.0", "reflect-metadata": "0.1.12", @@ -41,7 +43,8 @@ "ts-node-iptc": "1.0.11", "typeconfig": "1.0.7", "typeorm": "0.2.9", - "winston": "2.4.2" + "winston": "2.4.2", + "xmldom": "0.1.27" }, "devDependencies": { "@angular-devkit/build-angular": "0.11.4", diff --git a/test/backend/unit/assets/test image öüóőúéáű-.,.jpg b/test/backend/unit/assets/test image öüóőúéáű-.,.jpg index 4f838c4a..546a986d 100644 Binary files a/test/backend/unit/assets/test image öüóőúéáű-.,.jpg and b/test/backend/unit/assets/test image öüóőúéáű-.,.jpg differ