Merge pull request #462 from bpatrik/feature/improved-dto

10% reduction on server resonse size

benchmark/README.md

@@ -150,6 +150,7 @@ See all benchmark results in [HISTORY.md](HISTORY.md)
 | | Autocomplete | 13.0 ms | items: 24, size: 979.00B |
 | | Add server timing | 0.1 ms | items: 24, size: 979.00B |
 | | Render result | 0.0 ms | items: 24, size: 979.00B |
+
 *Measurements run 20 times and an average was calculated.
 Full runtime: 1522539.0ms
 

src/backend/middlewares/GalleryMWs.ts

@@ -124,6 +124,7 @@ export class GalleryMWs {
 
     const cleanUpMedia = (media: MediaDTO[]): void => {
       media.forEach((m): void => {
+        delete m.id;
         if (MediaDTOUtils.isPhoto(m)) {
           delete (m as VideoDTO).metadata.bitRate;
           delete (m as VideoDTO).metadata.duration;
@@ -131,11 +132,12 @@ export class GalleryMWs {
           delete (m as PhotoDTO).metadata.rating;
           delete (m as PhotoDTO).metadata.caption;
           delete (m as PhotoDTO).metadata.cameraData;
-          delete (m as PhotoDTO).metadata.orientation;
-          delete (m as PhotoDTO).metadata.orientation;
           delete (m as PhotoDTO).metadata.keywords;
           delete (m as PhotoDTO).metadata.positionData;
         }
+        if (m.directory) {
+          delete (m.directory as any).id;
+        }
         Utils.removeNullOrEmptyObj(m);
       });
     };

src/backend/model/database/sql/enitites/MediaEntity.ts

@@ -1,7 +1,6 @@
 import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, TableInheritance, Unique} from 'typeorm';
 import {DirectoryEntity} from './DirectoryEntity';
 import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../../common/entities/MediaDTO';
-import {OrientationTypes} from 'ts-exif-parser';
 import {FaceRegionEntry} from './FaceRegionEntry';
 import {columnCharsetCS} from './EntityUtils';
 import {CameraMetadata, GPSMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
@@ -57,8 +56,6 @@ export class GPSMetadataEntity implements GPSMetadata {
   latitude: number;
   @Column('float', {nullable: true})
   longitude: number;
-  @Column('int', {nullable: true})
-  altitude: number;
 }
 
 
@@ -129,9 +126,6 @@ export class MediaMetadataEntity implements MediaMetadata {
   @Column('tinyint', {unsigned: true})
   rating: 0 | 1 | 2 | 3 | 4 | 5;
 
-  @Column('tinyint', {unsigned: true, default: OrientationTypes.TOP_LEFT})
-  orientation: OrientationTypes;
-
   @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media)
   faces: FaceRegionEntry[];
 

src/backend/model/threading/MetadataLoader.ts

@@ -97,7 +97,6 @@ export class MetadataLoader {
           }
           const metadata: PhotoMetadata = {
             size: {width: 1, height: 1},
-            orientation: OrientationTypes.TOP_LEFT,
             creationDate: 0,
             fileSize: 0
           };
@@ -138,11 +137,11 @@ export class MetadataLoader {
                 }
                 if (Utils.isFloat32(exif.tags.ExposureTime)) {
                   metadata.cameraData = metadata.cameraData || {};
-                  metadata.cameraData.exposure = parseFloat('' + exif.tags.ExposureTime);
+                  metadata.cameraData.exposure = parseFloat(parseFloat('' + exif.tags.ExposureTime).toFixed(4));
                 }
                 if (Utils.isFloat32(exif.tags.FNumber)) {
                   metadata.cameraData = metadata.cameraData || {};
-                  metadata.cameraData.fStop = parseFloat('' + exif.tags.FNumber);
+                  metadata.cameraData.fStop = parseFloat(parseFloat('' + exif.tags.FNumber).toFixed(2));
                 }
               }
               if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
@@ -150,13 +149,10 @@ export class MetadataLoader {
                 metadata.positionData.GPSData = {};
 
                 if (Utils.isFloat32(exif.tags.GPSLongitude)) {
-                  metadata.positionData.GPSData.longitude = exif.tags.GPSLongitude;
+                  metadata.positionData.GPSData.longitude = parseFloat(exif.tags.GPSLongitude.toFixed(6));
                 }
                 if (Utils.isFloat32(exif.tags.GPSLatitude)) {
-                  metadata.positionData.GPSData.latitude = exif.tags.GPSLatitude;
+                  metadata.positionData.GPSData.latitude = parseFloat(exif.tags.GPSLatitude.toFixed(6));
-                }
-                if (Utils.isInt32(exif.tags.GPSAltitude)) {
-                  metadata.positionData.GPSData.altitude = exif.tags.GPSAltitude;
                 }
               }
               if (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) {
@@ -221,7 +217,9 @@ export class MetadataLoader {
               const exif = ExifReader.load(data);
               if (exif.Rating) {
                 metadata.rating = (parseInt(exif.Rating.value, 10) as any);
-                if(metadata.rating < 0) { metadata.rating = 0; }
+                if (metadata.rating < 0) {
+                  metadata.rating = 0;
+                }
               }
               if (exif.subject && exif.subject.value && exif.subject.value.length > 0) {
                 if (metadata.keywords === undefined) {
@@ -234,8 +232,8 @@ export class MetadataLoader {
                 }
               }
               if (exif.Orientation) {
-                metadata.orientation = (parseInt(exif.Orientation.value as any, 10) as any);
+                const orientation = (parseInt(exif.Orientation.value as any, 10) as any);
-                if (OrientationTypes.BOTTOM_LEFT < metadata.orientation) {
+                if (OrientationTypes.BOTTOM_LEFT < orientation) {
                   // noinspection JSSuspiciousNameCombination
                   const height = metadata.size.width;
                   // noinspection JSSuspiciousNameCombination

src/common/DataStructureVersion.ts

@@ -1,4 +1,4 @@
 /**
  * This version indicates that the SQL sql/entities/*Entity.ts files got changed and the db needs to be recreated
  */
-export const DataStructureVersion = 26;
+export const DataStructureVersion = 27;

src/common/entities/MediaDTO.ts

@@ -34,7 +34,6 @@ export const MediaDTOUtils = {
         (media as PhotoDTO).metadata.positionData.state ||
         (media as PhotoDTO).metadata.positionData.country ||
         ((media as PhotoDTO).metadata.positionData.GPSData &&
-          (media as PhotoDTO).metadata.positionData.GPSData.altitude &&
           (media as PhotoDTO).metadata.positionData.GPSData.latitude &&
           (media as PhotoDTO).metadata.positionData.GPSData.longitude));
   },

src/common/entities/PhotoDTO.ts

@@ -36,7 +36,6 @@ export interface PhotoMetadata extends MediaMetadata {
   keywords?: string[];
   cameraData?: CameraMetadata;
   positionData?: PositionMetaData;
-  orientation: OrientationTypes;
   size: MediaDimension;
   creationDate: number;
   fileSize: number;
@@ -52,9 +51,8 @@ export interface PositionMetaData {
 }
 
 export interface GPSMetadata {
-  latitude?: number;
+  latitude?: number; // float with precision: 6
-  longitude?: number;
+  longitude?: number; // float with precision: 6
-  altitude?: number;
 }
 
 
@@ -62,8 +60,8 @@ export interface CameraMetadata {
   ISO?: number;
   model?: string;
   make?: string;
-  fStop?: number;
+  fStop?: number; // float with precision: 2
-  exposure?: number;
+  exposure?: number; // float with precision: 4
   focalLength?: number;
   lens?: string;
 }

src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.html

@@ -128,8 +128,8 @@
       </div>
       <div class="details-sub row" *ngIf="hasGPS()">
         <div class="col-12">
-          {{PositionData.GPSData.latitude.toFixed(3)}},
+          {{PositionData.GPSData.latitude.toFixed(6)}},
-          {{PositionData.GPSData.longitude.toFixed(3)}}
+          {{PositionData.GPSData.longitude.toFixed(6)}}
         </div>
       </div>
     </div>

test/backend/assets/date_issue.json

@@ -1,7 +1,7 @@
 {
   "cameraData": {
     "ISO": 500,
-    "exposure": 0.03333333333333333,
+    "exposure": 0.0333,
     "fStop": 2,
     "focalLength": 2.96,
     "make": "HUAWEI",
@@ -9,7 +9,6 @@
   },
   "creationDate": 1460826466000,
   "fileSize": 1980,
-  "orientation": 1,
   "size": {
     "height": 1,
     "width": 1

test/backend/assets/edge_case_exif_data/before_epoch.json

@@ -27,12 +27,10 @@
     "USA Road trip",
     "Yosemite"
   ],
-  "orientation": 1,
   "positionData": {
     "GPSData": {
-      "altitude": 1960,
+      "latitude": 37.748058,
-      "latitude": 37.74805833333333,
+      "longitude": -119.514023
-      "longitude": -119.51402333333333
     },
     "city": "Yosemite Nemzeti Park",
     "country": "United States",

test/backend/assets/edge_case_exif_data/date_error.json

@@ -9,7 +9,6 @@
   },
   "creationDate": -2211753600000,
   "fileSize": 72850,
-  "orientation": 1,
   "size": {
     "height": 768,
     "width": 1024

test/backend/assets/edge_case_exif_data/iso_error.json

@@ -3,7 +3,6 @@
     "creationDate"
   ],
   "fileSize": 71311,
-  "orientation": 1,
   "size": {
     "height": 1200,
     "width": 1783

test/backend/assets/edge_case_exif_data/no_metadata.json

@@ -3,7 +3,6 @@
     "creationDate"
   ],
   "fileSize": 5860,
-  "orientation": 1,
   "size": {
     "height": 7,
     "width": 10

test/backend/assets/exiftool.json

@@ -25,12 +25,10 @@
     "USA",
     "Yosemite"
   ],
-  "orientation": 1,
   "positionData": {
     "GPSData": {
-      "altitude": 1960,
       "latitude": 37.746765,
-      "longitude": -119.52988333333333
+      "longitude": -119.529883
     },
     "country": "United States",
     "state": "California"

test/backend/assets/old_photo.json

@@ -9,7 +9,6 @@
   },
   "creationDate": 1126455782000,
   "fileSize": 2582,
-  "orientation": 1,
   "size": {
     "height": 5,
     "width": 7

test/backend/assets/orientation/Landscape.json

@@ -1,5 +1,4 @@
 {
-  "orientation": 1,
   "size": {
     "height": 1200,
     "width": 1800

test/backend/assets/orientation/Portrait.json

@@ -1,5 +1,4 @@
 {
-  "orientation": 1,
   "size": {
     "height": 1800,
     "width": 1200

test/backend/assets/orientation/broken_orientation_exif.json

@@ -3,7 +3,6 @@
     "width": 3024,
     "height": 4032
   },
-  "orientation": 6,
   "creationDate": 1518964712000,
   "fileSize": 256001,
   "cameraData": {
@@ -11,14 +10,13 @@
     "make": "Google",
     "ISO": 116,
     "focalLength": 4.442,
-    "exposure": 0.008335,
+    "exposure": 0.0083,
     "fStop": 1.8
   },
   "positionData": {
     "GPSData": {
-      "longitude": -73.96425083333334,
+      "longitude": -73.964251,
-      "latitude": 40.77845111111111,
+      "latitude": 40.778451
-      "altitude": 2.2
     }
   }
 }

test/backend/assets/orientation/broken_orientation_exif2.json

@@ -9,7 +9,6 @@
   },
   "creationDate": 1185452882000,
   "fileSize": 124036,
-  "orientation": 1,
   "size": {
     "height": 3072,
     "width": 2304

test/backend/assets/test image öüóőúéáű-.,.json

@@ -1,7 +1,7 @@
 {
   "cameraData": {
     "ISO": 3200,
-    "exposure": 0.00125,
+    "exposure": 0.0013,
     "fStop": 5.6,
     "focalLength": 85,
     "lens": "EF-S15-85mm f/3.5-5.6 IS USM",
@@ -36,11 +36,9 @@
     "USA",
     "űáéúőóüö ŰÁÉÚŐÓÜÖ"
   ],
-  "orientation": 1,
   "positionData": {
     "GPSData": {
-      "altitude": 90,
+      "latitude": 37.871093,
-      "latitude": 37.871093333333334,
       "longitude": -122.25678
     },
     "city": "test city őúéáűóöí-.,)(=",

test/backend/assets/test_png.json

@@ -21,7 +21,6 @@
   ],
   "fileSize": 4381,
   "keywords": [],
-  "orientation": 1,
   "size": {
     "height": 26,
     "width": 26

test/backend/assets/two_ratings.json

@@ -9,7 +9,6 @@
   },
   "creationDate": 1619181527000,
   "fileSize": 4877,
-  "orientation": 1,
   "rating":3,
   "size": {
     "height": 5,

test/backend/assets/xmp/xmp_subject.json

@@ -1,7 +1,7 @@
 {
   "cameraData": {
     "ISO": 50,
-    "exposure": 0.007751937984496124,
+    "exposure": 0.0078,
     "fStop": 2.4,
     "focalLength": 4.32,
     "make": "samsung",
@@ -9,7 +9,6 @@
   },
   "creationDate": 1614703656000,
   "fileSize": 4709,
-  "orientation": 1,
   "keywords": [
     "Max",
     "Spaß",

test/backend/unit/model/sql/IndexingManager.spec.ts

@@ -419,8 +419,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
     p2.metadata.cameraData.exposure = maxFloat;
     p1.metadata.cameraData.focalLength = minFloat;
     p2.metadata.cameraData.focalLength = maxFloat;
-    p1.metadata.positionData.GPSData.altitude = -2147483647;
-    p2.metadata.positionData.GPSData.altitude = 2147483646;
     p1.metadata.positionData.GPSData.latitude = maxFloat;
     p2.metadata.positionData.GPSData.latitude = minFloat;
     p1.metadata.positionData.GPSData.longitude = maxFloat;

test/backend/unit/model/sql/SearchManager.spec.ts

@@ -1067,8 +1067,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
       ObjectManagers.getInstance().LocationManager.getGPSData = async (): Promise<GPSMetadata> => {
         return {
           longitude: 10,
-          latitude: 10,
+          latitude: 10
-          altitude: 0
         };
       };
 

test/backend/unit/model/sql/TestHelper.ts

@@ -74,7 +74,6 @@ export class TestHelper {
     m.size = sd;
     m.creationDate = Date.now();
     m.fileSize = 123456789;
-    m.orientation = OrientationTypes.TOP_LEFT;
     // m.rating = 0; no rating by default
 
     // TODO: remove when typeorm is fixed
@@ -323,7 +322,6 @@ export class TestHelper {
     };
 
     const gps: GPSMetadata = {
-      altitude: rndInt(1000),
       latitude: rndInt(1000),
       longitude: rndInt(1000)
     };
@@ -349,7 +347,6 @@ export class TestHelper {
       size: sd,
       creationDate: Date.now() + ++TestHelper.creationCounter,
       fileSize: rndInt(10000),
-      orientation: OrientationTypes.TOP_LEFT,
       caption: rndStr(),
       rating: rndInt(5) as any,
     };

test/backend/unit/model/threading/MetaDataLoader.spec.ts

@@ -95,12 +95,12 @@ describe('MetadataLoader', () => {
 
   });
 
+  // TODO: deprecated tests. We do not save orientation anymore.
   describe('should read orientation', () => {
     for (let i = 0; i <= 8; ++i) {
       it('Landscape ' + i, async () => {
         const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/orientation/Landscape_' + i + '.jpg'));
         const expected = require(path.join(__dirname, '/../../../assets/orientation/Landscape.json'));
-        expected.orientation = i;
         delete data.fileSize;
         delete data.creationDate;
         expect(Utils.clone(data)).to.be.deep.equal(expected);
@@ -108,7 +108,6 @@ describe('MetadataLoader', () => {
       it('Portrait ' + i, async () => {
         const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/orientation/Portrait_' + i + '.jpg'));
         const expected = require(path.join(__dirname, '/../../../assets/orientation/Portrait.json'));
-        expected.orientation = i;
         delete data.fileSize;
         delete data.creationDate;
         expect(Utils.clone(data)).to.be.deep.equal(expected);