Multiples small tweaks to save on the server response size:

* Removing ids from DTO as it is not needed on the client side. ~4% saving * Removing orientation information from DB and server response. ~3% saving * removing altitude from GPS data and reducing GPS, exposure and fstop precision ~3% Altogether 10% saving expected. #437
2025-01-14 14:43:17 +08:00 · 2022-03-25 21:51:42 +01:00 · 2022-03-25 21:51:42 +01:00 · 47d864abe5
commit 47d864abe5
parent fa28a6c647
26 changed files with 32 additions and 67 deletions
--- a/src/backend/middlewares/GalleryMWs.ts
+++ b/src/backend/middlewares/GalleryMWs.ts
@ -124,6 +124,7 @@ export class GalleryMWs {

    const cleanUpMedia = (media: MediaDTO[]): void => {
      media.forEach((m): void => {
+        delete m.id;
        if (MediaDTOUtils.isPhoto(m)) {
          delete (m as VideoDTO).metadata.bitRate;
          delete (m as VideoDTO).metadata.duration;
@ -131,11 +132,12 @@ export class GalleryMWs {
          delete (m as PhotoDTO).metadata.rating;
          delete (m as PhotoDTO).metadata.caption;
          delete (m as PhotoDTO).metadata.cameraData;
-          delete (m as PhotoDTO).metadata.orientation;
-          delete (m as PhotoDTO).metadata.orientation;
          delete (m as PhotoDTO).metadata.keywords;
          delete (m as PhotoDTO).metadata.positionData;
        }
+        if (m.directory) {
+          delete (m.directory as any).id;
+        }
        Utils.removeNullOrEmptyObj(m);
      });
    };
--- a/src/backend/model/database/sql/enitites/MediaEntity.ts
+++ b/src/backend/model/database/sql/enitites/MediaEntity.ts
@ -1,7 +1,6 @@
 import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, TableInheritance, Unique} from 'typeorm';
 import {DirectoryEntity} from './DirectoryEntity';
 import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../../common/entities/MediaDTO';
-import {OrientationTypes} from 'ts-exif-parser';
 import {FaceRegionEntry} from './FaceRegionEntry';
 import {columnCharsetCS} from './EntityUtils';
 import {CameraMetadata, GPSMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
@ -57,8 +56,6 @@ export class GPSMetadataEntity implements GPSMetadata {
  latitude: number;
  @Column('float', {nullable: true})
  longitude: number;
-  @Column('int', {nullable: true})
-  altitude: number;
 }


@ -129,9 +126,6 @@ export class MediaMetadataEntity implements MediaMetadata {
  @Column('tinyint', {unsigned: true})
  rating: 0 | 1 | 2 | 3 | 4 | 5;

-  @Column('tinyint', {unsigned: true, default: OrientationTypes.TOP_LEFT})
-  orientation: OrientationTypes;
-
  @OneToMany(type => FaceRegionEntry, faceRegion => faceRegion.media)
  faces: FaceRegionEntry[];

--- a/src/backend/model/threading/MetadataLoader.ts
+++ b/src/backend/model/threading/MetadataLoader.ts
@ -97,7 +97,6 @@ export class MetadataLoader {
          }
          const metadata: PhotoMetadata = {
            size: {width: 1, height: 1},
-            orientation: OrientationTypes.TOP_LEFT,
            creationDate: 0,
            fileSize: 0
          };
@ -138,11 +137,11 @@ export class MetadataLoader {
                }
                if (Utils.isFloat32(exif.tags.ExposureTime)) {
                  metadata.cameraData = metadata.cameraData || {};
-                  metadata.cameraData.exposure = parseFloat('' + exif.tags.ExposureTime);
+                  metadata.cameraData.exposure = parseFloat(parseFloat('' + exif.tags.ExposureTime).toFixed(4));
                }
                if (Utils.isFloat32(exif.tags.FNumber)) {
                  metadata.cameraData = metadata.cameraData || {};
-                  metadata.cameraData.fStop = parseFloat('' + exif.tags.FNumber);
+                  metadata.cameraData.fStop = parseFloat(parseFloat('' + exif.tags.FNumber).toFixed(2));
                }
              }
              if (!isNaN(exif.tags.GPSLatitude) || exif.tags.GPSLongitude || exif.tags.GPSAltitude) {
@ -150,13 +149,10 @@ export class MetadataLoader {
                metadata.positionData.GPSData = {};

                if (Utils.isFloat32(exif.tags.GPSLongitude)) {
-                  metadata.positionData.GPSData.longitude = exif.tags.GPSLongitude;
+                  metadata.positionData.GPSData.longitude = parseFloat(exif.tags.GPSLongitude.toFixed(6));
                }
                if (Utils.isFloat32(exif.tags.GPSLatitude)) {
-                  metadata.positionData.GPSData.latitude = exif.tags.GPSLatitude;
-                }
-                if (Utils.isInt32(exif.tags.GPSAltitude)) {
-                  metadata.positionData.GPSData.altitude = exif.tags.GPSAltitude;
+                  metadata.positionData.GPSData.latitude = parseFloat(exif.tags.GPSLatitude.toFixed(6));
                }
              }
              if (exif.tags.CreateDate || exif.tags.DateTimeOriginal || exif.tags.ModifyDate) {
@ -221,7 +217,9 @@ export class MetadataLoader {
              const exif = ExifReader.load(data);
              if (exif.Rating) {
                metadata.rating = (parseInt(exif.Rating.value, 10) as any);
-                if(metadata.rating < 0) { metadata.rating = 0; }
+                if (metadata.rating < 0) {
+                  metadata.rating = 0;
+                }
              }
              if (exif.subject && exif.subject.value && exif.subject.value.length > 0) {
                if (metadata.keywords === undefined) {
@ -234,8 +232,8 @@ export class MetadataLoader {
                }
              }
              if (exif.Orientation) {
-                metadata.orientation = (parseInt(exif.Orientation.value as any, 10) as any);
-                if (OrientationTypes.BOTTOM_LEFT < metadata.orientation) {
+                const orientation = (parseInt(exif.Orientation.value as any, 10) as any);
+                if (OrientationTypes.BOTTOM_LEFT < orientation) {
                  // noinspection JSSuspiciousNameCombination
                  const height = metadata.size.width;
                  // noinspection JSSuspiciousNameCombination
--- a/src/common/DataStructureVersion.ts
+++ b/src/common/DataStructureVersion.ts
@ -1,4 +1,4 @@
 /**
 * This version indicates that the SQL sql/entities/*Entity.ts files got changed and the db needs to be recreated
 */
-export const DataStructureVersion = 26;
+export const DataStructureVersion = 27;
--- a/src/common/entities/MediaDTO.ts
+++ b/src/common/entities/MediaDTO.ts
@ -34,7 +34,6 @@ export const MediaDTOUtils = {
        (media as PhotoDTO).metadata.positionData.state ||
        (media as PhotoDTO).metadata.positionData.country ||
        ((media as PhotoDTO).metadata.positionData.GPSData &&
-          (media as PhotoDTO).metadata.positionData.GPSData.altitude &&
          (media as PhotoDTO).metadata.positionData.GPSData.latitude &&
          (media as PhotoDTO).metadata.positionData.GPSData.longitude));
  },
--- a/src/common/entities/PhotoDTO.ts
+++ b/src/common/entities/PhotoDTO.ts
@ -36,7 +36,6 @@ export interface PhotoMetadata extends MediaMetadata {
  keywords?: string[];
  cameraData?: CameraMetadata;
  positionData?: PositionMetaData;
-  orientation: OrientationTypes;
  size: MediaDimension;
  creationDate: number;
  fileSize: number;
@ -52,9 +51,8 @@ export interface PositionMetaData {
 }

 export interface GPSMetadata {
-  latitude?: number;
-  longitude?: number;
-  altitude?: number;
+  latitude?: number; // float with precision: 6
+  longitude?: number; // float with precision: 6
 }


@ -62,8 +60,8 @@ export interface CameraMetadata {
  ISO?: number;
  model?: string;
  make?: string;
-  fStop?: number;
-  exposure?: number;
+  fStop?: number; // float with precision: 2
+  exposure?: number; // float with precision: 4
  focalLength?: number;
  lens?: string;
 }
--- a/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.html
+++ b/src/frontend/app/ui/gallery/lightbox/infopanel/info-panel.lightbox.gallery.component.html
@ -128,8 +128,8 @@
      </div>
      <div class="details-sub row" *ngIf="hasGPS()">
        <div class="col-12">
-          {{PositionData.GPSData.latitude.toFixed(3)}},
-          {{PositionData.GPSData.longitude.toFixed(3)}}
+          {{PositionData.GPSData.latitude.toFixed(6)}},
+          {{PositionData.GPSData.longitude.toFixed(6)}}
        </div>
      </div>
    </div>
--- a/test/backend/assets/date_issue.json
+++ b/test/backend/assets/date_issue.json
@ -1,7 +1,7 @@
 {
  "cameraData": {
    "ISO": 500,
-    "exposure": 0.03333333333333333,
+    "exposure": 0.0333,
    "fStop": 2,
    "focalLength": 2.96,
    "make": "HUAWEI",
@ -9,7 +9,6 @@
  },
  "creationDate": 1460826466000,
  "fileSize": 1980,
-  "orientation": 1,
  "size": {
    "height": 1,
    "width": 1
--- a/test/backend/assets/edge_case_exif_data/before_epoch.json
+++ b/test/backend/assets/edge_case_exif_data/before_epoch.json
@ -27,12 +27,10 @@
    "USA Road trip",
    "Yosemite"
  ],
-  "orientation": 1,
  "positionData": {
    "GPSData": {
-      "altitude": 1960,
-      "latitude": 37.74805833333333,
-      "longitude": -119.51402333333333
+      "latitude": 37.748058,
+      "longitude": -119.514023
    },
    "city": "Yosemite Nemzeti Park",
    "country": "United States",
--- a/test/backend/assets/edge_case_exif_data/date_error.json
+++ b/test/backend/assets/edge_case_exif_data/date_error.json
@ -9,7 +9,6 @@
  },
  "creationDate": -2211753600000,
  "fileSize": 72850,
-  "orientation": 1,
  "size": {
    "height": 768,
    "width": 1024
--- a/test/backend/assets/edge_case_exif_data/iso_error.json
+++ b/test/backend/assets/edge_case_exif_data/iso_error.json
@ -3,7 +3,6 @@
    "creationDate"
  ],
  "fileSize": 71311,
-  "orientation": 1,
  "size": {
    "height": 1200,
    "width": 1783
--- a/test/backend/assets/edge_case_exif_data/no_metadata.json
+++ b/test/backend/assets/edge_case_exif_data/no_metadata.json
@ -3,7 +3,6 @@
    "creationDate"
  ],
  "fileSize": 5860,
-  "orientation": 1,
  "size": {
    "height": 7,
    "width": 10
--- a/test/backend/assets/exiftool.json
+++ b/test/backend/assets/exiftool.json
@ -25,12 +25,10 @@
    "USA",
    "Yosemite"
  ],
-  "orientation": 1,
  "positionData": {
    "GPSData": {
-      "altitude": 1960,
      "latitude": 37.746765,
-      "longitude": -119.52988333333333
+      "longitude": -119.529883
    },
    "country": "United States",
    "state": "California"
--- a/test/backend/assets/old_photo.json
+++ b/test/backend/assets/old_photo.json
@ -9,7 +9,6 @@
  },
  "creationDate": 1126455782000,
  "fileSize": 2582,
-  "orientation": 1,
  "size": {
    "height": 5,
    "width": 7
--- a/test/backend/assets/orientation/Landscape.json
+++ b/test/backend/assets/orientation/Landscape.json
@ -1,5 +1,4 @@
 {
-  "orientation": 1,
  "size": {
    "height": 1200,
    "width": 1800
--- a/test/backend/assets/orientation/Portrait.json
+++ b/test/backend/assets/orientation/Portrait.json
@ -1,5 +1,4 @@
 {
-  "orientation": 1,
  "size": {
    "height": 1800,
    "width": 1200
--- a/test/backend/assets/orientation/broken_orientation_exif.json
+++ b/test/backend/assets/orientation/broken_orientation_exif.json
@ -3,7 +3,6 @@
    "width": 3024,
    "height": 4032
  },
-  "orientation": 6,
  "creationDate": 1518964712000,
  "fileSize": 256001,
  "cameraData": {
@ -11,14 +10,13 @@
    "make": "Google",
    "ISO": 116,
    "focalLength": 4.442,
-    "exposure": 0.008335,
+    "exposure": 0.0083,
    "fStop": 1.8
  },
  "positionData": {
    "GPSData": {
-      "longitude": -73.96425083333334,
-      "latitude": 40.77845111111111,
-      "altitude": 2.2
+      "longitude": -73.964251,
+      "latitude": 40.778451
    }
  }
 }
--- a/test/backend/assets/orientation/broken_orientation_exif2.json
+++ b/test/backend/assets/orientation/broken_orientation_exif2.json
@ -9,7 +9,6 @@
  },
  "creationDate": 1185452882000,
  "fileSize": 124036,
-  "orientation": 1,
  "size": {
    "height": 3072,
    "width": 2304
--- a/öüóőúéáű-.,.json
+++ b/öüóőúéáű-.,.json
@ -1,7 +1,7 @@
 {
  "cameraData": {
    "ISO": 3200,
-    "exposure": 0.00125,
+    "exposure": 0.0013,
    "fStop": 5.6,
    "focalLength": 85,
    "lens": "EF-S15-85mm f/3.5-5.6 IS USM",
@ -36,11 +36,9 @@
    "USA",
    "űáéúőóüö ŰÁÉÚŐÓÜÖ"
  ],
-  "orientation": 1,
  "positionData": {
    "GPSData": {
-      "altitude": 90,
-      "latitude": 37.871093333333334,
+      "latitude": 37.871093,
      "longitude": -122.25678
    },
    "city": "test city őúéáűóöí-.,)(=",
--- a/test/backend/assets/test_png.json
+++ b/test/backend/assets/test_png.json
@ -21,7 +21,6 @@
  ],
  "fileSize": 4381,
  "keywords": [],
-  "orientation": 1,
  "size": {
    "height": 26,
    "width": 26
--- a/test/backend/assets/two_ratings.json
+++ b/test/backend/assets/two_ratings.json
@ -9,7 +9,6 @@
  },
  "creationDate": 1619181527000,
  "fileSize": 4877,
-  "orientation": 1,
  "rating":3,
  "size": {
    "height": 5,
--- a/test/backend/assets/xmp/xmp_subject.json
+++ b/test/backend/assets/xmp/xmp_subject.json
@ -1,7 +1,7 @@
 {
  "cameraData": {
    "ISO": 50,
-    "exposure": 0.007751937984496124,
+    "exposure": 0.0078,
    "fStop": 2.4,
    "focalLength": 4.32,
    "make": "samsung",
@ -9,7 +9,6 @@
  },
  "creationDate": 1614703656000,
  "fileSize": 4709,
-  "orientation": 1,
  "keywords": [
    "Max",
    "Spaß",
--- a/test/backend/unit/model/sql/IndexingManager.spec.ts
+++ b/test/backend/unit/model/sql/IndexingManager.spec.ts
@ -419,8 +419,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
    p2.metadata.cameraData.exposure = maxFloat;
    p1.metadata.cameraData.focalLength = minFloat;
    p2.metadata.cameraData.focalLength = maxFloat;
-    p1.metadata.positionData.GPSData.altitude = -2147483647;
-    p2.metadata.positionData.GPSData.altitude = 2147483646;
    p1.metadata.positionData.GPSData.latitude = maxFloat;
    p2.metadata.positionData.GPSData.latitude = minFloat;
    p1.metadata.positionData.GPSData.longitude = maxFloat;
--- a/test/backend/unit/model/sql/SearchManager.spec.ts
+++ b/test/backend/unit/model/sql/SearchManager.spec.ts
@ -1067,8 +1067,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
      ObjectManagers.getInstance().LocationManager.getGPSData = async (): Promise<GPSMetadata> => {
        return {
          longitude: 10,
-          latitude: 10,
-          altitude: 0
+          latitude: 10
        };
      };

--- a/test/backend/unit/model/sql/TestHelper.ts
+++ b/test/backend/unit/model/sql/TestHelper.ts
@ -74,7 +74,6 @@ export class TestHelper {
    m.size = sd;
    m.creationDate = Date.now();
    m.fileSize = 123456789;
-    m.orientation = OrientationTypes.TOP_LEFT;
    // m.rating = 0; no rating by default

    // TODO: remove when typeorm is fixed
@ -323,7 +322,6 @@ export class TestHelper {
    };

    const gps: GPSMetadata = {
-      altitude: rndInt(1000),
      latitude: rndInt(1000),
      longitude: rndInt(1000)
    };
@ -349,7 +347,6 @@ export class TestHelper {
      size: sd,
      creationDate: Date.now() + ++TestHelper.creationCounter,
      fileSize: rndInt(10000),
-      orientation: OrientationTypes.TOP_LEFT,
      caption: rndStr(),
      rating: rndInt(5) as any,
    };
--- a/test/backend/unit/model/threading/MetaDataLoader.spec.ts
+++ b/test/backend/unit/model/threading/MetaDataLoader.spec.ts
@ -95,12 +95,12 @@ describe('MetadataLoader', () => {

  });

+  // TODO: deprecated tests. We do not save orientation anymore.
  describe('should read orientation', () => {
    for (let i = 0; i <= 8; ++i) {
      it('Landscape ' + i, async () => {
        const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/orientation/Landscape_' + i + '.jpg'));
        const expected = require(path.join(__dirname, '/../../../assets/orientation/Landscape.json'));
-        expected.orientation = i;
        delete data.fileSize;
        delete data.creationDate;
        expect(Utils.clone(data)).to.be.deep.equal(expected);
@ -108,7 +108,6 @@ describe('MetadataLoader', () => {
      it('Portrait ' + i, async () => {
        const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../../assets/orientation/Portrait_' + i + '.jpg'));
        const expected = require(path.join(__dirname, '/../../../assets/orientation/Portrait.json'));
-        expected.orientation = i;
        delete data.fileSize;
        delete data.creationDate;
        expect(Utils.clone(data)).to.be.deep.equal(expected);