Implementing on-the-fly GPX compression.

Its a lossy compression that can be finetuned in the config.

#504

src/backend/middlewares/MetaFileMWs.ts

@@ -0,0 +1,40 @@
+import {NextFunction, Request, Response} from 'express';
+import * as fs from 'fs';
+import { Config } from '../../common/config/private/Config';
+import { GPXProcessing } from '../model/GPXProcessing';
+
+export class MetaFileMWs {
+  public static async compressGPX(
+    req: Request,
+    res: Response,
+    next: NextFunction
+  ): Promise<void> {
+    if (!req.resultPipe) {
+      return next();
+    }
+    // if conversion is not enabled redirect, so browser can cache the full
+    if (Config.Client.MetaFile.GPXCompressing.enabled === false) {
+      return res.redirect(req.originalUrl.slice(0, -1 * '\\bestFit'.length));
+    }
+    const fullPath = req.resultPipe as string;
+
+    const compressedGPX = GPXProcessing.generateConvertedPath(
+      fullPath,
+    );
+
+    // check if converted photo exist
+    if (fs.existsSync(compressedGPX) === true) {
+      req.resultPipe = compressedGPX;
+      return next();
+    }
+
+    if (Config.Server.MetaFile.GPXCompressing.onTheFly === true) {
+      req.resultPipe = await GPXProcessing.compressGPX(fullPath);
+      return next();
+    }
+
+    // not converted and won't be now
+    return res.redirect(req.originalUrl.slice(0, -1 * '\\bestFit'.length));
+  }
+}
+

src/backend/model/GPXProcessing.ts

@@ -0,0 +1,127 @@
+import * as path from 'path';
+import {constants as fsConstants, promises as fsp} from 'fs';
+import * as xml2js from 'xml2js';
+import {ProjectPath} from '../ProjectPath';
+import {Config} from '../../common/config/private/Config';
+
+type gpxEntry = { '$': { lat: string, lon: string }, ele: string[], time: string[], extensions: unknown };
+
+export class GPXProcessing {
+
+
+  public static generateConvertedPath(filePath: string): string {
+    const file = path.basename(filePath);
+    return path.join(
+      ProjectPath.TranscodedFolder,
+      ProjectPath.getRelativePathToImages(path.dirname(filePath)),
+      file
+    );
+  }
+
+  public static async isValidConvertedPath(
+    convertedPath: string
+  ): Promise<boolean> {
+    const origFilePath = path.join(
+      ProjectPath.ImageFolder,
+      path.relative(
+        ProjectPath.TranscodedFolder,
+        convertedPath
+      )
+    );
+
+
+    try {
+      await fsp.access(origFilePath, fsConstants.R_OK);
+    } catch (e) {
+      return false;
+    }
+
+    return true;
+  }
+
+
+  static async compressedGPXExist(
+    filePath: string
+  ): Promise<boolean> {
+    // compressed gpx path
+    const outPath = GPXProcessing.generateConvertedPath(filePath);
+
+    // check if file already exist
+    try {
+      await fsp.access(outPath, fsConstants.R_OK);
+      return true;
+    } catch (e) {
+      // ignoring errors
+    }
+    return false;
+  }
+
+  public static async compressGPX(
+    filePath: string,
+  ): Promise<string> {
+    // generate compressed gpx path
+    const outPath = GPXProcessing.generateConvertedPath(filePath);
+
+    // check if file already exist
+    try {
+      await fsp.access(outPath, fsConstants.R_OK);
+      return outPath;
+    } catch (e) {
+      // ignoring errors
+    }
+
+
+    const outDir = path.dirname(outPath);
+
+    await fsp.mkdir(outDir, {recursive: true});
+    const gpxStr = await fsp.readFile(filePath);
+    const gpxObj = await (new xml2js.Parser()).parseStringPromise(gpxStr);
+    const items: gpxEntry[] = gpxObj.gpx.trk[0].trkseg[0].trkpt;
+
+    const distance = (entry1: gpxEntry, entry2: gpxEntry) => {
+      const lat1 = parseFloat(entry1.$.lat);
+      const lon1 = parseFloat(entry1.$.lon);
+      const lat2 = parseFloat(entry2.$.lat);
+      const lon2 = parseFloat(entry2.$.lon);
+
+      // credits to: https://www.movable-type.co.uk/scripts/latlong.html
+      const R = 6371e3; // metres
+      const φ1 = lat1 * Math.PI / 180; // φ, λ in radians
+      const φ2 = lat2 * Math.PI / 180;
+      const Δφ = (lat2 - lat1) * Math.PI / 180;
+      const Δλ = (lon2 - lon1) * Math.PI / 180;
+
+      const a = Math.sin(Δφ / 2) * Math.sin(Δφ / 2) +
+        Math.cos(φ1) * Math.cos(φ2) *
+        Math.sin(Δλ / 2) * Math.sin(Δλ / 2);
+      const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+      const d = R * c; // in metres
+      return d;
+    };
+    const gpxEntryFilter = (value: gpxEntry, i: number, list: gpxEntry[]) => {
+      if (i === 0 || i >= list.length - 1) { // always keep the first and last items
+        return true;
+      }
+      const timeDelta = (Date.parse(list[i].time[0]) - Date.parse(list[i - 1].time[0])); // mill sec.
+      const dist = distance(list[i - 1], list[i]); // meters
+
+      return !(timeDelta < Config.Server.MetaFile.GPXCompressing.minTimeDistance &&
+        dist < Config.Server.MetaFile.GPXCompressing.minDistance);
+    };
+
+    gpxObj.gpx.trk[0].trkseg[0].trkpt = items.filter(gpxEntryFilter).map((v) => {
+      v.$.lon = parseFloat(v.$.lon).toFixed(6);
+      v.$.lat = parseFloat(v.$.lat).toFixed(6);
+      delete v.ele;
+      delete v.extensions;
+      return v;
+    });
+
+    await fsp.writeFile(outPath, (new xml2js.Builder({renderOpts: {pretty: false}})).buildObject(gpxObj));
+
+    return outPath;
+  }
+
+}
+

src/backend/routes/GalleryRouter.ts

@@ -9,6 +9,7 @@ import { VersionMWs } from '../middlewares/VersionMWs';
 import { SupportedFormats } from '../../common/SupportedFormats';
 import { PhotoConverterMWs } from '../middlewares/thumbnail/PhotoConverterMWs';
 import { ServerTimingMWs } from '../middlewares/ServerTimingMWs';
+import {MetaFileMWs} from '../middlewares/MetaFileMWs';
 
 export class GalleryRouter {
   public static route(app: Express): void {
@@ -21,6 +22,7 @@ export class GalleryRouter {
     this.addGetBestFitVideo(app);
     this.addGetVideo(app);
     this.addGetMetaFile(app);
+    this.addGetBestFitMetaFile(app);
     this.addRandom(app);
     this.addDirectoryList(app);
     this.addDirectoryZip(app);
@@ -158,6 +160,26 @@ export class GalleryRouter {
     );
   }
 
+  protected static addGetBestFitMetaFile(app: Express): void {
+    app.get(
+      [
+        '/api/gallery/content/:mediaPath(*.(' +
+        SupportedFormats.MetaFiles.join('|') +
+        '))/bestFit',
+      ],
+      // common part
+      AuthenticationMWs.authenticate,
+      AuthenticationMWs.normalizePathParam('mediaPath'),
+      AuthenticationMWs.authorisePath('mediaPath', false),
+
+      // specific part
+      GalleryMWs.loadFile,
+      MetaFileMWs.compressGPX,
+      ServerTimingMWs.addServerTiming,
+      RenderingMWs.renderFile
+    );
+  }
+
   protected static addRandom(app: Express): void {
     app.get(
       ['/api/gallery/random/:searchQueryDTO'],

src/common/config/private/PrivateConfig.ts

@@ -5,7 +5,7 @@ import {
   JobTrigger,
   JobTriggerType,
 } from '../../entities/job/JobScheduleDTO';
-import { ClientConfig } from '../public/ClientConfig';
+import {ClientConfig, ClientMetaFileConfig} from '../public/ClientConfig';
 import {SubConfigClass} from 'typeconfig/src/decorators/class/SubConfigClass';
 import {ConfigProperty} from 'typeconfig/src/decorators/property/ConfigPropoerty';
 import {DefaultsJobs} from '../../entities/job/JobDTO';
@@ -148,6 +148,25 @@ export class ServerThumbnailConfig {
   personFaceMargin: number = 0.6; // in ration [0-1]
 }
 
+@SubConfigClass()
+export class ServerGPXCompressingConfig {
+  @ConfigProperty({
+    description: 'Compresses gpx files on-the-fly, when they are requested.',
+  })
+  onTheFly: boolean = true;
+  @ConfigProperty({type: 'unsignedInt', description: 'Filters out entry that are closer than this in meters.'})
+  minDistance: number = 5;
+  @ConfigProperty({type: 'unsignedInt', description: 'Filters out entry that are closer than this in time in milliseconds.'})
+  minTimeDistance: number = 5000;
+}
+
+@SubConfigClass()
+export class ServerMetaFileConfig {
+  @ConfigProperty()
+  GPXCompressing: ServerGPXCompressingConfig = new ServerGPXCompressingConfig();
+}
+
+
 @SubConfigClass()
 export class ServerSharingConfig {
   @ConfigProperty()
@@ -163,9 +182,9 @@ export class ServerIndexingConfig {
   @ConfigProperty({
     arrayType: 'string',
     description:
-      "If an entry starts with '/' it is treated as an absolute path." +
-      " If it doesn't start with '/' but contains a '/', the path is relative to the image directory." +
-      " If it doesn't contain a '/', any folder with this name will be excluded.",
+      'If an entry starts with \'/\' it is treated as an absolute path.' +
+      ' If it doesn\'t start with \'/\' but contains a \'/\', the path is relative to the image directory.' +
+      ' If it doesn\'t contain a \'/\', any folder with this name will be excluded.',
   })
   excludeFolderList: string[] = ['.Trash-1000', '.dtrash', '$RECYCLE.BIN'];
   @ConfigProperty({
@@ -182,7 +201,7 @@ export class ServerThreadingConfig {
   enabled: boolean = true;
   @ConfigProperty({
     description:
-      "Number of threads that are used to generate thumbnails. If 0, number of 'CPU cores -1' threads will be used.",
+      'Number of threads that are used to generate thumbnails. If 0, number of \'CPU cores -1\' threads will be used.',
   })
   thumbnailThreads: number = 0; // if zero-> CPU count -1
 }
@@ -482,6 +501,8 @@ export class ServerConfig {
   Log: ServerLogConfig = new ServerLogConfig();
   @ConfigProperty()
   Jobs: ServerJobConfig = new ServerJobConfig();
+  @ConfigProperty()
+  MetaFile: ServerMetaFileConfig = new ServerMetaFileConfig();
 }
 
 export interface IPrivateConfig {

src/common/config/public/ClientConfig.ts

@@ -182,6 +182,12 @@ export class ClientPhotoConfig {
   loadFullImageOnZoom: boolean = true;
 }
 
+@SubConfigClass()
+export class ClientGPXCompressingConfig {
+  @ConfigProperty()
+  enabled: boolean = true;
+}
+
 @SubConfigClass()
 export class ClientMediaConfig {
   @ConfigProperty()
@@ -198,6 +204,11 @@ export class ClientMetaFileConfig {
     description: 'Reads *.gpx files and renders them on the map.',
   })
   gpx: boolean = true;
+  @ConfigProperty({
+    description: 'Reads *.gpx files and renders them on the map.',
+  })
+  @ConfigProperty()
+  GPXCompressing: ClientGPXCompressingConfig = new ClientGPXCompressingConfig();
   @ConfigProperty({
     description:
       'Reads *.md files in a directory and shows the next to the map.',