Caching preview in the DB to speed up directory querying #381

src/backend/model/database/sql/GalleryManager.ts

@@ -9,7 +9,7 @@ import {ProjectPath} from '../../../ProjectPath';
 import {Config} from '../../../../common/config/private/Config';
 import {ISQLGalleryManager} from './IGalleryManager';
 import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
-import {Connection} from 'typeorm';
+import {Brackets, Connection, WhereExpression} from 'typeorm';
 import {MediaEntity} from './enitites/MediaEntity';
 import {VideoEntity} from './enitites/VideoEntity';
 import {DiskMangerWorker} from '../../threading/DiskMangerWorker';
@@ -217,20 +217,66 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
   }
 
   /**
-   * Sets preview for the directory
+   * Sets preview for the directory and caches it in the DB
    */
   public async fillPreviewForSubDir(connection: Connection, dir: SubDirectoryDTO): Promise<void> {
 
-    dir.media = [];
+    if (!dir.preview || !dir.validPreview) {
-    dir.preview = await ObjectManagers.getInstance().PreviewManager.getPreviewForDirectory(dir);
-    dir.isPartial = true;
 
+      dir.preview = await ObjectManagers.getInstance().PreviewManager.getPreviewForDirectory(dir);
+      // write preview back to db
+      await connection.createQueryBuilder()
+        .update(DirectoryEntity).set({preview: dir.preview, validPreview: true}).where('id = :dir', {
+          dir: dir.id
+        }).execute();
+    }
+
+
+    dir.media = [];
+    dir.isPartial = true;
     if (dir.preview) {
       dir.preview.readyThumbnails = [];
       dir.preview.readyIcon = false;
     }
   }
 
+  public async onNewDataVersion(changedDir: ParentDirectoryDTO): Promise<void> {
+    // Invalidating Album preview
+    let fullPath = DiskMangerWorker.normalizeDirPath(path.join(changedDir.path, changedDir.name));
+    const query = (await SQLConnection.getConnection())
+      .createQueryBuilder()
+      .update(DirectoryEntity)
+      .set({validPreview: false});
+
+    let i = 0;
+    const root = DiskMangerWorker.pathFromRelativeDirName('.');
+    while (fullPath !== root) {
+      const name = DiskMangerWorker.dirName(fullPath);
+      const parentPath = DiskMangerWorker.pathFromRelativeDirName(fullPath);
+      fullPath = parentPath;
+      ++i;
+      query.orWhere(new Brackets((q: WhereExpression) => {
+        const param: { [key: string]: string } = {};
+        param['name' + i] = name;
+        param['path' + i] = parentPath;
+        q.where(`path = :path${i}`, param);
+        q.andWhere(`name = :name${i}`, param);
+      }));
+    }
+
+    ++i;
+    query.orWhere(new Brackets((q: WhereExpression) => {
+      const param: { [key: string]: string } = {};
+      param['name' + i] = DiskMangerWorker.dirName('.');
+      param['path' + i] = DiskMangerWorker.pathFromRelativeDirName('.');
+      q.where(`path = :path${i}`, param);
+      q.andWhere(`name = :name${i}`, param);
+    }));
+
+
+    await query.execute();
+  }
+
   protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<ParentDirectoryDTO> {
     const query = connection
       .getRepository(DirectoryEntity)
@@ -240,7 +286,16 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
         path: directoryParent
       })
       .leftJoinAndSelect('directory.directories', 'directories')
-      .leftJoinAndSelect('directory.media', 'media');
+      .leftJoinAndSelect('directory.media', 'media')
+      .leftJoinAndSelect('directories.preview', 'preview')
+      .leftJoinAndSelect('preview.directory', 'previewDirectory')
+      .select(['directory',
+        'directories',
+        'media',
+        'preview.name',
+        'previewDirectory.name',
+        'previewDirectory.path']);
+
 
     // TODO: do better filtering
     // NOTE: it should not cause an issue as it also do not shave to the DB
@@ -253,7 +308,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
     return await query.getOne();
   }
 
-
   protected async fillParentDir(connection: Connection, dir: ParentDirectoryDTO): Promise<void> {
     if (dir.media) {
       const indexedFaces = await connection.getRepository(FaceRegionEntry)

src/backend/model/database/sql/enitites/DirectoryEntity.ts

@@ -56,8 +56,13 @@ export class DirectoryEntity implements ParentDirectoryDTO<MediaDTO>, SubDirecto
   public directories: DirectoryEntity[];
 
   // not saving to database, it is only assigned when querying the DB
+  @ManyToOne(type => MediaEntity, {onDelete: 'SET NULL'})
   public preview: MediaEntity;
 
+  // On galley change, preview will be invalid
+  @Column({default: false})
+  validPreview: boolean;
+
   @OneToMany(type => MediaEntity, media => media.directory)
   public media: MediaEntity[];
 

src/backend/model/threading/DiskMangerWorker.ts

@@ -34,11 +34,11 @@ export class DiskMangerWorker {
     return path.join(this.normalizeDirPath(path.join(parent.path, parent.name)), path.sep);
   }
 
-  public static dirName(name: string): any {
+  public static dirName(dirPath: string): string {
-    if (name.trim().length === 0) {
+    if (dirPath.trim().length === 0) {
       return '.';
     }
-    return path.basename(name);
+    return path.basename(dirPath);
   }
 
   public static async excludeDir(name: string, relativeDirectoryName: string, absoluteDirectoryName: string): Promise<boolean> {
@@ -102,6 +102,7 @@ export class DiskMangerWorker {
       isPartial: false,
       mediaCount: 0,
       preview: null,
+      validPreview: false,
       media: [],
       metaFile: []
     };

src/common/DataStructureVersion.ts

@@ -1,4 +1,4 @@
 /**
  * This version indicates that the SQL sql/entities/*Entity.ts files got changed and the db needs to be recreated
  */
-export const DataStructureVersion = 25;
+export const DataStructureVersion = 26;

src/common/entities/DirectoryDTO.ts

@@ -38,6 +38,7 @@ export interface DirectoryBaseDTO<S extends FileDTO = MediaDTO> extends Director
   media?: S[];
   metaFile?: FileDTO[];
   preview?: PreviewPhotoDTO;
+  validPreview?: boolean; // does not go to the client side
 }
 
 export interface ParentDirectoryDTO<S extends FileDTO = MediaDTO> extends DirectoryBaseDTO<S> {
@@ -64,6 +65,7 @@ export interface SubDirectoryDTO<S extends FileDTO = MediaDTO> extends Directory
   parent: ParentDirectoryDTO<S>;
   mediaCount: number;
   preview: PreviewPhotoDTO;
+  validPreview?: boolean; // does not go to the client side
 }
 
 export const DirectoryDTOUtils = {
@@ -116,6 +118,8 @@ export const DirectoryDTOUtils = {
       });
     }
 
+    delete dir.validPreview; // should not go to the client side;
+
     return dir;
 
   },

test/backend/DBTestHelper.ts

@@ -4,13 +4,16 @@ import * as fs from 'fs';
 import {SQLConnection} from '../../src/backend/model/database/sql/SQLConnection';
 import {DatabaseType} from '../../src/common/config/private/PrivateConfig';
 import {ProjectPath} from '../../src/backend/ProjectPath';
-import {DirectoryBaseDTO, ParentDirectoryDTO} from '../../src/common/entities/DirectoryDTO';
+import {DirectoryBaseDTO, ParentDirectoryDTO, SubDirectoryDTO} from '../../src/common/entities/DirectoryDTO';
 import {ObjectManagers} from '../../src/backend/model/ObjectManagers';
 import {DiskMangerWorker} from '../../src/backend/model/threading/DiskMangerWorker';
 import {IndexingManager} from '../../src/backend/model/database/sql/IndexingManager';
 import {GalleryManager} from '../../src/backend/model/database/sql/GalleryManager';
 import {Connection} from 'typeorm';
 import {Utils} from '../../src/common/Utils';
+import {TestHelper} from './unit/model/sql/TestHelper';
+import {VideoDTO} from '../../src/common/entities/VideoDTO';
+import {PhotoDTO} from '../../src/common/entities/PhotoDTO';
 
 declare let describe: any;
 const savedDescribe = describe;
@@ -42,6 +45,36 @@ export class DBTestHelper {
   };
   public static readonly savedDescribe = savedDescribe;
   tempDir: string;
+  public readonly testGalleyEntities: {
+    dir: ParentDirectoryDTO,
+    subDir: SubDirectoryDTO,
+    subDir2: SubDirectoryDTO,
+    v: VideoDTO,
+    p: PhotoDTO,
+    p2: PhotoDTO,
+    p3: PhotoDTO,
+    p4: PhotoDTO
+  } = {
+    /**
+     * dir
+     * |- v
+     * |- p
+     * |- p2
+     * |-> subDir
+     *     |- p3
+     * |-> subDir2
+     *     |- p4
+     */
+
+    dir: null,
+    subDir: null,
+    subDir2: null,
+    v: null,
+    p: null,
+    p2: null,
+    p3: null,
+    p4: null
+  };
 
   constructor(public dbType: DatabaseType) {
     this.tempDir = path.join(__dirname, './tmp');
@@ -127,7 +160,6 @@ export class DBTestHelper {
     }
   }
 
-
   public async clearDB(): Promise<void> {
     if (this.dbType === DatabaseType.sqlite) {
       await this.clearUpSQLite();
@@ -138,6 +170,26 @@ export class DBTestHelper {
     }
   }
 
+  public async setUpTestGallery(): Promise<void> {
+    const directory: ParentDirectoryDTO = TestHelper.getDirectoryEntry();
+    this.testGalleyEntities.subDir = TestHelper.getDirectoryEntry(directory, 'The Phantom Menace');
+    this.testGalleyEntities.subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi');
+    this.testGalleyEntities.p = TestHelper.getRandomizedPhotoEntry(directory, 'Photo1');
+    this.testGalleyEntities.p2 = TestHelper.getRandomizedPhotoEntry(directory, 'Photo2');
+    this.testGalleyEntities.p3 = TestHelper.getRandomizedPhotoEntry(this.testGalleyEntities.subDir, 'Photo3');
+    this.testGalleyEntities.p4 = TestHelper.getRandomizedPhotoEntry(this.testGalleyEntities.subDir2, 'Photo4');
+    this.testGalleyEntities.v = TestHelper.getVideoEntry1(directory);
+
+    this.testGalleyEntities.dir = await DBTestHelper.persistTestDir(directory);
+    this.testGalleyEntities.subDir = this.testGalleyEntities.dir.directories[0];
+    this.testGalleyEntities.subDir2 = this.testGalleyEntities.dir.directories[1];
+    this.testGalleyEntities.p = (this.testGalleyEntities.dir.media.filter(m => m.name === this.testGalleyEntities.p.name)[0] as any);
+    this.testGalleyEntities.p2 = (this.testGalleyEntities.dir.media.filter(m => m.name === this.testGalleyEntities.p2.name)[0] as any);
+    this.testGalleyEntities.v = (this.testGalleyEntities.dir.media.filter(m => m.name === this.testGalleyEntities.v.name)[0] as any);
+    this.testGalleyEntities.p3 = (this.testGalleyEntities.dir.directories[0].media[0] as any);
+    this.testGalleyEntities.p4 = (this.testGalleyEntities.dir.directories[1].media[0] as any);
+  }
+
   private async initMySQL(): Promise<void> {
     await this.resetMySQL();
   }

test/backend/unit/model/sql/AlbumManager.spec.ts

@@ -29,50 +29,12 @@ describe = DBTestHelper.describe(); // fake it os IDE plays nicely (recognize th
 
 describe('AlbumManager', (sqlHelper: DBTestHelper) => {
   describe = tmpDescribe;
-  /**
-   * dir
-   * |- v
-   * |- p
-   * |- p2
-   * |-> subDir
-   *     |- p3
-   * |-> subDir2
-   *     |- p4
-   */
-
-  let dir: ParentDirectoryDTO;
-  let subDir: SubDirectoryDTO;
-  let subDir2: SubDirectoryDTO;
-  let v: VideoDTO;
-  let p: PhotoDTO;
-  let p2: PhotoDTO;
-  let p3: PhotoDTO;
-  let p4: PhotoDTO;
 
 
-  const setUpTestGallery = async (): Promise<void> => {
-    const directory: ParentDirectoryDTO = TestHelper.getDirectoryEntry();
-    subDir = TestHelper.getDirectoryEntry(directory, 'The Phantom Menace');
-    subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi');
-    p = TestHelper.getRandomizedPhotoEntry(directory, 'Photo1');
-    p2 = TestHelper.getRandomizedPhotoEntry(directory, 'Photo2');
-    p3 = TestHelper.getRandomizedPhotoEntry(subDir, 'Photo3');
-    p4 = TestHelper.getRandomizedPhotoEntry(subDir2, 'Photo4');
-    v = TestHelper.getVideoEntry1(directory);
-
-    dir = await DBTestHelper.persistTestDir(directory);
-    subDir = dir.directories[0];
-    subDir2 = dir.directories[1];
-    p = (dir.media.filter(m => m.name === p.name)[0] as any);
-    p2 = (dir.media.filter(m => m.name === p2.name)[0] as any);
-    v = (dir.media.filter(m => m.name === v.name)[0] as any);
-    p3 = (dir.directories[0].media[0] as any);
-    p4 = (dir.directories[1].media[0] as any);
-  };
 
   const setUpSqlDB = async () => {
     await sqlHelper.initDB();
-    await setUpTestGallery();
+    await sqlHelper.setUpTestGallery();
     await ObjectManagers.InitSQLManagers();
   };
 
@@ -192,7 +154,7 @@ describe('AlbumManager', (sqlHelper: DBTestHelper) => {
       searchQuery: query,
       locked: false,
       count: 1,
-      preview: toAlbumPreview(p)
+      preview: toAlbumPreview(sqlHelper.testGalleyEntities.p)
     } as SavedSearchDTO]));
 
 

test/backend/unit/model/sql/GalleryManager.spec.ts

@@ -1,13 +1,88 @@
 import {DBTestHelper} from '../../../DBTestHelper';
 import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager';
+import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers';
+import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLConnection';
+import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity';
+import {ParentDirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO';
+import {Connection} from 'typeorm';
 
+const deepEqualInAnyOrder = require('deep-equal-in-any-order');
+const chai = require('chai');
+
+chai.use(deepEqualInAnyOrder);
+const {expect} = chai;
 
 // to help WebStorm to handle the test cases
 declare let describe: any;
+declare const before: any;
 declare const after: any;
+const tmpDescribe = describe;
 describe = DBTestHelper.describe();
 
-describe('GalleryManager', (sqlHelper: DBTestHelper) => {
 
+class GalleryManagerTest extends GalleryManager {
+
+
+  public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<ParentDirectoryDTO> {
+    return super.selectParentDir(connection, directoryName, directoryParent);
+  }
+
+  public async fillParentDir(connection: Connection, dir: ParentDirectoryDTO): Promise<void> {
+    return super.fillParentDir(connection, dir);
+  }
+
+}
+
+describe('GalleryManager', (sqlHelper: DBTestHelper) => {
+  describe = tmpDescribe;
+
+
+  const setUpSqlDB = async () => {
+    await sqlHelper.initDB();
+    await sqlHelper.setUpTestGallery();
+    await ObjectManagers.InitSQLManagers();
+  };
+
+  before(setUpSqlDB);
+  after(sqlHelper.clearDB);
+
+  it('should invalidate and update preview', async () => {
+    const gm = new GalleryManagerTest();
+    const conn = await SQLConnection.getConnection();
+
+    const selectDir = async () => {
+      return await conn.getRepository(DirectoryEntity).findOne({id: sqlHelper.testGalleyEntities.subDir.id}, {
+        join: {
+          alias: 'dir',
+          leftJoinAndSelect: {preview: 'dir.preview'}
+        }
+      });
+    };
+
+
+    let subdir = await selectDir();
+
+    expect(subdir.validPreview).to.equal(true);
+    expect(subdir.preview.id).to.equal(1);
+
+    // new version should invalidate
+    await gm.onNewDataVersion(sqlHelper.testGalleyEntities.subDir as ParentDirectoryDTO);
+    subdir = await selectDir();
+    expect(subdir.validPreview).to.equal(false);
+    // during invalidation, we do not remove the previous preview (it's good to show at least some photo)
+    expect(subdir.preview.id).to.equal(1);
+
+    await conn.createQueryBuilder()
+      .update(DirectoryEntity)
+      .set({validPreview: false, preview: null}).execute();
+    expect((await selectDir()).preview).to.equal(null);
+
+    const res = await gm.selectParentDir(conn, sqlHelper.testGalleyEntities.dir.name, sqlHelper.testGalleyEntities.dir.path);
+    await gm.fillParentDir(conn, res);
+    subdir = await selectDir();
+    expect(subdir.validPreview).to.equal(true);
+    expect(subdir.preview.id).to.equal(1);
+
+  });
 
 });

test/backend/unit/model/sql/PreviewManager.spec.ts

@@ -134,6 +134,7 @@ describe('PreviewManager', (sqlHelper: DBTestHelper) => {
     delete tmpDir.directories;
     delete tmpDir.media;
     delete tmpDir.preview;
+    delete tmpDir.validPreview;
     delete tmpDir.metaFile;
     const ret = Utils.clone(m);
     delete (ret.directory as DirectoryBaseDTO).id;

test/backend/unit/model/sql/SearchManager.spec.ts

@@ -206,6 +206,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
     delete tmpDir.directories;
     delete tmpDir.media;
     delete tmpDir.preview;
+    delete tmpDir.validPreview;
     delete tmpDir.metaFile;
     const ret = Utils.clone(m);
     delete (ret.directory as DirectoryBaseDTO).lastScanned;
@@ -230,6 +231,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
     delete d.directories;
     delete d.media;
     delete d.preview;
+    delete d.validPreview;
     delete d.metaFile;
     const ret = Utils.clone(d);
     d.directories = tmpD;

test/backend/unit/model/sql/TestHelper.ts

@@ -260,6 +260,7 @@ export class TestHelper {
       directories: [],
       metaFile: [],
       preview: null,
+      validPreview: false,
       media: [],
       lastModified: Date.now(),
       lastScanned: null,