fixing issue when indexing sub directory before parent directory

backend/model/interfaces/IPersonManager.ts

@@ -12,8 +12,6 @@ export interface IPersonManager {
 
   saveAll(names: string[]): Promise<void>;
 
-  keywordsToPerson(media: MediaDTO[]): Promise<void>;
-
   onGalleryIndexUpdate(): Promise<void>;
 
   updatePerson(name: string, partialPerson: PersonDTO): Promise<PersonEntry>;

backend/model/memory/PersonManager.ts

@@ -1,5 +1,4 @@
 import {IPersonManager} from '../interfaces/IPersonManager';
-import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {PersonDTO} from '../../../common/entities/PersonDTO';
 
@@ -13,10 +12,6 @@ export class PersonManager implements IPersonManager {
     throw new Error('Method not implemented.');
   }
 
-  keywordsToPerson(media: MediaDTO[]): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
-
   get(name: string): Promise<any> {
     throw new Error('not supported by memory DB');
   }

backend/model/sql/IndexingManager.ts

@@ -15,6 +15,7 @@ import {NotificationManager} from '../NotifocationManager';
 import {FaceRegionEntry} from './enitites/FaceRegionEntry';
 import {ObjectManagers} from '../ObjectManagers';
 import {IIndexingManager} from '../interfaces/IIndexingManager';
+import {DiskMangerWorker} from '../threading/DiskMangerWorker';
 
 const LOG_TAG = '[IndexingManager]';
 
@@ -43,7 +44,7 @@ export class IndexingManager implements IIndexingManager {
     });
   }
 
-  // Todo fix it, once typeorm support connection pools ofr sqlite
+  // Todo fix it, once typeorm support connection pools for sqlite
   protected async queueForSave(scannedDirectory: DirectoryDTO) {
     if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
       dir.path === scannedDirectory.path &&
@@ -89,30 +90,31 @@ export class IndexingManager implements IIndexingManager {
 
   protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) {
     const directoryRepository = connection.getRepository(DirectoryEntity);
-    // TODO: fix when first opened directory is not root
+
+    // update subdirectories that does not have a parent
+    await directoryRepository
+      .createQueryBuilder()
+      .update(DirectoryEntity)
+      .set({parent: <any>currentDirId})
+      .where('path = :path',
+        {path: DiskMangerWorker.pathFromParent(scannedDirectory)})
+      .andWhere('name NOT LIKE :root', {root: DiskMangerWorker.dirName('.')})
+      .andWhere('parent IS NULL')
+      .execute();
+
     // save subdirectories
     const childDirectories = await directoryRepository.createQueryBuilder('directory')
+      .leftJoinAndSelect('directory.parent', 'parent')
       .where('directory.parent = :dir', {
         dir: currentDirId
       }).getMany();
 
     for (let i = 0; i < scannedDirectory.directories.length; i++) {
       // Was this child Dir already indexed before?
-      let directory: DirectoryEntity = null;
+      const dirIndex = childDirectories.findIndex(d => d.name === scannedDirectory.directories[i].name);
-      for (let j = 0; j < childDirectories.length; j++) {
-        if (childDirectories[j].name === scannedDirectory.directories[i].name) {
-          directory = childDirectories[j];
-          childDirectories.splice(j, 1);
-          break;
-        }
-      }
 
-      if (directory != null) { // update existing directory
+      if (dirIndex !== -1) { // directory found
-        if (!directory.parent || !directory.parent.id) { // set parent if not set yet
+        childDirectories.splice(dirIndex, 1);
-          directory.parent = <any>{id: currentDirId};
-          delete directory.media;
-          await directoryRepository.save(directory);
-        }
       } else { // dir does not exists yet
         scannedDirectory.directories[i].parent = <any>{id: currentDirId};
         (<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
@@ -127,7 +129,7 @@ export class IndexingManager implements IIndexingManager {
 
   }
 
-  protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) {
+  protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO): Promise<void> {
     const fileRepository = connection.getRepository(FileEntity);
     // save files
     const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
@@ -189,7 +191,6 @@ export class IndexingManager implements IIndexingManager {
       const scannedFaces = (<PhotoMetadata>media[i].metadata).faces || [];
       delete (<PhotoMetadata>media[i].metadata).faces;
 
-      // let mediaItemId: number = null;
       if (mediaItem == null) { // not in DB yet
         media[i].directory = null;
         mediaItem = <any>Utils.clone(media[i]);

backend/model/sql/PersonManager.ts

@@ -1,7 +1,6 @@
 import {IPersonManager} from '../interfaces/IPersonManager';
 import {SQLConnection} from './SQLConnection';
 import {PersonEntry} from './enitites/PersonEntry';
-import {MediaDTO} from '../../../common/entities/MediaDTO';
 import {PhotoDTO} from '../../../common/entities/PhotoDTO';
 import {MediaEntity} from './enitites/MediaEntity';
 import {FaceRegionEntry} from './enitites/FaceRegionEntry';
@@ -70,30 +69,6 @@ export class PersonManager implements IPersonManager {
     return this.persons;
   }
 
-  // TODO dead code, remove it
-  async keywordsToPerson(media: MediaDTO[]) {
-    await this.loadAll();
-    const personFilter = (keyword: string) => this.persons.find(p => p.name.toLowerCase() === keyword.toLowerCase());
-    (<PhotoDTO[]>media).forEach(m => {
-      if (!m.metadata.keywords || m.metadata.keywords.length === 0) {
-        return;
-      }
-
-      const personKeywords = m.metadata.keywords.filter(k => personFilter(k));
-      if (personKeywords.length === 0) {
-        return;
-      }
-      // remove persons from keywords
-      m.metadata.keywords = m.metadata.keywords.filter(k => !personFilter(k));
-      m.metadata.faces = m.metadata.faces || [];
-      personKeywords.forEach((pk: string) => {
-        m.metadata.faces.push({
-          name: pk
-        });
-      });
-
-    });
-  }
 
   async get(name: string): Promise<PersonEntry> {
 

backend/model/threading/DiskMangerWorker.ts

@@ -37,15 +37,31 @@ export class DiskMangerWorker {
     return Math.max(stat.ctime.getTime(), stat.mtime.getTime());
   }
 
-  public static normalizeDirPath(dirPath: string) {
+  public static normalizeDirPath(dirPath: string): string {
     return path.normalize(path.join('.' + path.sep, dirPath));
   }
 
+  public static pathFromRelativeDirName(relativeDirectoryName: string): string {
+    return path.join(path.dirname(this.normalizeDirPath(relativeDirectoryName)), path.sep);
+  }
+
+
+  public static pathFromParent(parent: { path: string, name: string }): string {
+    return path.join(this.normalizeDirPath(path.join(parent.path, parent.name)), path.sep);
+  }
+
+  public static dirName(name: string) {
+    if (name.trim().length === 0) {
+      return '.';
+    }
+    return path.basename(name);
+  }
+
   public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise<DirectoryDTO> {
     return new Promise<DirectoryDTO>((resolve, reject) => {
       relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName);
-      const directoryName = path.basename(relativeDirectoryName);
+      const directoryName = DiskMangerWorker.dirName(relativeDirectoryName);
-      const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep);
+      const directoryParent = this.pathFromRelativeDirName(relativeDirectoryName);
       const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName);
 
       const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName));

test/backend/unit/model/sql/IndexingManager.ts

@@ -16,6 +16,7 @@ import {ObjectManagers} from '../../../../../backend/model/ObjectManagers';
 import {PersonManager} from '../../../../../backend/model/sql/PersonManager';
 import {SQLTestHelper} from '../../../SQLTestHelper';
 import {VersionManager} from '../../../../../backend/model/sql/VersionManager';
+import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMangerWorker';
 
 class GalleryManagerTest extends GalleryManager {
 
@@ -182,6 +183,77 @@ describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
   });
 
 
+  it('should save parent after child', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+
+    const parent = TestHelper.getRandomizedDirectoryEntry(null, 'parentDir');
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+
+    const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir');
+    subDir.path = DiskMangerWorker.pathFromParent(parent);
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
+
+
+    DirectoryDTO.removeReferences(subDir);
+    await im.saveToDB(Utils.clone(subDir));
+
+    parent.directories.push(subDir);
+
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    subDir.isPartial = true;
+    delete subDir.directories;
+    delete subDir.metaFile;
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
+
+  it('should save root parent after child', async () => {
+    const gm = new GalleryManagerTest();
+    const im = new IndexingManagerTest();
+
+    const parent = TestHelper.getRandomizedDirectoryEntry(null, '.');
+    const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
+
+    const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir');
+    subDir.path = DiskMangerWorker.pathFromParent(parent);
+    const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
+    const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
+
+
+    DirectoryDTO.removeReferences(subDir);
+    await im.saveToDB(Utils.clone(subDir));
+
+    parent.directories.push(subDir);
+
+
+    DirectoryDTO.removeReferences(parent);
+    await im.saveToDB(Utils.clone(parent));
+
+    const conn = await SQLConnection.getConnection();
+    const selected = await gm.selectParentDir(conn, parent.name, parent.path);
+    await gm.fillParentDir(conn, selected);
+
+    DirectoryDTO.removeReferences(selected);
+    removeIds(selected);
+    subDir.isPartial = true;
+    delete subDir.directories;
+    delete subDir.metaFile;
+    expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
+      .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
+  });
+
   it('should save parent directory', async () => {
     const gm = new GalleryManagerTest();
     const im = new IndexingManagerTest();

test/backend/unit/model/sql/PersonManager.ts

@@ -11,38 +11,5 @@ declare const it: any;
 
 describe('PersonManager', () => {
 
-  it('should upgrade keywords to person', async () => {
-    const pm = new PersonManager();
-    pm.loadAll = () => Promise.resolve();
-    pm.persons = [{name: 'Han Solo', id: 0, faces: [], count: 0, isFavourite: false},
-      {name: 'Anakin', id: 2, faces: [], count: 0, isFavourite: false}];
-
-    const p_noFaces = <PhotoDTO>{
-      metadata: {
-        keywords: ['Han Solo', 'just a keyword']
-      }
-    };
-
-    const p_wFace = <PhotoDTO>{
-      metadata: {
-        keywords: ['Han Solo', 'Anakin'],
-        faces: [{name: 'Obivan'}]
-      }
-    };
-
-    const cmp = (a: FaceRegion, b: FaceRegion) => {
-      return a.name.localeCompare(b.name);
-    };
-
-    await pm.keywordsToPerson([p_noFaces]);
-    expect(p_noFaces.metadata.keywords).to.be.deep.equal(['just a keyword']);
-    expect(p_noFaces.metadata.faces.sort(cmp)).to.eql([{name: 'Han Solo'}].sort(cmp));
-
-    await pm.keywordsToPerson([p_wFace]);
-    expect(p_wFace.metadata.keywords).to.be.deep.equal([]);
-    expect(p_wFace.metadata.faces.sort(cmp)).to.be
-      .eql([{name: 'Han Solo'}, {name: 'Obivan'}, {name: 'Anakin'}].sort(cmp));
-
-  });
 
 });

test/backend/unit/model/sql/TestHelper.ts

@@ -6,7 +6,6 @@ import {
   PhotoMetadataEntity,
   PositionMetaDataEntity
 } from '../../../../../backend/model/sql/enitites/PhotoEntity';
-import * as path from 'path';
 import {OrientationTypes} from 'ts-exif-parser';
 import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
 import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
@@ -14,6 +13,7 @@ import {MediaDimension} from '../../../../../common/entities/MediaDTO';
 import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
 import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
 import {FileDTO} from '../../../../../common/entities/FileDTO';
+import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMangerWorker';
 
 export class TestHelper {
 
@@ -153,8 +153,8 @@ export class TestHelper {
 
     const dir: DirectoryDTO = {
       id: null,
-      name: forceStr || Math.random().toString(36).substring(7),
+      name: DiskMangerWorker.dirName(forceStr || Math.random().toString(36).substring(7)),
-      path: '.',
+      path: DiskMangerWorker.pathFromParent({path: '', name: '.'}),
       mediaCount: 0,
       directories: [],
       metaFile: [],
@@ -164,7 +164,7 @@ export class TestHelper {
       parent: null
     };
     if (parent !== null) {
-      dir.path = path.join(parent.path, parent.name);
+      dir.path = DiskMangerWorker.pathFromParent(parent);
       parent.directories.push(dir);
     }
     return dir;