diff --git a/backend/model/interfaces/IPersonManager.ts b/backend/model/interfaces/IPersonManager.ts index 1afcd91b..f0b0796f 100644 --- a/backend/model/interfaces/IPersonManager.ts +++ b/backend/model/interfaces/IPersonManager.ts @@ -12,8 +12,6 @@ export interface IPersonManager { saveAll(names: string[]): Promise; - keywordsToPerson(media: MediaDTO[]): Promise; - onGalleryIndexUpdate(): Promise; updatePerson(name: string, partialPerson: PersonDTO): Promise; diff --git a/backend/model/memory/PersonManager.ts b/backend/model/memory/PersonManager.ts index 2f93db6b..9a1249ea 100644 --- a/backend/model/memory/PersonManager.ts +++ b/backend/model/memory/PersonManager.ts @@ -1,5 +1,4 @@ import {IPersonManager} from '../interfaces/IPersonManager'; -import {MediaDTO} from '../../../common/entities/MediaDTO'; import {PhotoDTO} from '../../../common/entities/PhotoDTO'; import {PersonDTO} from '../../../common/entities/PersonDTO'; @@ -13,10 +12,6 @@ export class PersonManager implements IPersonManager { throw new Error('Method not implemented.'); } - keywordsToPerson(media: MediaDTO[]): Promise { - throw new Error('Method not implemented.'); - } - get(name: string): Promise { throw new Error('not supported by memory DB'); } diff --git a/backend/model/sql/IndexingManager.ts b/backend/model/sql/IndexingManager.ts index 164aabed..d4a63487 100644 --- a/backend/model/sql/IndexingManager.ts +++ b/backend/model/sql/IndexingManager.ts @@ -15,6 +15,7 @@ import {NotificationManager} from '../NotifocationManager'; import {FaceRegionEntry} from './enitites/FaceRegionEntry'; import {ObjectManagers} from '../ObjectManagers'; import {IIndexingManager} from '../interfaces/IIndexingManager'; +import {DiskMangerWorker} from '../threading/DiskMangerWorker'; const LOG_TAG = '[IndexingManager]'; @@ -43,7 +44,7 @@ export class IndexingManager implements IIndexingManager { }); } - // Todo fix it, once typeorm support connection pools ofr sqlite + // Todo fix it, once typeorm support connection pools for sqlite protected async queueForSave(scannedDirectory: DirectoryDTO) { if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name && dir.path === scannedDirectory.path && @@ -89,30 +90,31 @@ export class IndexingManager implements IIndexingManager { protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) { const directoryRepository = connection.getRepository(DirectoryEntity); - // TODO: fix when first opened directory is not root + + // update subdirectories that does not have a parent + await directoryRepository + .createQueryBuilder() + .update(DirectoryEntity) + .set({parent: currentDirId}) + .where('path = :path', + {path: DiskMangerWorker.pathFromParent(scannedDirectory)}) + .andWhere('name NOT LIKE :root', {root: DiskMangerWorker.dirName('.')}) + .andWhere('parent IS NULL') + .execute(); + // save subdirectories const childDirectories = await directoryRepository.createQueryBuilder('directory') + .leftJoinAndSelect('directory.parent', 'parent') .where('directory.parent = :dir', { dir: currentDirId }).getMany(); for (let i = 0; i < scannedDirectory.directories.length; i++) { // Was this child Dir already indexed before? - let directory: DirectoryEntity = null; - for (let j = 0; j < childDirectories.length; j++) { - if (childDirectories[j].name === scannedDirectory.directories[i].name) { - directory = childDirectories[j]; - childDirectories.splice(j, 1); - break; - } - } + const dirIndex = childDirectories.findIndex(d => d.name === scannedDirectory.directories[i].name); - if (directory != null) { // update existing directory - if (!directory.parent || !directory.parent.id) { // set parent if not set yet - directory.parent = {id: currentDirId}; - delete directory.media; - await directoryRepository.save(directory); - } + if (dirIndex !== -1) { // directory found + childDirectories.splice(dirIndex, 1); } else { // dir does not exists yet scannedDirectory.directories[i].parent = {id: currentDirId}; (scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet @@ -127,7 +129,7 @@ export class IndexingManager implements IIndexingManager { } - protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) { + protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO): Promise { const fileRepository = connection.getRepository(FileEntity); // save files const indexedMetaFiles = await fileRepository.createQueryBuilder('file') @@ -189,7 +191,6 @@ export class IndexingManager implements IIndexingManager { const scannedFaces = (media[i].metadata).faces || []; delete (media[i].metadata).faces; - // let mediaItemId: number = null; if (mediaItem == null) { // not in DB yet media[i].directory = null; mediaItem = Utils.clone(media[i]); diff --git a/backend/model/sql/PersonManager.ts b/backend/model/sql/PersonManager.ts index f80356c5..7a1d9640 100644 --- a/backend/model/sql/PersonManager.ts +++ b/backend/model/sql/PersonManager.ts @@ -1,7 +1,6 @@ import {IPersonManager} from '../interfaces/IPersonManager'; import {SQLConnection} from './SQLConnection'; import {PersonEntry} from './enitites/PersonEntry'; -import {MediaDTO} from '../../../common/entities/MediaDTO'; import {PhotoDTO} from '../../../common/entities/PhotoDTO'; import {MediaEntity} from './enitites/MediaEntity'; import {FaceRegionEntry} from './enitites/FaceRegionEntry'; @@ -70,30 +69,6 @@ export class PersonManager implements IPersonManager { return this.persons; } - // TODO dead code, remove it - async keywordsToPerson(media: MediaDTO[]) { - await this.loadAll(); - const personFilter = (keyword: string) => this.persons.find(p => p.name.toLowerCase() === keyword.toLowerCase()); - (media).forEach(m => { - if (!m.metadata.keywords || m.metadata.keywords.length === 0) { - return; - } - - const personKeywords = m.metadata.keywords.filter(k => personFilter(k)); - if (personKeywords.length === 0) { - return; - } - // remove persons from keywords - m.metadata.keywords = m.metadata.keywords.filter(k => !personFilter(k)); - m.metadata.faces = m.metadata.faces || []; - personKeywords.forEach((pk: string) => { - m.metadata.faces.push({ - name: pk - }); - }); - - }); - } async get(name: string): Promise { diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 1b199d06..ba59c8b4 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -37,15 +37,31 @@ export class DiskMangerWorker { return Math.max(stat.ctime.getTime(), stat.mtime.getTime()); } - public static normalizeDirPath(dirPath: string) { + public static normalizeDirPath(dirPath: string): string { return path.normalize(path.join('.' + path.sep, dirPath)); } + public static pathFromRelativeDirName(relativeDirectoryName: string): string { + return path.join(path.dirname(this.normalizeDirPath(relativeDirectoryName)), path.sep); + } + + + public static pathFromParent(parent: { path: string, name: string }): string { + return path.join(this.normalizeDirPath(path.join(parent.path, parent.name)), path.sep); + } + + public static dirName(name: string) { + if (name.trim().length === 0) { + return '.'; + } + return path.basename(name); + } + public static scanDirectory(relativeDirectoryName: string, maxPhotos: number = null, photosOnly: boolean = false): Promise { return new Promise((resolve, reject) => { relativeDirectoryName = this.normalizeDirPath(relativeDirectoryName); - const directoryName = path.basename(relativeDirectoryName); - const directoryParent = path.join(path.dirname(relativeDirectoryName), path.sep); + const directoryName = DiskMangerWorker.dirName(relativeDirectoryName); + const directoryParent = this.pathFromRelativeDirName(relativeDirectoryName); const absoluteDirectoryName = path.join(ProjectPath.ImageFolder, relativeDirectoryName); const stat = fs.statSync(path.join(ProjectPath.ImageFolder, relativeDirectoryName)); diff --git a/test/backend/unit/model/sql/IndexingManager.ts b/test/backend/unit/model/sql/IndexingManager.ts index 74cefc21..cc54c3ad 100644 --- a/test/backend/unit/model/sql/IndexingManager.ts +++ b/test/backend/unit/model/sql/IndexingManager.ts @@ -16,6 +16,7 @@ import {ObjectManagers} from '../../../../../backend/model/ObjectManagers'; import {PersonManager} from '../../../../../backend/model/sql/PersonManager'; import {SQLTestHelper} from '../../../SQLTestHelper'; import {VersionManager} from '../../../../../backend/model/sql/VersionManager'; +import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMangerWorker'; class GalleryManagerTest extends GalleryManager { @@ -182,6 +183,77 @@ describe('IndexingManager', (sqlHelper: SQLTestHelper) => { }); + it('should save parent after child', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + + const parent = TestHelper.getRandomizedDirectoryEntry(null, 'parentDir'); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + + const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir'); + subDir.path = DiskMangerWorker.pathFromParent(parent); + const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); + const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); + + + DirectoryDTO.removeReferences(subDir); + await im.saveToDB(Utils.clone(subDir)); + + parent.directories.push(subDir); + + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + DirectoryDTO.removeReferences(selected); + removeIds(selected); + subDir.isPartial = true; + delete subDir.directories; + delete subDir.metaFile; + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + + + it('should save root parent after child', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + + const parent = TestHelper.getRandomizedDirectoryEntry(null, '.'); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + + const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir'); + subDir.path = DiskMangerWorker.pathFromParent(parent); + const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); + const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); + + + DirectoryDTO.removeReferences(subDir); + await im.saveToDB(Utils.clone(subDir)); + + parent.directories.push(subDir); + + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + DirectoryDTO.removeReferences(selected); + removeIds(selected); + subDir.isPartial = true; + delete subDir.directories; + delete subDir.metaFile; + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + it('should save parent directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); diff --git a/test/backend/unit/model/sql/PersonManager.ts b/test/backend/unit/model/sql/PersonManager.ts index 4aa4e124..7e55ded5 100644 --- a/test/backend/unit/model/sql/PersonManager.ts +++ b/test/backend/unit/model/sql/PersonManager.ts @@ -11,38 +11,5 @@ declare const it: any; describe('PersonManager', () => { - it('should upgrade keywords to person', async () => { - const pm = new PersonManager(); - pm.loadAll = () => Promise.resolve(); - pm.persons = [{name: 'Han Solo', id: 0, faces: [], count: 0, isFavourite: false}, - {name: 'Anakin', id: 2, faces: [], count: 0, isFavourite: false}]; - - const p_noFaces = { - metadata: { - keywords: ['Han Solo', 'just a keyword'] - } - }; - - const p_wFace = { - metadata: { - keywords: ['Han Solo', 'Anakin'], - faces: [{name: 'Obivan'}] - } - }; - - const cmp = (a: FaceRegion, b: FaceRegion) => { - return a.name.localeCompare(b.name); - }; - - await pm.keywordsToPerson([p_noFaces]); - expect(p_noFaces.metadata.keywords).to.be.deep.equal(['just a keyword']); - expect(p_noFaces.metadata.faces.sort(cmp)).to.eql([{name: 'Han Solo'}].sort(cmp)); - - await pm.keywordsToPerson([p_wFace]); - expect(p_wFace.metadata.keywords).to.be.deep.equal([]); - expect(p_wFace.metadata.faces.sort(cmp)).to.be - .eql([{name: 'Han Solo'}, {name: 'Obivan'}, {name: 'Anakin'}].sort(cmp)); - - }); }); diff --git a/test/backend/unit/model/sql/TestHelper.ts b/test/backend/unit/model/sql/TestHelper.ts index ba3311fa..9d145d8d 100644 --- a/test/backend/unit/model/sql/TestHelper.ts +++ b/test/backend/unit/model/sql/TestHelper.ts @@ -6,7 +6,6 @@ import { PhotoMetadataEntity, PositionMetaDataEntity } from '../../../../../backend/model/sql/enitites/PhotoEntity'; -import * as path from 'path'; import {OrientationTypes} from 'ts-exif-parser'; import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity'; import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity'; @@ -14,6 +13,7 @@ import {MediaDimension} from '../../../../../common/entities/MediaDTO'; import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO'; import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; import {FileDTO} from '../../../../../common/entities/FileDTO'; +import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMangerWorker'; export class TestHelper { @@ -153,8 +153,8 @@ export class TestHelper { const dir: DirectoryDTO = { id: null, - name: forceStr || Math.random().toString(36).substring(7), - path: '.', + name: DiskMangerWorker.dirName(forceStr || Math.random().toString(36).substring(7)), + path: DiskMangerWorker.pathFromParent({path: '', name: '.'}), mediaCount: 0, directories: [], metaFile: [], @@ -164,7 +164,7 @@ export class TestHelper { parent: null }; if (parent !== null) { - dir.path = path.join(parent.path, parent.name); + dir.path = DiskMangerWorker.pathFromParent(parent); parent.directories.push(dir); } return dir;