import {expect} from 'chai'; import * as fs from 'fs'; import {Config} from '../../../../../src/common/config/private/Config'; import {SQLConnection} from '../../../../../src/backend/model/database/sql/SQLConnection'; import {GalleryManager} from '../../../../../src/backend/model/database/sql/GalleryManager'; import {DirectoryDTO} from '../../../../../src/common/entities/DirectoryDTO'; import {TestHelper} from './TestHelper'; import {Connection} from 'typeorm'; import {DirectoryEntity} from '../../../../../src/backend/model/database/sql/enitites/DirectoryEntity'; import {Utils} from '../../../../../src/common/Utils'; import {MediaDTO} from '../../../../../src/common/entities/MediaDTO'; import {FileDTO} from '../../../../../src/common/entities/FileDTO'; import {IndexingManager} from '../../../../../src/backend/model/database/sql/IndexingManager'; import {ObjectManagers} from '../../../../../src/backend/model/ObjectManagers'; import {PersonManager} from '../../../../../src/backend/model/database/sql/PersonManager'; import {SQLTestHelper} from '../../../SQLTestHelper'; import {VersionManager} from '../../../../../src/backend/model/database/sql/VersionManager'; import {DiskMangerWorker} from '../../../../../src/backend/model/threading/DiskMangerWorker'; import {ServerConfig} from '../../../../../src/common/config/private/PrivateConfig'; class GalleryManagerTest extends GalleryManager { public async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise { return super.selectParentDir(connection, directoryName, directoryParent); } public async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise { return super.fillParentDir(connection, dir); } } class IndexingManagerTest extends IndexingManager { public async queueForSave(scannedDirectory: DirectoryDTO): Promise { return super.queueForSave(scannedDirectory); } public async saveToDB(scannedDirectory: DirectoryDTO): Promise { return super.saveToDB(scannedDirectory); } } // to help WebStorm to handle the test cases declare let describe: any; declare const after: any; declare const it: any; describe = SQLTestHelper.describe; describe('IndexingManager', (sqlHelper: SQLTestHelper) => { beforeEach(async () => { await sqlHelper.initDB(); ObjectManagers.getInstance().PersonManager = new PersonManager(); ObjectManagers.getInstance().VersionManager = new VersionManager(); }); after(async () => { await sqlHelper.clearDB(); }); const removeIds = (dir: DirectoryDTO) => { delete dir.id; dir.media.forEach((media: MediaDTO) => { delete media.id; }); if (dir.metaFile) { if (dir.metaFile.length === 0) { delete dir.metaFile; } else { dir.metaFile.forEach((file: FileDTO) => { delete file.id; }); } } if (dir.directories) { dir.directories.forEach((directory: DirectoryDTO) => { removeIds(directory); }); } }; it('should support case sensitive file names', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); p1.name = 'test.jpg'; p2.name = 'Test.jpg'; DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should support case sensitive directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const subDir1 = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); const p1 = TestHelper.getRandomizedPhotoEntry(subDir1, 'subPhoto1', 0); const subDir2 = TestHelper.getRandomizedDirectoryEntry(parent, 'SUBDIR'); const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto1', 0); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir1.isPartial = true; delete subDir1.directories; delete subDir1.metaFile; subDir2.isPartial = true; delete subDir2.directories; delete subDir2.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should support case sensitive directory path', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent1 = TestHelper.getRandomizedDirectoryEntry(null, 'parent'); const parent2 = TestHelper.getRandomizedDirectoryEntry(null, 'PARENT'); const subDir1 = TestHelper.getRandomizedDirectoryEntry(parent1, 'subDir'); const p1 = TestHelper.getRandomizedPhotoEntry(subDir1, 'subPhoto1', 0); const subDir2 = TestHelper.getRandomizedDirectoryEntry(parent2, 'subDir'); const p2 = TestHelper.getRandomizedPhotoEntry(subDir2, 'subPhoto1', 0); DirectoryDTO.removeReferences(parent1); await im.saveToDB(Utils.clone(parent1)); DirectoryDTO.removeReferences(parent2); await im.saveToDB(Utils.clone(parent2)); const conn = await SQLConnection.getConnection(); { const selected = await gm.selectParentDir(conn, parent1.name, parent1.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir1.isPartial = true; delete subDir1.directories; delete subDir1.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent1))); } { const selected = await gm.selectParentDir(conn, parent2.name, parent2.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir2.isPartial = true; delete subDir2.directories; delete subDir2.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent2))); } }); it('should save parent after child', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(null, 'parentDir'); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir'); subDir.path = DiskMangerWorker.pathFromParent(parent); const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); DirectoryDTO.removeReferences(subDir); await im.saveToDB(Utils.clone(subDir)); parent.directories.push(subDir); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir.isPartial = true; delete subDir.directories; delete subDir.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should save root parent after child', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(null, '.'); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const subDir = TestHelper.getRandomizedDirectoryEntry(null, 'subDir'); subDir.path = DiskMangerWorker.pathFromParent(parent); const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); DirectoryDTO.removeReferences(subDir); await im.saveToDB(Utils.clone(subDir)); parent.directories.push(subDir); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir.isPartial = true; delete subDir.directories; delete subDir.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should save parent directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0); const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir.isPartial = true; delete subDir.directories; delete subDir.metaFile; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should save photos with extreme parameters', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); const minFloat = 1.1 * Math.pow(10, -38); const maxFloat = 3.4 * Math.pow(10, +38); p1.metadata.cameraData.fStop = minFloat; p2.metadata.cameraData.fStop = maxFloat; p1.metadata.cameraData.exposure = minFloat; p2.metadata.cameraData.exposure = maxFloat; p1.metadata.cameraData.focalLength = minFloat; p2.metadata.cameraData.focalLength = maxFloat; p1.metadata.positionData.GPSData.altitude = -2147483647; p2.metadata.positionData.GPSData.altitude = 2147483646; p1.metadata.positionData.GPSData.latitude = maxFloat; p2.metadata.positionData.GPSData.latitude = minFloat; p1.metadata.positionData.GPSData.longitude = maxFloat; p2.metadata.positionData.GPSData.longitude = minFloat; DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should skip meta files', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); DirectoryDTO.removeReferences(parent); Config.Client.MetaFile.enabled = true; await im.saveToDB(Utils.clone(parent)); Config.Client.MetaFile.enabled = false; const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); delete parent.metaFile; DirectoryDTO.removeReferences(selected); removeIds(selected); expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); it('should update sub directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); const parent = TestHelper.getRandomizedDirectoryEntry(); parent.name = 'parent'; const p1 = TestHelper.getRandomizedPhotoEntry(parent); const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); subDir.name = 'subDir'; const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1'); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2'); const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3'); DirectoryDTO.removeReferences(subDir); await im.saveToDB(Utils.clone(subDir)); const conn = await SQLConnection.getConnection(); const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); await gm.fillParentDir(conn, selected); // subDir.isPartial = true; // delete subDir.directories; DirectoryDTO.removeReferences(selected); delete subDir.parent; delete subDir.metaFile; removeIds(selected); // selected.directories[0].parent = selected; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(subDir))); }); it('should avoid race condition', async () => { const conn = await SQLConnection.getConnection(); const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); Config.Client.MetaFile.enabled = true; const parent = TestHelper.getRandomizedDirectoryEntry(); const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1'); const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 1); const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 1); DirectoryDTO.removeReferences(parent); const s1 = im.queueForSave(Utils.clone(parent)); const s2 = im.queueForSave(Utils.clone(parent)); const s3 = im.queueForSave(Utils.clone(parent)); await Promise.all([s1, s2, s3]); const selected = await gm.selectParentDir(conn, parent.name, parent.path); await gm.fillParentDir(conn, selected); DirectoryDTO.removeReferences(selected); removeIds(selected); subDir.isPartial = true; delete subDir.directories; delete subDir.metaFile; delete sp1.metadata.faces; delete sp2.metadata.faces; expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); }); (it('should save 1500 photos', async () => { const conn = await SQLConnection.getConnection(); const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); Config.Client.MetaFile.enabled = true; const parent = TestHelper.getRandomizedDirectoryEntry(); DirectoryDTO.removeReferences(parent); await im.saveToDB(Utils.clone(parent)); const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir'); for (let i = 0; i < 1500; i++) { TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i); } DirectoryDTO.removeReferences(parent); await im.saveToDB(subDir); const selected = await gm.selectParentDir(conn, subDir.name, subDir.path); expect(selected.media.length).to.deep.equal(subDir.media.length); }) as any).timeout(40000); SQLTestHelper.savedDescribe('Test listDirectory', () => { const statSync = fs.statSync; let dirTime = 0; const indexedTime = { lastScanned: 0, lastModified: 0 }; beforeEach(() => { dirTime = 0; ObjectManagers.getInstance().IndexingManager = new IndexingManagerTest(); indexedTime.lastModified = 0; indexedTime.lastScanned = 0; }); afterEach(() => { // @ts-ignore fs.statSync = statSync; }); it('with re indexing severity low', async () => { Config.Server.Indexing.reIndexingSensitivity = ServerConfig.ReIndexingSensitivity.low; // @ts-ignore fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); const gm = new GalleryManagerTest(); gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => { return Promise.resolve(indexedTime); }; gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => { return Promise.resolve(); }; ObjectManagers.getInstance().IndexingManager.indexDirectory = (...args) => { return Promise.resolve('indexing'); }; indexedTime.lastScanned = null; expect(await gm.listDirectory('./')).to.be.equal('indexing'); indexedTime.lastModified = 0; dirTime = 1; expect(await gm.listDirectory('./')).to.be.equal('indexing'); indexedTime.lastScanned = 10; indexedTime.lastModified = 1; dirTime = 1; expect(await gm.listDirectory('./')).to.be.equal(indexedTime); expect(await gm.listDirectory('./', 1, 10)) .to.be.equal(null); }); }); });