xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00
Code Actions Projects Activity

improving sql, fixing tests

Browse Source
This commit is contained in:
Patrik J. Braun 2019-01-13 17:38:39 +01:00
parent 22aecea263
commit 550d8d4f5f
23 changed files with 761 additions and 548 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
backend/middlewares/AdminMWs.ts
View File

@ -399,7 +399,7 @@ export class AdminMWs {
public static startIndexing(req: Request, res: Response, next: NextFunction) {
try {
const createThumbnails: boolean = (<IndexingDTO>req.body).createThumbnails || false;
ObjectManagerRepository.getInstance().IndexingManager.startIndexing(createThumbnails);
ObjectManagerRepository.getInstance().IndexingTaskManager.startIndexing(createThumbnails);
req.resultPipe = 'ok';
return next();
} catch (err) {
@ -413,7 +413,7 @@ export class AdminMWs {
public static getIndexingProgress(req: Request, res: Response, next: NextFunction) {
try {
req.resultPipe = ObjectManagerRepository.getInstance().IndexingManager.getProgress();
req.resultPipe = ObjectManagerRepository.getInstance().IndexingTaskManager.getProgress();
return next();
} catch (err) {
if (err instanceof Error) {
@ -425,7 +425,7 @@ export class AdminMWs {
public static cancelIndexing(req: Request, res: Response, next: NextFunction) {
try {
ObjectManagerRepository.getInstance().IndexingManager.cancelIndexing();
ObjectManagerRepository.getInstance().IndexingTaskManager.cancelIndexing();
req.resultPipe = 'ok';
return next();
} catch (err) {
@ -438,7 +438,7 @@ export class AdminMWs {
public static async resetIndexes(req: Express.Request, res: Response, next: NextFunction) {
try {
await ObjectManagerRepository.getInstance().IndexingManager.reset();
await ObjectManagerRepository.getInstance().IndexingTaskManager.reset();
req.resultPipe = 'ok';
return next();
} catch (err) {

45
backend/model/ObjectManagerRepository.ts
View File

@ -4,7 +4,9 @@ import {ISearchManager} from './interfaces/ISearchManager';
import {SQLConnection} from './sql/SQLConnection';
import {ISharingManager} from './interfaces/ISharingManager';
import {Logger} from '../Logger';
import {IIndexingTaskManager} from './interfaces/IIndexingTaskManager';
import {IIndexingManager} from './interfaces/IIndexingManager';
import {IPersonManager} from './interfaces/IPersonManager';
export class ObjectManagerRepository {
@ -15,6 +17,16 @@ export class ObjectManagerRepository {
private _searchManager: ISearchManager;
private _sharingManager: ISharingManager;
private _indexingManager: IIndexingManager;
private _indexingTaskManager: IIndexingTaskManager;
private _personManager: IPersonManager;
get PersonManager(): IPersonManager {
return this._personManager;
}
set PersonManager(value: IPersonManager) {
this._personManager = value;
}
get IndexingManager(): IIndexingManager {
return this._indexingManager;
@ -24,19 +36,14 @@ export class ObjectManagerRepository {
this._indexingManager = value;
}
public static getInstance() {
if (this._instance === null) {
this._instance = new ObjectManagerRepository();
}
return this._instance;
get IndexingTaskManager(): IIndexingTaskManager {
return this._indexingTaskManager;
}
public static async reset() {
await SQLConnection.close();
this._instance = null;
set IndexingTaskManager(value: IIndexingTaskManager) {
this._indexingTaskManager = value;
}
get GalleryManager(): IGalleryManager {
return this._galleryManager;
}
@ -69,18 +76,34 @@ export class ObjectManagerRepository {
this._sharingManager = value;
}
public static getInstance() {
if (this._instance === null) {
this._instance = new ObjectManagerRepository();
}
return this._instance;
}
public static async reset() {
await SQLConnection.close();
this._instance = null;
}
public static async InitMemoryManagers() {
await ObjectManagerRepository.reset();
const GalleryManager = require('./memory/GalleryManager').GalleryManager;
const UserManager = require('./memory/UserManager').UserManager;
const SearchManager = require('./memory/SearchManager').SearchManager;
const SharingManager = require('./memory/SharingManager').SharingManager;
const IndexingTaskManager = require('./memory/IndexingTaskManager').IndexingTaskManager;
const IndexingManager = require('./memory/IndexingManager').IndexingManager;
const PersonManager = require('./memory/PersonManager').PersonManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
}
public static async InitSQLManagers() {
@ -90,12 +113,16 @@ export class ObjectManagerRepository {
const UserManager = require('./sql/UserManager').UserManager;
const SearchManager = require('./sql/SearchManager').SearchManager;
const SharingManager = require('./sql/SharingManager').SharingManager;
const IndexingTaskManager = require('./sql/IndexingManager').IndexingTaskManager;
const IndexingManager = require('./sql/IndexingManager').IndexingManager;
const PersonManager = require('./sql/PersonManager').PersonManager;
ObjectManagerRepository.getInstance().GalleryManager = new GalleryManager();
ObjectManagerRepository.getInstance().UserManager = new UserManager();
ObjectManagerRepository.getInstance().SearchManager = new SearchManager();
ObjectManagerRepository.getInstance().SharingManager = new SharingManager();
ObjectManagerRepository.getInstance().IndexingTaskManager = new IndexingTaskManager();
ObjectManagerRepository.getInstance().IndexingManager = new IndexingManager();
ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
Logger.debug('SQL DB inited');
}

10
backend/model/interfaces/IIndexingManager.ts
View File

@ -1,11 +1,5 @@
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
export interface IIndexingManager {
startIndexing(createThumbnails?: boolean): void;
getProgress(): IndexingProgressDTO;
cancelIndexing(): void;
reset(): Promise<void>;
indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
}

11
backend/model/interfaces/IIndexingTaskManager.ts Normal file
View File

@ -0,0 +1,11 @@
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
export interface IIndexingTaskManager {
startIndexing(createThumbnails?: boolean): void;
getProgress(): IndexingProgressDTO;
cancelIndexing(): void;
reset(): Promise<void>;
}

7
backend/model/interfaces/IPersonManager.ts Normal file
View File

@ -0,0 +1,7 @@
import {PersonEntry} from '../sql/enitites/PersonEntry';
export interface IPersonManager {
get(name: string): Promise<PersonEntry>;
saveAll(names: string[]): Promise<void>;
}

14
backend/model/memory/IndexingManager.ts
View File

@ -1,21 +1,11 @@
import {IIndexingManager} from '../interfaces/IIndexingManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
export class IndexingManager implements IIndexingManager {
startIndexing(): void {
indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
throw new Error('not supported by memory DB');
}
getProgress(): IndexingProgressDTO {
throw new Error('not supported by memory DB');
}
cancelIndexing(): void {
throw new Error('not supported by memory DB');
}
reset(): Promise<void> {
throw new Error('Method not implemented.');
}
}

21
backend/model/memory/IndexingTaskManager.ts Normal file
View File

@ -0,0 +1,21 @@
import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
export class IndexingTaskManager implements IIndexingTaskManager {
startIndexing(): void {
throw new Error('not supported by memory DB');
}
getProgress(): IndexingProgressDTO {
throw new Error('not supported by memory DB');
}
cancelIndexing(): void {
throw new Error('not supported by memory DB');
}
reset(): Promise<void> {
throw new Error('Method not implemented.');
}
}

11
backend/model/memory/PersonManager.ts Normal file
View File

@ -0,0 +1,11 @@
import {IPersonManager} from '../interfaces/IPersonManager';
export class IndexingTaskManager implements IPersonManager {
get(name: string): Promise<any> {
throw new Error('not supported by memory DB');
}
saveAll(names: string[]): Promise<void> {
throw new Error('not supported by memory DB');
}
}

398
backend/model/sql/GalleryManager.ts
View File

@ -4,94 +4,25 @@ import * as path from 'path';
import * as fs from 'fs';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {SQLConnection} from './SQLConnection';
import {DiskManager} from '../DiskManger';
import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
import {Utils} from '../../../common/Utils';
import {PhotoEntity} from './enitites/PhotoEntity';
import {ProjectPath} from '../../ProjectPath';
import {Config} from '../../../common/config/private/Config';
import {ISQLGalleryManager} from './IGalleryManager';
import {DatabaseType, ReIndexingSensitivity} from '../../../common/config/private/IPrivateConfig';
import {FaceRegion, PhotoDTO, PhotoMetadata} from '../../../common/entities/PhotoDTO';
import {PhotoDTO} from '../../../common/entities/PhotoDTO';
import {OrientationType} from '../../../common/entities/RandomQueryDTO';
import {Brackets, Connection, Transaction, TransactionRepository, Repository} from 'typeorm';
import {Brackets, Connection} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {VideoEntity} from './enitites/VideoEntity';
import {FileEntity} from './enitites/FileEntity';
import {FileDTO} from '../../../common/entities/FileDTO';
import {NotificationManager} from '../NotifocationManager';
import {DiskMangerWorker} from '../threading/DiskMangerWorker';
import {Logger} from '../../Logger';
import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import {PersonEntry} from './enitites/PersonEntry';
import {ObjectManagerRepository} from '../ObjectManagerRepository';
const LOG_TAG = '[GalleryManager]';
export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
private savingQueue: DirectoryDTO[] = [];
private isSaving = false;
protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
const query = connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.where('directory.name = :name AND directory.path = :path', {
name: directoryName,
path: directoryParent
})
.leftJoinAndSelect('directory.directories', 'directories')
.leftJoinAndSelect('directory.media', 'media');
if (Config.Client.MetaFile.enabled === true) {
query.leftJoinAndSelect('directory.metaFile', 'metaFile');
}
return await query.getOne();
}
protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
if (dir.media) {
const indexedFaces = await connection.getRepository(FaceRegionEntry)
.createQueryBuilder('face')
.leftJoinAndSelect('face.media', 'media')
.where('media.directory = :directory', {
directory: dir.id
})
.leftJoinAndSelect('face.person', 'person')
.getMany();
for (let i = 0; i < dir.media.length; i++) {
dir.media[i].directory = dir;
dir.media[i].readyThumbnails = [];
dir.media[i].readyIcon = false;
(<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
.filter(fe => fe.media.id === dir.media[i].id)
.map(f => ({box: f.box, name: f.person.name}));
}
}
if (dir.directories) {
for (let i = 0; i < dir.directories.length; i++) {
dir.directories[i].media = await connection
.getRepository(MediaEntity)
.createQueryBuilder('media')
.where('media.directory = :dir', {
dir: dir.directories[i].id
})
.orderBy('media.metadata.creationDate', 'ASC')
.limit(Config.Server.indexing.folderPreviewSize)
.getMany();
dir.directories[i].isPartial = true;
for (let j = 0; j < dir.directories[i].media.length; j++) {
dir.directories[i].media[j].directory = dir.directories[i];
dir.directories[i].media[j].readyThumbnails = [];
dir.directories[i].media[j].readyIcon = false;
}
}
}
}
public async listDirectory(relativeDirectoryName: string,
knownLastModified?: number,
@ -124,7 +55,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
if (dir.lastModified !== lastModified) {
Logger.silly(LOG_TAG, 'Reindexing reason: lastModified mismatch: known: '
+ dir.lastModified + ', current:' + lastModified);
return this.indexDirectory(relativeDirectoryName);
return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
}
@ -136,7 +67,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
Logger.silly(LOG_TAG, 'lazy reindexing reason: cache timeout: lastScanned: '
+ (Date.now() - dir.lastScanned) + ', cachedFolderTimeout:' + Config.Server.indexing.cachedFolderTimeout);
this.indexDirectory(relativeDirectoryName).catch((err) => {
ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName).catch((err) => {
console.error(err);
});
}
@ -146,33 +77,11 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
// never scanned (deep indexed), do it and return with it
Logger.silly(LOG_TAG, 'Reindexing reason: never scanned');
return this.indexDirectory(relativeDirectoryName);
return ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(relativeDirectoryName);
}
public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
return new Promise(async (resolve, reject) => {
try {
const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
// returning with the result
scannedDirectory.media.forEach(p => p.readyThumbnails = []);
resolve(scannedDirectory);
this.queueForSave(scannedDirectory).catch(console.error);
} catch (error) {
NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
console.error(error);
return reject(error);
}
});
}
public async getRandomPhoto(queryFilter: RandomQuery): Promise<PhotoDTO> {
const connection = await SQLConnection.getConnection();
const photosRepository = connection.getRepository(PhotoEntity);
@ -230,236 +139,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
}
// Todo fix it, once typeorm support connection pools ofr sqlite
protected async queueForSave(scannedDirectory: DirectoryDTO) {
if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
dir.path === scannedDirectory.path &&
dir.lastModified === scannedDirectory.lastModified &&
dir.lastScanned === scannedDirectory.lastScanned &&
(dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
(dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
return;
}
this.savingQueue.push(scannedDirectory);
while (this.isSaving === false && this.savingQueue.length > 0) {
await this.saveToDB(this.savingQueue[0]);
this.savingQueue.shift();
}
}
protected async saveToDB(scannedDirectory: DirectoryDTO) {
console.log('saving');
this.isSaving = true;
try {
const connection = await SQLConnection.getConnection();
// saving to db
const directoryRepository = connection.getRepository(DirectoryEntity);
const mediaRepository = connection.getRepository(MediaEntity);
const fileRepository = connection.getRepository(FileEntity);
let currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
.where('directory.name = :name AND directory.path = :path', {
name: scannedDirectory.name,
path: scannedDirectory.path
}).getOne();
if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
currentDir.lastModified = scannedDirectory.lastModified;
currentDir.lastScanned = scannedDirectory.lastScanned;
currentDir.mediaCount = scannedDirectory.mediaCount;
currentDir = await directoryRepository.save(currentDir);
} else {
currentDir = await directoryRepository.save(<DirectoryEntity>scannedDirectory);
}
// TODO: fix when first opened directory is not root
// save subdirectories
const childDirectories = await directoryRepository.createQueryBuilder('directory')
.where('directory.parent = :dir', {
dir: currentDir.id
}).getMany();
for (let i = 0; i < scannedDirectory.directories.length; i++) {
// Was this child Dir already indexed before?
let directory: DirectoryEntity = null;
for (let j = 0; j < childDirectories.length; j++) {
if (childDirectories[j].name === scannedDirectory.directories[i].name) {
directory = childDirectories[j];
childDirectories.splice(j, 1);
break;
}
}
if (directory != null) { // update existing directory
if (!directory.parent || !directory.parent.id) { // set parent if not set yet
directory.parent = currentDir;
delete directory.media;
await directoryRepository.save(directory);
}
} else { // dir does not exists yet
scannedDirectory.directories[i].parent = currentDir;
(<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
const d = await directoryRepository.save(<DirectoryEntity>scannedDirectory.directories[i]);
for (let j = 0; j < scannedDirectory.directories[i].media.length; j++) {
scannedDirectory.directories[i].media[j].directory = d;
}
await this.saveMedia(connection, scannedDirectory.directories[i].media);
}
}
// Remove child Dirs that are not anymore in the parent dir
await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
// save media
const indexedMedia = (await mediaRepository.createQueryBuilder('media')
.where('media.directory = :dir', {
dir: currentDir.id
})
.getMany());
const mediaToSave = [];
for (let i = 0; i < scannedDirectory.media.length; i++) {
let media: MediaDTO = null;
for (let j = 0; j < indexedMedia.length; j++) {
if (indexedMedia[j].name === scannedDirectory.media[i].name) {
media = indexedMedia[j];
indexedMedia.splice(j, 1);
break;
}
}
if (media == null) { // not in DB yet
scannedDirectory.media[i].directory = null;
media = Utils.clone(scannedDirectory.media[i]);
scannedDirectory.media[i].directory = scannedDirectory;
media.directory = currentDir;
mediaToSave.push(media);
} else {
delete (<PhotoMetadata>media.metadata).faces;
if (!Utils.equalsFilter(media.metadata, scannedDirectory.media[i].metadata)) {
media.metadata = scannedDirectory.media[i].metadata;
mediaToSave.push(media);
}
}
const scannedFaces = (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
delete (<PhotoMetadata>scannedDirectory.media[i].metadata).faces;
const mediaEntry = await this.saveAMedia(connection, media);
await this.saveFaces(connection, mediaEntry, scannedFaces);
}
// await this.saveMedia(connection, mediaToSave);
await mediaRepository.remove(indexedMedia);
// save files
const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
.where('file.directory = :dir', {
dir: currentDir.id
}).getMany();
const metaFilesToSave = [];
for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
let metaFile: FileDTO = null;
for (let j = 0; j < indexedMetaFiles.length; j++) {
if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
metaFile = indexedMetaFiles[j];
indexedMetaFiles.splice(j, 1);
break;
}
}
if (metaFile == null) { // not in DB yet
scannedDirectory.metaFile[i].directory = null;
metaFile = Utils.clone(scannedDirectory.metaFile[i]);
scannedDirectory.metaFile[i].directory = scannedDirectory;
metaFile.directory = currentDir;
metaFilesToSave.push(metaFile);
}
}
await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
} catch (e) {
throw e;
} finally {
this.isSaving = false;
}
}
protected async saveFaces(connection: Connection, media: MediaEntity, scannedFaces: FaceRegion[]) {
const faceRepository = connection.getRepository(FaceRegionEntry);
const personRepository = connection.getRepository(PersonEntry);
const indexedPersons = await personRepository.createQueryBuilder('person').getMany();
const indexedFaces = await faceRepository.createQueryBuilder('face')
.where('face.media = :media', {
media: media.id
})
.leftJoinAndSelect('face.person', 'person')
.getMany();
const getPerson = async (name: string) => {
let person = indexedPersons.find(p => p.name === name);
if (!person) {
person = <any>await personRepository.save({name: name});
indexedPersons.push(person);
}
return person;
};
const faceToSave = [];
for (let i = 0; i < scannedFaces.length; i++) {
let face: FaceRegionEntry = null;
for (let j = 0; j < indexedFaces.length; j++) {
if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
indexedFaces[j].box.width === scannedFaces[i].box.width &&
indexedFaces[j].box.x === scannedFaces[i].box.x &&
indexedFaces[j].box.y === scannedFaces[i].box.y &&
indexedFaces[j].person.name === scannedFaces[i].name) {
face = indexedFaces[j];
indexedFaces.splice(j, 1);
break;
}
}
if (face == null) {
(<FaceRegionEntry>scannedFaces[i]).person = await getPerson(scannedFaces[i].name);
(<FaceRegionEntry>scannedFaces[i]).media = media;
// console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person, (<FaceRegionEntry>scannedFaces[i]).media);
// console.log('inserting', (<FaceRegionEntry>scannedFaces[i]).person.id, (<FaceRegionEntry>scannedFaces[i]).media.id);
faceToSave.push(scannedFaces[i]);
}
}
await faceRepository.save(faceToSave, {chunk: Math.max(Math.ceil(faceToSave.length / 500), 1)});
await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
}
protected async saveAMedia(connection: Connection, media: MediaDTO): Promise<MediaEntity> {
if (MediaDTO.isPhoto(media)) {
return await <any>connection.getRepository(PhotoEntity).save(media);
}
return await <any>connection.getRepository(VideoEntity).save(media);
}
protected async saveMedia(connection: Connection, mediaList: MediaDTO[]): Promise<MediaEntity[]> {
const chunked = Utils.chunkArrays(mediaList, 100);
let list: MediaEntity[] = [];
for (let i = 0; i < chunked.length; i++) {
list = list.concat(await connection.getRepository(PhotoEntity).save(<PhotoEntity[]>chunked[i].filter(m => MediaDTO.isPhoto(m))));
list = list.concat(await connection.getRepository(VideoEntity).save(<VideoEntity[]>chunked[i].filter(m => MediaDTO.isVideo(m))));
}
return list;
}
async countDirectories(): Promise<number> {
const connection = await SQLConnection.getConnection();
return await connection.getRepository(DirectoryEntity)
@ -490,5 +169,68 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
.getCount();
}
protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
const query = connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.where('directory.name = :name AND directory.path = :path', {
name: directoryName,
path: directoryParent
})
.leftJoinAndSelect('directory.directories', 'directories')
.leftJoinAndSelect('directory.media', 'media');
if (Config.Client.MetaFile.enabled === true) {
query.leftJoinAndSelect('directory.metaFile', 'metaFile');
}
return await query.getOne();
}
protected async fillParentDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
if (dir.media) {
const indexedFaces = await connection.getRepository(FaceRegionEntry)
.createQueryBuilder('face')
.leftJoinAndSelect('face.media', 'media')
.where('media.directory = :directory', {
directory: dir.id
})
.leftJoinAndSelect('face.person', 'person')
.select(['face.id', 'face.box.x',
'face.box.y', 'face.box.width', 'face.box.height',
'media.id', 'person.name', 'person.id'])
.getMany();
for (let i = 0; i < dir.media.length; i++) {
dir.media[i].directory = dir;
dir.media[i].readyThumbnails = [];
dir.media[i].readyIcon = false;
(<PhotoDTO>dir.media[i]).metadata.faces = indexedFaces
.filter(fe => fe.media.id === dir.media[i].id)
.map(f => ({box: f.box, name: f.person.name}));
}
}
if (dir.directories) {
for (let i = 0; i < dir.directories.length; i++) {
dir.directories[i].media = await connection
.getRepository(MediaEntity)
.createQueryBuilder('media')
.where('media.directory = :dir', {
dir: dir.directories[i].id
})
.orderBy('media.metadata.creationDate', 'ASC')
.limit(Config.Server.indexing.folderPreviewSize)
.getMany();
dir.directories[i].isPartial = true;
for (let j = 0; j < dir.directories[i].media.length; j++) {
dir.directories[i].media[j].directory = dir.directories[i];
dir.directories[i].media[j].readyThumbnails = [];
dir.directories[i].media[j].readyIcon = false;
}
}
}
}
}

2
backend/model/sql/IGalleryManager.ts
View File

@ -6,8 +6,6 @@ export interface ISQLGalleryManager extends IGalleryManager {
knownLastModified?: number,
knownLastScanned?: number): Promise<DirectoryDTO>;
indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO>;
countDirectories(): Promise<number>;
countPhotos(): Promise<number>;

404
backend/model/sql/IndexingManager.ts
View File

@ -1,119 +1,325 @@
import {IIndexingManager} from '../interfaces/IIndexingManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {ObjectManagerRepository} from '../ObjectManagerRepository';
import {ISQLGalleryManager} from './IGalleryManager';
import * as path from 'path';
import * as fs from 'fs';
import {SQLConnection} from './SQLConnection';
import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {Logger} from '../../Logger';
import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
import {Config} from '../../../common/config/private/Config';
import {SQLConnection} from './SQLConnection';
import {DiskManager} from '../DiskManger';
import {PhotoEntity} from './enitites/PhotoEntity';
import {Utils} from '../../../common/Utils';
import {FaceRegion, PhotoMetadata} from '../../../common/entities/PhotoDTO';
import {Connection, Repository} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {ProjectPath} from '../../ProjectPath';
import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
import {VideoEntity} from './enitites/VideoEntity';
import {FileEntity} from './enitites/FileEntity';
import {FileDTO} from '../../../common/entities/FileDTO';
import {NotificationManager} from '../NotifocationManager';
import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import {ObjectManagerRepository} from '../ObjectManagerRepository';
const LOG_TAG = '[IndexingManager]';
export class IndexingManager implements IIndexingManager {
directoriesToIndex: string[] = [];
indexingProgress: IndexingProgressDTO = null;
enabled = false;
private indexNewDirectory = async (createThumbnails: boolean = false) => {
if (this.directoriesToIndex.length === 0) {
this.indexingProgress = null;
if (global.gc) {
global.gc();
}
return;
}
const directory = this.directoriesToIndex.shift();
this.indexingProgress.current = directory;
this.indexingProgress.left = this.directoriesToIndex.length;
const scanned = await (<ISQLGalleryManager>ObjectManagerRepository.getInstance().GalleryManager).indexDirectory(directory);
if (this.enabled === false) {
return;
}
this.indexingProgress.indexed++;
this.indexingProgress.time.current = Date.now();
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
}
if (createThumbnails) {
for (let i = 0; i < scanned.media.length; i++) {
try {
const media = scanned.media[i];
const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
const thPath = path.join(ProjectPath.ThumbnailFolder,
ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
if (fs.existsSync(thPath)) { // skip existing thumbnails
continue;
}
await ThumbnailWorker.render(<RendererInput>{
type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
mediaPath: mPath,
size: Config.Client.Thumbnail.thumbnailSizes[0],
thPath: thPath,
makeSquare: false,
qualityPriority: Config.Server.thumbnail.qualityPriority
}, Config.Server.thumbnail.processingLibrary);
} catch (e) {
console.error(e);
Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
}
export class IndexingManager {
private savingQueue: DirectoryDTO[] = [];
private isSaving = false;
public indexDirectory(relativeDirectoryName: string): Promise<DirectoryDTO> {
return new Promise(async (resolve, reject) => {
try {
const scannedDirectory = await DiskManager.scanDirectory(relativeDirectoryName);
// returning with the result
scannedDirectory.media.forEach(p => p.readyThumbnails = []);
resolve(scannedDirectory);
this.queueForSave(scannedDirectory).catch(console.error);
} catch (error) {
NotificationManager.warning('Unknown indexing error for: ' + relativeDirectoryName, error.toString());
console.error(error);
return reject(error);
}
}
process.nextTick(() => {
this.indexNewDirectory(createThumbnails);
});
};
}
// Todo fix it, once typeorm support connection pools ofr sqlite
protected async queueForSave(scannedDirectory: DirectoryDTO) {
if (this.savingQueue.findIndex(dir => dir.name === scannedDirectory.name &&
dir.path === scannedDirectory.path &&
dir.lastModified === scannedDirectory.lastModified &&
dir.lastScanned === scannedDirectory.lastScanned &&
(dir.media || dir.media.length) === (scannedDirectory.media || scannedDirectory.media.length) &&
(dir.metaFile || dir.metaFile.length) === (scannedDirectory.metaFile || scannedDirectory.metaFile.length)) !== -1) {
return;
}
this.savingQueue.push(scannedDirectory);
while (this.isSaving === false && this.savingQueue.length > 0) {
await this.saveToDB(this.savingQueue[0]);
this.savingQueue.shift();
}
}
protected async saveParentDir(connection: Connection, scannedDirectory: DirectoryDTO): Promise<number> {
const directoryRepository = connection.getRepository(DirectoryEntity);
const currentDir: DirectoryEntity = await directoryRepository.createQueryBuilder('directory')
.where('directory.name = :name AND directory.path = :path', {
name: scannedDirectory.name,
path: scannedDirectory.path
}).getOne();
if (!!currentDir) {// Updated parent dir (if it was in the DB previously)
currentDir.lastModified = scannedDirectory.lastModified;
currentDir.lastScanned = scannedDirectory.lastScanned;
currentDir.mediaCount = scannedDirectory.mediaCount;
await directoryRepository.save(currentDir);
return currentDir.id;
startIndexing(createThumbnails: boolean = false): void {
if (this.directoriesToIndex.length === 0 && this.enabled === false) {
Logger.info(LOG_TAG, 'Starting indexing');
this.indexingProgress = {
indexed: 0,
left: 0,
current: '',
time: {
start: Date.now(),
current: Date.now()
}
};
this.directoriesToIndex.push('/');
this.enabled = true;
this.indexNewDirectory(createThumbnails);
} else {
Logger.info(LOG_TAG, 'Already indexing..');
return (await directoryRepository.insert(<DirectoryEntity>{
mediaCount: scannedDirectory.mediaCount,
lastModified: scannedDirectory.lastModified,
lastScanned: scannedDirectory.lastScanned,
name: scannedDirectory.name,
path: scannedDirectory.path
})).identifiers[0].id;
}
}
getProgress(): IndexingProgressDTO {
return this.indexingProgress;
protected async saveChildDirs(connection: Connection, currentDirId: number, scannedDirectory: DirectoryDTO) {
const directoryRepository = connection.getRepository(DirectoryEntity);
// TODO: fix when first opened directory is not root
// save subdirectories
const childDirectories = await directoryRepository.createQueryBuilder('directory')
.where('directory.parent = :dir', {
dir: currentDirId
}).getMany();
for (let i = 0; i < scannedDirectory.directories.length; i++) {
// Was this child Dir already indexed before?
let directory: DirectoryEntity = null;
for (let j = 0; j < childDirectories.length; j++) {
if (childDirectories[j].name === scannedDirectory.directories[i].name) {
directory = childDirectories[j];
childDirectories.splice(j, 1);
break;
}
}
if (directory != null) { // update existing directory
if (!directory.parent || !directory.parent.id) { // set parent if not set yet
directory.parent = <any>{id: currentDirId};
delete directory.media;
await directoryRepository.save(directory);
}
} else { // dir does not exists yet
scannedDirectory.directories[i].parent = <any>{id: currentDirId};
(<DirectoryEntity>scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet
const d = await directoryRepository.insert(<DirectoryEntity>scannedDirectory.directories[i]);
await this.saveMedia(connection, d.identifiers[0].id, scannedDirectory.directories[i].media);
}
}
// Remove child Dirs that are not anymore in the parent dir
await directoryRepository.remove(childDirectories, {chunk: Math.max(Math.ceil(childDirectories.length / 500), 1)});
}
cancelIndexing(): void {
Logger.info(LOG_TAG, 'Canceling indexing');
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
if (global.gc) {
global.gc();
protected async saveMetaFiles(connection: Connection, currentDirID: number, scannedDirectory: DirectoryDTO) {
const fileRepository = connection.getRepository(FileEntity);
// save files
const indexedMetaFiles = await fileRepository.createQueryBuilder('file')
.where('file.directory = :dir', {
dir: currentDirID
}).getMany();
const metaFilesToSave = [];
for (let i = 0; i < scannedDirectory.metaFile.length; i++) {
let metaFile: FileDTO = null;
for (let j = 0; j < indexedMetaFiles.length; j++) {
if (indexedMetaFiles[j].name === scannedDirectory.metaFile[i].name) {
metaFile = indexedMetaFiles[j];
indexedMetaFiles.splice(j, 1);
break;
}
}
if (metaFile == null) { // not in DB yet
scannedDirectory.metaFile[i].directory = null;
metaFile = Utils.clone(scannedDirectory.metaFile[i]);
scannedDirectory.metaFile[i].directory = scannedDirectory;
metaFile.directory = <any>{id: currentDirID};
metaFilesToSave.push(metaFile);
}
}
await fileRepository.save(metaFilesToSave, {chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1)});
await fileRepository.remove(indexedMetaFiles, {chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1)});
}
protected async saveMedia(connection: Connection, parentDirId: number, media: MediaDTO[]) {
const mediaRepository = connection.getRepository(MediaEntity);
const photoRepository = connection.getRepository(PhotoEntity);
const videoRepository = connection.getRepository(VideoEntity);
// save media
let indexedMedia = (await mediaRepository.createQueryBuilder('media')
.where('media.directory = :dir', {
dir: parentDirId
})
.getMany());
const mediaChange: any = {
saveP: [],
saveV: [],
insertP: [],
insertV: []
};
const facesPerPhoto: { faces: FaceRegionEntry[], mediaName: string }[] = [];
for (let i = 0; i < media.length; i++) {
let mediaItem: MediaEntity = null;
for (let j = 0; j < indexedMedia.length; j++) {
if (indexedMedia[j].name === media[i].name) {
mediaItem = indexedMedia[j];
indexedMedia.splice(j, 1);
break;
}
}
const scannedFaces = (<PhotoMetadata>media[i].metadata).faces || [];
delete (<PhotoMetadata>media[i].metadata).faces;
// let mediaItemId: number = null;
if (mediaItem == null) { // not in DB yet
media[i].directory = null;
mediaItem = <any>Utils.clone(media[i]);
mediaItem.directory = <any>{id: parentDirId};
(MediaDTO.isPhoto(mediaItem) ? mediaChange.insertP : mediaChange.insertV).push(mediaItem);
} else {
delete (<PhotoMetadata>mediaItem.metadata).faces;
if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
mediaItem.metadata = <any>media[i].metadata;
(MediaDTO.isPhoto(mediaItem) ? mediaChange.saveP : mediaChange.saveV).push(mediaItem);
}
}
facesPerPhoto.push({faces: scannedFaces as FaceRegionEntry[], mediaName: mediaItem.name});
}
await this.saveChunk(photoRepository, mediaChange.saveP, 100);
await this.saveChunk(videoRepository, mediaChange.saveV, 100);
await this.saveChunk(photoRepository, mediaChange.insertP, 100);
await this.saveChunk(videoRepository, mediaChange.insertV, 100);
indexedMedia = (await mediaRepository.createQueryBuilder('media')
.where('media.directory = :dir', {
dir: parentDirId
})
.select(['media.name', 'media.id'])
.getMany());
const faces: FaceRegionEntry[] = [];
facesPerPhoto.forEach(group => {
const mIndex = indexedMedia.findIndex(m => m.name === group.mediaName);
group.faces.forEach((sf: FaceRegionEntry) => sf.media = <any>{id: indexedMedia[mIndex].id});
faces.push(...group.faces);
indexedMedia.splice(mIndex, 1);
});
await this.saveFaces(connection, parentDirId, faces);
await mediaRepository.remove(indexedMedia);
}
protected async saveFaces(connection: Connection, parentDirId: number, scannedFaces: FaceRegion[]) {
const faceRepository = connection.getRepository(FaceRegionEntry);
const persons: string[] = [];
for (let i = 0; i < scannedFaces.length; i++) {
if (persons.indexOf(scannedFaces[i].name) === -1) {
persons.push(scannedFaces[i].name);
}
}
await ObjectManagerRepository.getInstance().PersonManager.saveAll(persons);
const indexedFaces = await faceRepository.createQueryBuilder('face')
.leftJoin('face.media', 'media')
.where('media.directory = :directory', {
directory: parentDirId
})
.leftJoinAndSelect('face.person', 'person')
.getMany();
const faceToInsert = [];
for (let i = 0; i < scannedFaces.length; i++) {
let face: FaceRegionEntry = null;
for (let j = 0; j < indexedFaces.length; j++) {
if (indexedFaces[j].box.height === scannedFaces[i].box.height &&
indexedFaces[j].box.width === scannedFaces[i].box.width &&
indexedFaces[j].box.x === scannedFaces[i].box.x &&
indexedFaces[j].box.y === scannedFaces[i].box.y &&
indexedFaces[j].person.name === scannedFaces[i].name) {
face = indexedFaces[j];
indexedFaces.splice(j, 1);
break;
}
}
if (face == null) {
(<FaceRegionEntry>scannedFaces[i]).person = await ObjectManagerRepository.getInstance().PersonManager.get(scannedFaces[i].name);
faceToInsert.push(scannedFaces[i]);
}
}
if (faceToInsert.length > 0) {
await this.insertChunk(faceRepository, faceToInsert, 100);
}
await faceRepository.remove(indexedFaces, {chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1)});
}
protected async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
this.isSaving = true;
try {
const connection = await SQLConnection.getConnection();
const currentDirId: number = await this.saveParentDir(connection, scannedDirectory);
await this.saveChildDirs(connection, currentDirId, scannedDirectory);
await this.saveMedia(connection, currentDirId, scannedDirectory.media);
await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
} catch (e) {
throw e;
} finally {
this.isSaving = false;
}
}
async reset(): Promise<void> {
Logger.info(LOG_TAG, 'Resetting DB');
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
const connection = await SQLConnection.getConnection();
return connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.delete()
.execute().then(() => {
});
private async saveChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<T[]> {
if (entities.length === 0) {
return [];
}
if (entities.length < size) {
return await repository.save(entities);
}
let list: T[] = [];
for (let i = 0; i < entities.length / size; i++) {
list = list.concat(await repository.save(entities.slice(i * size, (i + 1) * size)));
}
return list;
}
private async insertChunk<T>(repository: Repository<any>, entities: T[], size: number): Promise<number[]> {
if (entities.length === 0) {
return [];
}
if (entities.length < size) {
return (await repository.insert(entities)).identifiers.map((i: any) => i.id);
}
let list: number[] = [];
for (let i = 0; i < entities.length / size; i++) {
list = list.concat((await repository.insert(entities.slice(i * size, (i + 1) * size))).identifiers.map(ids => ids.id));
}
return list;
}
}

118
backend/model/sql/IndexingTaskManager.ts Normal file
View File

@ -0,0 +1,118 @@
import {IIndexingTaskManager} from '../interfaces/IIndexingTaskManager';
import {IndexingProgressDTO} from '../../../common/entities/settings/IndexingProgressDTO';
import {ObjectManagerRepository} from '../ObjectManagerRepository';
import * as path from 'path';
import * as fs from 'fs';
import {SQLConnection} from './SQLConnection';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {Logger} from '../../Logger';
import {RendererInput, ThumbnailSourceType, ThumbnailWorker} from '../threading/ThumbnailWorker';
import {Config} from '../../../common/config/private/Config';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {ProjectPath} from '../../ProjectPath';
import {ThumbnailGeneratorMWs} from '../../middlewares/thumbnail/ThumbnailGeneratorMWs';
const LOG_TAG = '[IndexingTaskManager]';
export class IndexingTaskManager implements IIndexingTaskManager {
directoriesToIndex: string[] = [];
indexingProgress: IndexingProgressDTO = null;
enabled = false;
private indexNewDirectory = async (createThumbnails: boolean = false) => {
if (this.directoriesToIndex.length === 0) {
this.indexingProgress = null;
if (global.gc) {
global.gc();
}
return;
}
const directory = this.directoriesToIndex.shift();
this.indexingProgress.current = directory;
this.indexingProgress.left = this.directoriesToIndex.length;
const scanned = await ObjectManagerRepository.getInstance().IndexingManager.indexDirectory(directory);
if (this.enabled === false) {
return;
}
this.indexingProgress.indexed++;
this.indexingProgress.time.current = Date.now();
for (let i = 0; i < scanned.directories.length; i++) {
this.directoriesToIndex.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
}
if (createThumbnails) {
for (let i = 0; i < scanned.media.length; i++) {
try {
const media = scanned.media[i];
const mPath = path.join(ProjectPath.ImageFolder, media.directory.path, media.directory.name, media.name);
const thPath = path.join(ProjectPath.ThumbnailFolder,
ThumbnailGeneratorMWs.generateThumbnailName(mPath, Config.Client.Thumbnail.thumbnailSizes[0]));
if (fs.existsSync(thPath)) { // skip existing thumbnails
continue;
}
await ThumbnailWorker.render(<RendererInput>{
type: MediaDTO.isVideo(media) ? ThumbnailSourceType.Video : ThumbnailSourceType.Image,
mediaPath: mPath,
size: Config.Client.Thumbnail.thumbnailSizes[0],
thPath: thPath,
makeSquare: false,
qualityPriority: Config.Server.thumbnail.qualityPriority
}, Config.Server.thumbnail.processingLibrary);
} catch (e) {
console.error(e);
Logger.error(LOG_TAG, 'Error during indexing job: ' + e.toString());
}
}
}
process.nextTick(() => {
this.indexNewDirectory(createThumbnails).catch(console.error);
});
};
startIndexing(createThumbnails: boolean = false): void {
if (this.directoriesToIndex.length === 0 && this.enabled === false) {
Logger.info(LOG_TAG, 'Starting indexing');
this.indexingProgress = {
indexed: 0,
left: 0,
current: '',
time: {
start: Date.now(),
current: Date.now()
}
};
this.directoriesToIndex.push('/');
this.enabled = true;
this.indexNewDirectory(createThumbnails).catch(console.error);
} else {
Logger.info(LOG_TAG, 'Already indexing..');
}
}
getProgress(): IndexingProgressDTO {
return this.indexingProgress;
}
cancelIndexing(): void {
Logger.info(LOG_TAG, 'Canceling indexing');
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
if (global.gc) {
global.gc();
}
}
async reset(): Promise<void> {
Logger.info(LOG_TAG, 'Resetting DB');
this.directoriesToIndex = [];
this.indexingProgress = null;
this.enabled = false;
const connection = await SQLConnection.getConnection();
return connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.delete()
.execute().then(() => {
});
}
}

50
backend/model/sql/PersonManager.ts Normal file
View File

@ -0,0 +1,50 @@
import {IPersonManager} from '../interfaces/IPersonManager';
import {PersonEntry} from './enitites/PersonEntry';
import {SQLConnection} from './SQLConnection';
const LOG_TAG = '[PersonManager]';
export class PersonManager implements IPersonManager {
persons: PersonEntry[] = [];
async get(name: string): Promise<PersonEntry> {
let person = this.persons.find(p => p.name === name);
if (!person) {
const connection = await SQLConnection.getConnection();
const personRepository = connection.getRepository(PersonEntry);
person = await personRepository.findOne({name: name});
if (!person) {
person = await personRepository.save(<PersonEntry>{name: name});
}
this.persons.push(person);
}
return person;
}
async saveAll(names: string[]): Promise<void> {
const toSave: { name: string }[] = [];
const connection = await SQLConnection.getConnection();
const personRepository = connection.getRepository(PersonEntry);
this.persons = await personRepository.find();
for (let i = 0; i < names.length; i++) {
const person = this.persons.find(p => p.name === names[i]);
if (!person) {
toSave.push({name: names[i]});
}
}
if (toSave.length > 0) {
for (let i = 0; i < toSave.length / 200; i++) {
await personRepository.insert(toSave.slice(i * 200, (i + 1) * 200));
}
this.persons = await personRepository.find();
}
}
}

2
backend/model/sql/SQLConnection.ts
View File

@ -44,7 +44,7 @@ export class SQLConnection {
VersionEntity
];
options.synchronize = false;
options.logging = 'all';
// options.logging = 'all';
this.connection = await createConnection(options);
await SQLConnection.schemeSync(this.connection);
}

0
backend/tsconfig.json → backend/tsconfigX.jsonX
View File

2
common/entities/PhotoDTO.ts
View File

@ -1,6 +1,6 @@
import {DirectoryDTO} from './DirectoryDTO';
import {OrientationTypes} from 'ts-exif-parser';
import {MediaDTO, MediaMetadata, MediaDimension} from './MediaDTO';
import {MediaDimension, MediaDTO, MediaMetadata} from './MediaDTO';
export interface PhotoDTO extends MediaDTO {
id: number;

2
package.json
View File

@ -42,7 +42,7 @@
"ts-exif-parser": "0.1.4",
"ts-node-iptc": "1.0.11",
"typeconfig": "1.0.7",
"typeorm": "0.2.9",
"typeorm": "0.2.11",
"winston": "2.4.2",
"xmldom": "0.1.27"
},

BIN
test/backend/unit/assets/test image öüóőúéáű-.,.jpg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 58 KiB

After

Width:  |  Height:  |  Size: 58 KiB

45
test/backend/unit/assets/test image öüóőúéáű-.,.json Normal file
View File

@ -0,0 +1,45 @@
{
"cameraData": {
"ISO": 3200,
"exposure": 0.00125,
"fStop": 5.6,
"focalLength": 85,
"lens": "EF-S15-85mm f/3.5-5.6 IS USM",
"make": "Canon",
"model": "óüöúőűáé ÓÜÖÚŐŰÁÉ"
},
"caption": "Test caption",
"creationDate": 1434018566000,
"faces": [
{
"box": {
"height": 19,
"width": 20,
"x": 82,
"y": 38
},
"name": "squirrel"
}
],
"fileSize": 59187,
"keywords": [
"Berkley",
"USA",
"űáéúőóüö ŰÁÉÚŐÓÜÖ"
],
"orientation": 1,
"positionData": {
"GPSData": {
"altitude": 90,
"latitude": 37.871093333333334,
"longitude": -122.25678
},
"city": "test city őúéáűóöí-.,)(=",
"country": "test country őúéáűóöí-.,)(=/%!+\"'",
"state": "test state őúéáűóöí-.,)("
},
"size": {
"height": 93,
"width": 140
}
}

58
test/backend/unit/model/sql/GalleryManager.ts → test/backend/unit/model/sql/IndexingManager.ts
View File

@ -12,6 +12,9 @@ import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/Directo
import {Utils} from '../../../../../common/Utils';
import {MediaDTO} from '../../../../../common/entities/MediaDTO';
import {FileDTO} from '../../../../../common/entities/FileDTO';
import {IndexingManager} from '../../../../../backend/model/sql/IndexingManager';
import {ObjectManagerRepository} from '../../../../../backend/model/ObjectManagerRepository';
import {PersonManager} from '../../../../../backend/model/sql/PersonManager';
class GalleryManagerTest extends GalleryManager {
@ -24,16 +27,21 @@ class GalleryManagerTest extends GalleryManager {
return super.fillParentDir(connection, dir);
}
public async saveToDB(scannedDirectory: DirectoryDTO) {
return super.saveToDB(scannedDirectory);
}
}
class IndexingManagerTest extends IndexingManager {
public async queueForSave(scannedDirectory: DirectoryDTO): Promise<void> {
return super.queueForSave(scannedDirectory);
}
public async saveToDB(scannedDirectory: DirectoryDTO): Promise<void> {
return super.saveToDB(scannedDirectory);
}
}
describe('GalleryManager', () => {
describe('IndexingManager', () => {
const tempDir = path.join(__dirname, '../../tmp');
@ -50,6 +58,7 @@ describe('GalleryManager', () => {
Config.Server.database.type = DatabaseType.sqlite;
Config.Server.database.sqlite.storage = dbPath;
ObjectManagerRepository.getInstance().PersonManager = new PersonManager();
};
@ -94,18 +103,19 @@ describe('GalleryManager', () => {
it('should save parent directory', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry();
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 0);
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 0);
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(Utils.clone(parent));
await im.saveToDB(Utils.clone(parent));
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
@ -122,13 +132,14 @@ describe('GalleryManager', () => {
it('should skip meta files', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry();
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
DirectoryDTO.removeReferences(parent);
Config.Client.MetaFile.enabled = true;
await gm.saveToDB(Utils.clone(parent));
await im.saveToDB(Utils.clone(parent));
Config.Client.MetaFile.enabled = false;
const conn = await SQLConnection.getConnection();
@ -144,6 +155,7 @@ describe('GalleryManager', () => {
it('should update sub directory', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry();
parent.name = 'parent';
@ -153,13 +165,13 @@ describe('GalleryManager', () => {
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(Utils.clone(parent));
await im.saveToDB(Utils.clone(parent));
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
const sp3 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto3');
DirectoryDTO.removeReferences(subDir);
await gm.saveToDB(Utils.clone(subDir));
await im.saveToDB(Utils.clone(subDir));
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
@ -179,20 +191,21 @@ describe('GalleryManager', () => {
it('should avoid race condition', async () => {
const conn = await SQLConnection.getConnection();
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
Config.Client.MetaFile.enabled = true;
const parent = TestHelper.getRandomizedDirectoryEntry();
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
const gpx = TestHelper.getRandomizedGPXEntry(parent, 'GPX1');
const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1');
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2');
const sp1 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto1', 1);
const sp2 = TestHelper.getRandomizedPhotoEntry(subDir, 'subPhoto2', 1);
DirectoryDTO.removeReferences(parent);
const s1 = gm.queueForSave(Utils.clone(parent));
const s2 = gm.queueForSave(Utils.clone(parent));
const s3 = gm.queueForSave(Utils.clone(parent));
const s1 = im.queueForSave(Utils.clone(parent));
const s2 = im.queueForSave(Utils.clone(parent));
const s3 = im.queueForSave(Utils.clone(parent));
await Promise.all([s1, s2, s3]);
@ -204,30 +217,33 @@ describe('GalleryManager', () => {
subDir.isPartial = true;
delete subDir.directories;
delete subDir.metaFile;
delete sp1.metadata.faces;
delete sp2.metadata.faces;
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
.to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
});
(<any>it('should save 1500 photos', async () => {
(it('should save 1500 photos', async () => {
const conn = await SQLConnection.getConnection();
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
Config.Client.MetaFile.enabled = true;
const parent = TestHelper.getRandomizedDirectoryEntry();
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(Utils.clone(parent));
await im.saveToDB(Utils.clone(parent));
const subDir = TestHelper.getRandomizedDirectoryEntry(parent, 'subDir');
for (let i = 0; i < 1500; i++) {
TestHelper.getRandomizedPhotoEntry(subDir, 'p' + i);
}
DirectoryDTO.removeReferences(parent);
await gm.saveToDB(subDir);
await im.saveToDB(subDir);
const selected = await gm.selectParentDir(conn, subDir.name, subDir.path);
expect(selected.media.length).to.deep.equal(subDir.media.length);
})).timeout(20000);
}) as any).timeout(40000);
describe('Test listDirectory', () => {
const statSync = fs.statSync;
@ -239,6 +255,8 @@ describe('GalleryManager', () => {
beforeEach(() => {
dirTime = 0;
ObjectManagerRepository.getInstance().IndexingManager = new IndexingManagerTest();
indexedTime.lastModified = 0;
indexedTime.lastScanned = 0;
});
@ -261,7 +279,7 @@ describe('GalleryManager', () => {
return Promise.resolve();
};
gm.indexDirectory = (...args) => {
ObjectManagerRepository.getInstance().IndexingManager.indexDirectory = (...args) => {
return <any>Promise.resolve('indexing');
};

37
test/backend/unit/model/sql/TestHelper.ts
View File

@ -1,7 +1,8 @@
import {MediaDimensionEntity} from '../../../../../backend/model/sql/enitites/MediaEntity';
import {
CameraMetadataEntity,
GPSMetadataEntity, PhotoEntity,
GPSMetadataEntity,
PhotoEntity,
PhotoMetadataEntity,
PositionMetaDataEntity
} from '../../../../../backend/model/sql/enitites/PhotoEntity';
@ -9,9 +10,8 @@ import * as path from 'path';
import {OrientationTypes} from 'ts-exif-parser';
import {DirectoryEntity} from '../../../../../backend/model/sql/enitites/DirectoryEntity';
import {VideoEntity, VideoMetadataEntity} from '../../../../../backend/model/sql/enitites/VideoEntity';
import {FileEntity} from '../../../../../backend/model/sql/enitites/FileEntity';
import {MediaDimension} from '../../../../../common/entities/MediaDTO';
import {CameraMetadata, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
import {CameraMetadata, FaceRegion, GPSMetadata, PhotoDTO, PhotoMetadata, PositionMetaData} from '../../../../../common/entities/PhotoDTO';
import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO';
import {FileDTO} from '../../../../../common/entities/FileDTO';
@ -157,14 +157,37 @@ export class TestHelper {
return d;
}
public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null) {
public static getRandomizedFace(media: PhotoDTO, forceStr: string = null) {
const rndStr = () => {
return forceStr + '_' + Math.random().toString(36).substring(7);
};
const rndInt = (max = 5000) => {
return Math.floor(Math.random() * max);
};
const f: FaceRegion = {
name: rndStr() + '.jpg',
box: {
x: rndInt(),
y: rndInt(),
width: rndInt(),
height: rndInt()
}
};
media.metadata.faces = (media.metadata.faces || []);
media.metadata.faces.push(f);
return f;
}
public static getRandomizedPhotoEntry(dir: DirectoryDTO, forceStr: string = null, faces: number = 2): PhotoDTO {
const rndStr = () => {
return forceStr + '_' + Math.random().toString(36).substring(7);
};
const rndInt = (max = 5000) => {
return Math.floor(Math.random() * max);
};
@ -215,6 +238,10 @@ export class TestHelper {
readyIcon: false
};
for (let i = 0; i < faces; i++) {
this.getRandomizedFace(d, 'Person ' + i);
}
dir.media.push(d);
return d;
}

32
test/backend/unit/model/threading/DiskMangerWorker.spec.ts
View File

@ -3,7 +3,7 @@ import {DiskMangerWorker} from '../../../../../backend/model/threading/DiskMange
import * as path from 'path';
import {Config} from '../../../../../common/config/private/Config';
import {ProjectPath} from '../../../../../backend/ProjectPath';
import {PhotoDTO} from '../../../../../common/entities/PhotoDTO';
import {Utils} from '../../../../../common/Utils';
describe('DiskMangerWorker', () => {
@ -12,33 +12,9 @@ describe('DiskMangerWorker', () => {
ProjectPath.ImageFolder = path.join(__dirname, '/../../assets');
const dir = await DiskMangerWorker.scanDirectory('/');
expect(dir.media.length).to.be.equals(2);
expect(dir.media[0].name).to.be.equals('test image öüóőúéáű-.,.jpg');
expect((<PhotoDTO>dir.media[0]).metadata.keywords).to.deep.equals(['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']);
expect(dir.media[0].metadata.fileSize).to.deep.equals(62786);
expect(dir.media[0].metadata.size).to.deep.equals({width: 140, height: 93});
expect((<PhotoDTO>dir.media[0]).metadata.cameraData).to.deep.equals({
ISO: 3200,
model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
make: 'Canon',
fStop: 5.6,
exposure: 0.00125,
focalLength: 85,
lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
});
expect((<PhotoDTO>dir.media[0]).metadata.positionData).to.deep.equals({
GPSData: {
latitude: 37.871093333333334,
longitude: -122.25678,
altitude: 102.4498997995992
},
country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
});
expect(dir.media[0].metadata.creationDate).to.be.equals(1434018566000);
const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
expect(Utils.clone(dir.media[0].name)).to.be.deep.equal('test image öüóőúéáű-.,.jpg');
expect(Utils.clone(dir.media[0].metadata)).to.be.deep.equal(expected);
});
});

32
test/backend/unit/model/threading/MetaDataLoader.spec.ts
View File

@ -20,36 +20,8 @@ describe('MetadataLoader', () => {
it('should load jpg', async () => {
const data = await MetadataLoader.loadPhotoMetadata(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.jpg'));
expect(Utils.clone(data)).to.be.deep.equal(Utils.clone({
size: {width: 140, height: 93},
orientation: 1,
caption: 'Test caption',
creationDate: 1434018566000,
fileSize: 62786,
cameraData:
{
ISO: 3200,
model: 'óüöúőűáé ÓÜÖÚŐŰÁÉ',
make: 'Canon',
fStop: 5.6,
exposure: 0.00125,
focalLength: 85,
lens: 'EF-S15-85mm f/3.5-5.6 IS USM'
},
positionData:
{
GPSData:
{
latitude: 37.871093333333334,
longitude: -122.25678,
altitude: 102.4498997995992
},
country: 'mmóüöúőűáé ÓÜÖÚŐŰÁÉmm-.,|\\mm',
state: 'óüöúőűáé ÓÜÖÚŐŰÁ',
city: 'óüöúőűáé ÓÜÖÚŐŰÁ'
},
keywords: ['Berkley', 'USA', 'űáéúőóüö ŰÁÉÚŐÓÜÖ']
}));
const expected = require(path.join(__dirname, '/../../assets/test image öüóőúéáű-.,.json'));
expect(Utils.clone(data)).to.be.deep.equal(expected);
});
});