From ee18d27f9deb0456218f989d1993bbaed59ef403 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Thu, 7 Feb 2019 10:21:23 -0500 Subject: [PATCH] changing mysql collate and charset to be case insensitive --- backend/model/sql/GalleryManager.ts | 2 +- backend/model/sql/SQLConnection.ts | 3 +- backend/model/sql/SearchManager.ts | 52 +++++++++---------- backend/model/sql/enitites/DirectoryEntity.ts | 3 +- backend/model/sql/enitites/EntityUtils.ts | 17 ++++++ backend/model/sql/enitites/FileEntity.ts | 3 +- backend/model/sql/enitites/MediaEntity.ts | 7 ++- common/entities/TaskDTO.ts | 28 ++++++++++ test/backend/integration/model/sql/typeorm.ts | 4 +- .../backend/unit/model/sql/IndexingManager.ts | 24 +++++++++ 10 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 backend/model/sql/enitites/EntityUtils.ts create mode 100644 common/entities/TaskDTO.ts diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 146ea9e7..379ca838 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -100,7 +100,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { }); if (queryFilter.recursive) { - qb.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + queryFilter.directory + '%'}); + qb.orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + queryFilter.directory + '%'}); } })); } diff --git a/backend/model/sql/SQLConnection.ts b/backend/model/sql/SQLConnection.ts index 6cc7cdef..ae828a0b 100644 --- a/backend/model/sql/SQLConnection.ts +++ b/backend/model/sql/SQLConnection.ts @@ -170,7 +170,8 @@ export class SQLConnection { port: 3306, username: config.mysql.username, password: config.mysql.password, - database: config.mysql.database + database: config.mysql.database, + charset: 'utf8mb4' }; } else if (config.type === DatabaseType.sqlite) { driver = { diff --git a/backend/model/sql/SearchManager.ts b/backend/model/sql/SearchManager.ts index 12d76500..987f0eeb 100644 --- a/backend/model/sql/SearchManager.ts +++ b/backend/model/sql/SearchManager.ts @@ -40,7 +40,7 @@ export class SearchManager implements ISearchManager { (await photoRepository .createQueryBuilder('photo') .select('DISTINCT(photo.metadata.keywords)') - .where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('photo.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => >(r.metadataKeywords).split(',')) @@ -52,7 +52,7 @@ export class SearchManager implements ISearchManager { result = result.concat(this.encapsulateAutoComplete((await personRepository .createQueryBuilder('person') .select('DISTINCT(person.name)') - .where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .orderBy('person.name') .getRawMany()) @@ -62,9 +62,9 @@ export class SearchManager implements ISearchManager { .createQueryBuilder('photo') .select('photo.metadata.positionData.country as country, ' + 'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city') - .where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('photo.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city') .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) @@ -78,7 +78,7 @@ export class SearchManager implements ISearchManager { result = result.concat(this.encapsulateAutoComplete((await photoRepository .createQueryBuilder('media') .select('DISTINCT(media.name)') - .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.photo)); @@ -87,7 +87,7 @@ export class SearchManager implements ISearchManager { result = result.concat(this.encapsulateAutoComplete((await photoRepository .createQueryBuilder('media') .select('DISTINCT(media.metadata.caption) as caption') - .where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.caption), SearchTypes.photo)); @@ -96,7 +96,7 @@ export class SearchManager implements ISearchManager { result = result.concat(this.encapsulateAutoComplete((await videoRepository .createQueryBuilder('media') .select('DISTINCT(media.name)') - .where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.video)); @@ -104,7 +104,7 @@ export class SearchManager implements ISearchManager { result = result.concat(this.encapsulateAutoComplete((await directoryRepository .createQueryBuilder('dir') .select('DISTINCT(dir.name)') - .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(Config.Client.Search.AutoComplete.maxItemsPerCategory) .getRawMany()) .map(r => r.name), SearchTypes.directory)); @@ -142,31 +142,31 @@ export class SearchManager implements ISearchManager { if (!searchType || searchType === SearchTypes.directory) { subQuery.leftJoin('media.directory', 'directory') - .orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + .orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) { - subQuery.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + subQuery.orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } if (!searchType || searchType === SearchTypes.photo) { - subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } if (!searchType || searchType === SearchTypes.person) { subQuery .leftJoin('media.metadata.faces', 'faces') .leftJoin('faces.person', 'person') - .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + .orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } if (!searchType || searchType === SearchTypes.position) { - subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } if (!searchType || searchType === SearchTypes.keyword) { - subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}); + subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}); } return subQuery; @@ -187,7 +187,7 @@ export class SearchManager implements ISearchManager { result.directories = await connection .getRepository(DirectoryEntity) .createQueryBuilder('dir') - .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(201) .getMany(); @@ -217,13 +217,13 @@ export class SearchManager implements ISearchManager { .leftJoin('media.directory', 'directory') .leftJoin('media.metadata.faces', 'faces') .leftJoin('faces.person', 'person') - .where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) - .orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) + .orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) , 'innerMedia', 'media.id=innerMedia.id') @@ -238,7 +238,7 @@ export class SearchManager implements ISearchManager { result.directories = await connection .getRepository(DirectoryEntity) .createQueryBuilder('dir') - .where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'}) + .where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'}) .limit(10) .getMany(); diff --git a/backend/model/sql/enitites/DirectoryEntity.ts b/backend/model/sql/enitites/DirectoryEntity.ts index 717b53c4..d8e11712 100644 --- a/backend/model/sql/enitites/DirectoryEntity.ts +++ b/backend/model/sql/enitites/DirectoryEntity.ts @@ -2,6 +2,7 @@ import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Uni import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO'; import {MediaEntity} from './MediaEntity'; import {FileEntity} from './FileEntity'; +import {columnCharsetCS} from './EntityUtils'; @Entity() @Unique(['name', 'path']) @@ -12,7 +13,7 @@ export class DirectoryEntity implements DirectoryDTO { id: number; @Index() - @Column() + @Column(columnCharsetCS) name: string; @Index() diff --git a/backend/model/sql/enitites/EntityUtils.ts b/backend/model/sql/enitites/EntityUtils.ts new file mode 100644 index 00000000..8dbb85e8 --- /dev/null +++ b/backend/model/sql/enitites/EntityUtils.ts @@ -0,0 +1,17 @@ +import {Config} from '../../../../common/config/private/Config'; +import {DatabaseType} from '../../../../common/config/private/IPrivateConfig'; +import {ColumnOptions} from 'typeorm/decorator/options/ColumnOptions'; + +export class ColumnCharsetCS implements ColumnOptions { + + public get charset(): string { + return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4' : null; + } + + public get collation(): string { + return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4_bin' : null; + + } +} + +export const columnCharsetCS = new ColumnCharsetCS(); diff --git a/backend/model/sql/enitites/FileEntity.ts b/backend/model/sql/enitites/FileEntity.ts index b185cfea..4b36c874 100644 --- a/backend/model/sql/enitites/FileEntity.ts +++ b/backend/model/sql/enitites/FileEntity.ts @@ -1,6 +1,7 @@ import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, Index} from 'typeorm'; import {DirectoryEntity} from './DirectoryEntity'; import {FileDTO} from '../../../../common/entities/FileDTO'; +import {columnCharsetCS} from './EntityUtils'; @Entity() @@ -10,7 +11,7 @@ export class FileEntity implements FileDTO { @PrimaryGeneratedColumn({unsigned: true}) id: number; - @Column('text') + @Column(columnCharsetCS) name: string; @Index() diff --git a/backend/model/sql/enitites/MediaEntity.ts b/backend/model/sql/enitites/MediaEntity.ts index 10d3693a..662731a7 100644 --- a/backend/model/sql/enitites/MediaEntity.ts +++ b/backend/model/sql/enitites/MediaEntity.ts @@ -4,6 +4,9 @@ import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entiti import {OrientationTypes} from 'ts-exif-parser'; import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity'; import {FaceRegionEntry} from './FaceRegionEntry'; +import {Config} from '../../../../common/config/private/Config'; +import {DatabaseType} from '../../../../common/config/private/IPrivateConfig'; +import {columnCharsetCS} from './EntityUtils'; export class MediaDimensionEntity implements MediaDimension { @@ -55,6 +58,8 @@ export class MediaMetadataEntity implements MediaMetadata { duration: number; } + + // TODO: fix inheritance once its working in typeorm @Entity() @Unique(['name', 'directory']) @@ -65,7 +70,7 @@ export abstract class MediaEntity implements MediaDTO { @PrimaryGeneratedColumn({unsigned: true}) id: number; - @Column() + @Column(columnCharsetCS) name: string; @Index() diff --git a/common/entities/TaskDTO.ts b/common/entities/TaskDTO.ts new file mode 100644 index 00000000..15004746 --- /dev/null +++ b/common/entities/TaskDTO.ts @@ -0,0 +1,28 @@ +export interface TaskType { + name: string; + parameter: any; +} + +export enum TaskTriggerType { + scheduled, periodic +} + +export interface TaskTrigger { + type: TaskTriggerType; +} + +export interface ScheduledTaskTrigger extends TaskTrigger { + type: TaskTriggerType.scheduled; + time: number; +} + +export interface PeriodicTaskTrigger extends TaskTrigger { + type: TaskTriggerType.periodic; + +} + +export interface TaskDTO { + priority: number; + type: TaskType; + trigger: TaskTrigger; +} diff --git a/test/backend/integration/model/sql/typeorm.ts b/test/backend/integration/model/sql/typeorm.ts index c159efcb..e7744627 100644 --- a/test/backend/integration/model/sql/typeorm.ts +++ b/test/backend/integration/model/sql/typeorm.ts @@ -184,7 +184,7 @@ describe('Typeorm integration', () => { const photos = await pr .createQueryBuilder('media') .orderBy('media.metadata.creationDate', 'ASC') - .where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + photo.metadata.positionData.city + '%'}) + .where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + photo.metadata.positionData.city + '%'}) .innerJoinAndSelect('media.directory', 'directory') .limit(10) .getMany(); @@ -206,7 +206,7 @@ describe('Typeorm integration', () => { const photos = await pr .createQueryBuilder('media') .orderBy('media.metadata.creationDate', 'ASC') - .where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + city + '%'}) + .where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + city + '%'}) .innerJoinAndSelect('media.directory', 'directory') .limit(10) .getMany(); diff --git a/test/backend/unit/model/sql/IndexingManager.ts b/test/backend/unit/model/sql/IndexingManager.ts index 489de840..3d7edbfa 100644 --- a/test/backend/unit/model/sql/IndexingManager.ts +++ b/test/backend/unit/model/sql/IndexingManager.ts @@ -44,6 +44,7 @@ class IndexingManagerTest extends IndexingManager { // to help WebStorm to handle the test cases declare let describe: any; declare const after: any; +declare const it: any; describe = SQLTestHelper.describe; describe('IndexingManager', (sqlHelper: SQLTestHelper) => { @@ -80,6 +81,29 @@ describe('IndexingManager', (sqlHelper: SQLTestHelper) => { } }; + it('should support case sensitive file names', async () => { + const gm = new GalleryManagerTest(); + const im = new IndexingManagerTest(); + + const parent = TestHelper.getRandomizedDirectoryEntry(); + const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1'); + const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2'); + p1.name = 'test.jpg'; + p2.name = 'Test.jpg'; + + DirectoryDTO.removeReferences(parent); + await im.saveToDB(Utils.clone(parent)); + + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, parent.name, parent.path); + await gm.fillParentDir(conn, selected); + + DirectoryDTO.removeReferences(selected); + removeIds(selected); + expect(Utils.clone(Utils.removeNullOrEmptyObj(selected))) + .to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent))); + }); + it('should save parent directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest();