xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00
Code Actions Projects Activity

changing mysql collate and charset to be case insensitive

Browse Source
This commit is contained in:
Patrik J. Braun 2019-02-07 10:21:23 -05:00
parent 6489fe3930
commit ee18d27f9d
10 changed files with 110 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/model/sql/GalleryManager.ts
View File

@ -100,7 +100,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
});
if (queryFilter.recursive) {
qb.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + queryFilter.directory + '%'});
qb.orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + queryFilter.directory + '%'});
}
}));
}

3
backend/model/sql/SQLConnection.ts
View File

@ -170,7 +170,8 @@ export class SQLConnection {
port: 3306,
username: config.mysql.username,
password: config.mysql.password,
database: config.mysql.database
database: config.mysql.database,
charset: 'utf8mb4'
};
} else if (config.type === DatabaseType.sqlite) {
driver = {

52
backend/model/sql/SearchManager.ts
View File

@ -40,7 +40,7 @@ export class SearchManager implements ISearchManager {
(await photoRepository
.createQueryBuilder('photo')
.select('DISTINCT(photo.metadata.keywords)')
.where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('photo.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => <Array<string>>(<string>r.metadataKeywords).split(','))
@ -52,7 +52,7 @@ export class SearchManager implements ISearchManager {
result = result.concat(this.encapsulateAutoComplete((await personRepository
.createQueryBuilder('person')
.select('DISTINCT(person.name)')
.where('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.orderBy('person.name')
.getRawMany())
@ -62,9 +62,9 @@ export class SearchManager implements ISearchManager {
.createQueryBuilder('photo')
.select('photo.metadata.positionData.country as country, ' +
'photo.metadata.positionData.state as state, photo.metadata.positionData.city as city')
.where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('photo.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
@ -78,7 +78,7 @@ export class SearchManager implements ISearchManager {
result = result.concat(this.encapsulateAutoComplete((await photoRepository
.createQueryBuilder('media')
.select('DISTINCT(media.name)')
.where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.name), SearchTypes.photo));
@ -87,7 +87,7 @@ export class SearchManager implements ISearchManager {
result = result.concat(this.encapsulateAutoComplete((await photoRepository
.createQueryBuilder('media')
.select('DISTINCT(media.metadata.caption) as caption')
.where('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.caption), SearchTypes.photo));
@ -96,7 +96,7 @@ export class SearchManager implements ISearchManager {
result = result.concat(this.encapsulateAutoComplete((await videoRepository
.createQueryBuilder('media')
.select('DISTINCT(media.name)')
.where('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.name), SearchTypes.video));
@ -104,7 +104,7 @@ export class SearchManager implements ISearchManager {
result = result.concat(this.encapsulateAutoComplete((await directoryRepository
.createQueryBuilder('dir')
.select('DISTINCT(dir.name)')
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(Config.Client.Search.AutoComplete.maxItemsPerCategory)
.getRawMany())
.map(r => r.name), SearchTypes.directory));
@ -142,31 +142,31 @@ export class SearchManager implements ISearchManager {
if (!searchType || searchType === SearchTypes.directory) {
subQuery.leftJoin('media.directory', 'directory')
.orWhere('directory.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
.orWhere('directory.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
if (!searchType || searchType === SearchTypes.photo || searchType === SearchTypes.video) {
subQuery.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
subQuery.orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
if (!searchType || searchType === SearchTypes.photo) {
subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
subQuery.orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
if (!searchType || searchType === SearchTypes.person) {
subQuery
.leftJoin('media.metadata.faces', 'faces')
.leftJoin('faces.person', 'person')
.orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
.orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
if (!searchType || searchType === SearchTypes.position) {
subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
subQuery.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
if (!searchType || searchType === SearchTypes.keyword) {
subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'});
subQuery.orWhere('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'});
}
return subQuery;
@ -187,7 +187,7 @@ export class SearchManager implements ISearchManager {
result.directories = await connection
.getRepository(DirectoryEntity)
.createQueryBuilder('dir')
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(201)
.getMany();
@ -217,13 +217,13 @@ export class SearchManager implements ISearchManager {
.leftJoin('media.directory', 'directory')
.leftJoin('media.metadata.faces', 'faces')
.leftJoin('faces.person', 'person')
.where('media.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.caption LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.orWhere('person.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('media.metadata.keywords LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.country LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.state LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('media.metadata.caption LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.orWhere('person.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
,
'innerMedia',
'media.id=innerMedia.id')
@ -238,7 +238,7 @@ export class SearchManager implements ISearchManager {
result.directories = await connection
.getRepository(DirectoryEntity)
.createQueryBuilder('dir')
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: '%' + text + '%'})
.where('dir.name LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + text + '%'})
.limit(10)
.getMany();

3
backend/model/sql/enitites/DirectoryEntity.ts
View File

@ -2,6 +2,7 @@ import {Column, Entity, Index, ManyToOne, OneToMany, PrimaryGeneratedColumn, Uni
import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {MediaEntity} from './MediaEntity';
import {FileEntity} from './FileEntity';
import {columnCharsetCS} from './EntityUtils';
@Entity()
@Unique(['name', 'path'])
@ -12,7 +13,7 @@ export class DirectoryEntity implements DirectoryDTO {
id: number;
@Index()
@Column()
@Column(columnCharsetCS)
name: string;
@Index()

17
backend/model/sql/enitites/EntityUtils.ts Normal file
View File

@ -0,0 +1,17 @@
import {Config} from '../../../../common/config/private/Config';
import {DatabaseType} from '../../../../common/config/private/IPrivateConfig';
import {ColumnOptions} from 'typeorm/decorator/options/ColumnOptions';
export class ColumnCharsetCS implements ColumnOptions {
public get charset(): string {
return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4' : null;
}
public get collation(): string {
return Config.Server.database.type === DatabaseType.mysql ? 'utf8mb4_bin' : null;
}
}
export const columnCharsetCS = new ColumnCharsetCS();

3
backend/model/sql/enitites/FileEntity.ts
View File

@ -1,6 +1,7 @@
import {Column, Entity, ManyToOne, PrimaryGeneratedColumn, Index} from 'typeorm';
import {DirectoryEntity} from './DirectoryEntity';
import {FileDTO} from '../../../../common/entities/FileDTO';
import {columnCharsetCS} from './EntityUtils';
@Entity()
@ -10,7 +11,7 @@ export class FileEntity implements FileDTO {
@PrimaryGeneratedColumn({unsigned: true})
id: number;
@Column('text')
@Column(columnCharsetCS)
name: string;
@Index()

7
backend/model/sql/enitites/MediaEntity.ts
View File

@ -4,6 +4,9 @@ import {MediaDimension, MediaDTO, MediaMetadata} from '../../../../common/entiti
import {OrientationTypes} from 'ts-exif-parser';
import {CameraMetadataEntity, PositionMetaDataEntity} from './PhotoEntity';
import {FaceRegionEntry} from './FaceRegionEntry';
import {Config} from '../../../../common/config/private/Config';
import {DatabaseType} from '../../../../common/config/private/IPrivateConfig';
import {columnCharsetCS} from './EntityUtils';
export class MediaDimensionEntity implements MediaDimension {
@ -55,6 +58,8 @@ export class MediaMetadataEntity implements MediaMetadata {
duration: number;
}
// TODO: fix inheritance once its working in typeorm
@Entity()
@Unique(['name', 'directory'])
@ -65,7 +70,7 @@ export abstract class MediaEntity implements MediaDTO {
@PrimaryGeneratedColumn({unsigned: true})
id: number;
@Column()
@Column(columnCharsetCS)
name: string;
@Index()

28
common/entities/TaskDTO.ts Normal file
View File

@ -0,0 +1,28 @@
export interface TaskType {
name: string;
parameter: any;
}
export enum TaskTriggerType {
scheduled, periodic
}
export interface TaskTrigger {
type: TaskTriggerType;
}
export interface ScheduledTaskTrigger extends TaskTrigger {
type: TaskTriggerType.scheduled;
time: number;
}
export interface PeriodicTaskTrigger extends TaskTrigger {
type: TaskTriggerType.periodic;
}
export interface TaskDTO {
priority: number;
type: TaskType;
trigger: TaskTrigger;
}

4
test/backend/integration/model/sql/typeorm.ts
View File

@ -184,7 +184,7 @@ describe('Typeorm integration', () => {
const photos = await pr
.createQueryBuilder('media')
.orderBy('media.metadata.creationDate', 'ASC')
.where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + photo.metadata.positionData.city + '%'})
.where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + photo.metadata.positionData.city + '%'})
.innerJoinAndSelect('media.directory', 'directory')
.limit(10)
.getMany();
@ -206,7 +206,7 @@ describe('Typeorm integration', () => {
const photos = await pr
.createQueryBuilder('media')
.orderBy('media.metadata.creationDate', 'ASC')
.where('media.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: '%' + city + '%'})
.where('media.metadata.positionData.city LIKE :text COLLATE utf8mb4_general_ci', {text: '%' + city + '%'})
.innerJoinAndSelect('media.directory', 'directory')
.limit(10)
.getMany();

24
test/backend/unit/model/sql/IndexingManager.ts
View File

@ -44,6 +44,7 @@ class IndexingManagerTest extends IndexingManager {
// to help WebStorm to handle the test cases
declare let describe: any;
declare const after: any;
declare const it: any;
describe = SQLTestHelper.describe;
describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
@ -80,6 +81,29 @@ describe('IndexingManager', (sqlHelper: SQLTestHelper) => {
}
};
it('should support case sensitive file names', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();
const parent = TestHelper.getRandomizedDirectoryEntry();
const p1 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo1');
const p2 = TestHelper.getRandomizedPhotoEntry(parent, 'Photo2');
p1.name = 'test.jpg';
p2.name = 'Test.jpg';
DirectoryDTO.removeReferences(parent);
await im.saveToDB(Utils.clone(parent));
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, parent.name, parent.path);
await gm.fillParentDir(conn, selected);
DirectoryDTO.removeReferences(selected);
removeIds(selected);
expect(Utils.clone(Utils.removeNullOrEmptyObj(selected)))
.to.deep.equal(Utils.clone(Utils.removeNullOrEmptyObj(parent)));
});
it('should save parent directory', async () => {
const gm = new GalleryManagerTest();
const im = new IndexingManagerTest();