1
0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00

Implementing search returning with directories too, not just media #309 #304

This commit is contained in:
Patrik J. Braun 2021-05-23 22:26:27 +02:00
parent e41ac5b990
commit 90aff86485
8 changed files with 241 additions and 89 deletions

View File

@ -142,6 +142,7 @@ export class GalleryMWs {
cleanUpMedia(cw.directory.media);
}
if (cw.searchResult) {
cw.searchResult.directories.forEach(d => DirectoryDTOUtils.packDirectory(d));
cleanUpMedia(cw.searchResult.media);
}

View File

@ -117,6 +117,7 @@ export class ObjectManagers {
}
await SQLConnection.close();
this.instance = null;
Logger.debug(LOG_TAG, 'Object manager reset');
}

View File

@ -216,6 +216,30 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
return await query.getOne();
}
public async fillPreviewForSubDir(connection: Connection, dir: DirectoryEntity): Promise<void> {
dir.media = [];
dir.preview = await connection
.getRepository(MediaEntity)
.createQueryBuilder('media')
.innerJoinAndSelect('media.directory', 'directory')
.where('media.directory = :dir', {
dir: dir.id
})
.orderBy('media.metadata.creationDate', 'DESC')
.limit(1)
.getOne();
dir.isPartial = true;
if (dir.preview) {
dir.preview.directory = dir;
dir.preview.readyThumbnails = [];
dir.preview.readyIcon = false;
} else {
await this.fillPreviewFromSubDir(connection, dir);
}
}
protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
const query = connection
.getRepository(DirectoryEntity)
@ -290,27 +314,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
}
if (dir.directories) {
for (const item of dir.directories) {
item.media = [];
item.preview = await connection
.getRepository(MediaEntity)
.createQueryBuilder('media')
.innerJoinAndSelect('media.directory', 'directory')
.where('media.directory = :dir', {
dir: item.id
})
.orderBy('media.metadata.creationDate', 'DESC')
.limit(1)
.getOne();
item.isPartial = true;
if (item.preview) {
item.preview.directory = item;
item.preview.readyThumbnails = [];
item.preview.readyIcon = false;
} else {
await this.fillPreviewFromSubDir(connection, item);
}
await this.fillPreviewForSubDir(connection, item);
}
}
}

View File

@ -1,6 +1,8 @@
import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {IGalleryManager} from '../interfaces/IGalleryManager';
import {DuplicatesDTO} from '../../../../common/entities/DuplicatesDTO';
import {Connection} from 'typeorm';
import {DirectoryEntity} from './enitites/DirectoryEntity';
export interface ISQLGalleryManager extends IGalleryManager {
listDirectory(relativeDirectoryName: string,
@ -18,4 +20,6 @@ export interface ISQLGalleryManager extends IGalleryManager {
getPossibleDuplicates(): Promise<DuplicatesDTO[]>;
selectDirStructure(directory: string): Promise<DirectoryDTO>;
fillPreviewForSubDir(connection: Connection, dir: DirectoryEntity): Promise<void>;
}

View File

@ -31,6 +31,7 @@ import {ObjectManagers} from '../../ObjectManagers';
import {Utils} from '../../../../common/Utils';
import {PhotoDTO} from '../../../../common/entities/PhotoDTO';
import {DatabaseType} from '../../../../common/config/private/PrivateConfig';
import {ISQLGalleryManager} from './IGalleryManager';
export class SearchManager implements ISearchManager {
@ -180,8 +181,26 @@ export class SearchManager implements ISearchManager {
result.resultOverflow = true;
}
const dirQuery = this.filterDirectoryQuery(query);
if (dirQuery !== null) {
result.directories = await connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.where(this.buildWhereQuery(dirQuery, true))
.limit(Config.Client.Search.maxDirectoryResult + 1)
.getMany();
// TODO: implement directory search. Search now only returns with photos and videos
// setting previews
if (result.directories) {
for (const item of result.directories) {
await (ObjectManagers.getInstance().GalleryManager as ISQLGalleryManager)
.fillPreviewForSubDir(connection, item as DirectoryEntity);
}
}
if (result.directories.length > Config.Client.Search.maxDirectoryResult) {
result.resultOverflow = true;
}
}
return result;
}
@ -202,6 +221,43 @@ export class SearchManager implements ISearchManager {
}
/**
* Returns only those part of a query tree that only contains directory related search queries
*/
private filterDirectoryQuery(query: SearchQueryDTO): SearchQueryDTO {
switch (query.type) {
case SearchQueryTypes.AND:
const andRet = {
type: SearchQueryTypes.AND,
list: (query as SearchListQuery).list.map(q => this.filterDirectoryQuery(q))
} as ANDSearchQuery;
// if any of the queries contain non dir query thw whole and query is a non dir query
if (andRet.list.indexOf(null) !== -1) {
return null;
}
return andRet;
case SearchQueryTypes.OR:
const orRet = {
type: SearchQueryTypes.OR,
list: (query as SearchListQuery).list.map(q => this.filterDirectoryQuery(q)).filter(q => q !== null)
} as ORSearchQuery;
if (orRet.list.length === 0) {
return null;
}
return orRet;
case SearchQueryTypes.any_text:
case SearchQueryTypes.directory:
return query;
case SearchQueryTypes.SOME_OF:
throw new Error('"Some of" queries should have been already flattened');
}
// of none of the above, its not a directory search
return null;
}
private async getGPSData(query: SearchQueryDTO): Promise<SearchQueryDTO> {
if ((query as ANDSearchQuery | ORSearchQuery).list) {
for (let i = 0; i < (query as ANDSearchQuery | ORSearchQuery).list.length; ++i) {
@ -216,21 +272,31 @@ export class SearchManager implements ISearchManager {
return query;
}
private buildWhereQuery(query: SearchQueryDTO, paramCounter = {value: 0}): Brackets {
/**
* Builds the SQL Where query from search query
* @param query input search query
* @param paramCounter Helper counter for generating parameterized query
* @param directoryOnly Only builds directory related queries
* @private
*/
private buildWhereQuery(query: SearchQueryDTO, directoryOnly = false, paramCounter = {value: 0}): Brackets {
switch (query.type) {
case SearchQueryTypes.AND:
return new Brackets((q): any => {
(query as ANDSearchQuery).list.forEach((sq): any => q.andWhere(this.buildWhereQuery(sq, paramCounter)));
(query as ANDSearchQuery).list.forEach((sq): any => q.andWhere(this.buildWhereQuery(sq, directoryOnly, paramCounter)));
return q;
});
case SearchQueryTypes.OR:
return new Brackets((q): any => {
(query as ANDSearchQuery).list.forEach((sq): any => q.orWhere(this.buildWhereQuery(sq, paramCounter)));
(query as ANDSearchQuery).list.forEach((sq): any => q.orWhere(this.buildWhereQuery(sq, directoryOnly, paramCounter)));
return q;
});
case SearchQueryTypes.distance:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
/**
* This is a best effort calculation, not fully accurate in order to have higher performance.
* see: https://stackoverflow.com/a/50506609
@ -276,6 +342,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.from_date:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as FromDateSearch).value === 'undefined') {
throw new Error('Invalid search query: Date Query should contain from value');
@ -292,6 +361,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.to_date:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as ToDateSearch).value === 'undefined') {
throw new Error('Invalid search query: Date Query should contain to value');
@ -307,6 +379,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.min_rating:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as MinRatingSearch).value === 'undefined') {
throw new Error('Invalid search query: Rating Query should contain minvalue');
@ -322,6 +397,9 @@ export class SearchManager implements ISearchManager {
return q;
});
case SearchQueryTypes.max_rating:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as MaxRatingSearch).value === 'undefined') {
throw new Error('Invalid search query: Rating Query should contain max value');
@ -339,6 +417,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.min_resolution:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as MinResolutionSearch).value === 'undefined') {
throw new Error('Invalid search query: Resolution Query should contain min value');
@ -356,6 +437,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.max_resolution:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if (typeof (query as MaxResolutionSearch).value === 'undefined') {
throw new Error('Invalid search query: Rating Query should contain min or max value');
@ -372,6 +456,9 @@ export class SearchManager implements ISearchManager {
});
case SearchQueryTypes.orientation:
if (directoryOnly) {
throw new Error('not supported in directoryOnly mode');
}
return new Brackets((q): any => {
if ((query as OrientationSearch).landscape) {
q.where('media.metadata.size.width >= media.metadata.size.height');
@ -391,7 +478,16 @@ export class SearchManager implements ISearchManager {
return new Brackets((q: WhereExpression) => {
const createMatchString = (str: string): string => {
return (query as TextSearch).matchType === TextSearchQueryMatchTypes.exact_match ? str : `%${str}%`;
if ((query as TextSearch).matchType === TextSearchQueryMatchTypes.exact_match) {
return str;
}
// MySQL uses C escape syntax in strings, details:
// https://stackoverflow.com/questions/14926386/how-to-search-for-slash-in-mysql-and-why-escaping-not-required-for-wher
if (Config.Server.Database.type === DatabaseType.mysql) {
/// this reqExp replaces the "\\" to "\\\\\"
return '%' + str.replace(new RegExp('\\\\', 'g'), '\\\\') + '%';
}
return `%${str}%`;
};
const LIKE = (query as TextSearch).negate ? 'NOT LIKE' : 'LIKE';
@ -426,17 +522,17 @@ export class SearchManager implements ISearchManager {
}));
}
if (query.type === SearchQueryTypes.any_text || query.type === SearchQueryTypes.file_name) {
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.file_name) {
q[whereFN](`media.name ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`,
textParam);
}
if (query.type === SearchQueryTypes.any_text || query.type === SearchQueryTypes.caption) {
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.caption) {
q[whereFN](`media.metadata.caption ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`,
textParam);
}
if (query.type === SearchQueryTypes.any_text || query.type === SearchQueryTypes.position) {
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.position) {
q[whereFN](`media.metadata.positionData.country ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`,
textParam)
[whereFN](`media.metadata.positionData.state ${LIKE} :text${paramCounter.value} COLLATE utf8_general_ci`,
@ -475,11 +571,11 @@ export class SearchManager implements ISearchManager {
};
if (query.type === SearchQueryTypes.any_text || query.type === SearchQueryTypes.person) {
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.person) {
matchArrayField('media.metadata.persons');
}
if (query.type === SearchQueryTypes.any_text || query.type === SearchQueryTypes.keyword) {
if ((query.type === SearchQueryTypes.any_text && !directoryOnly) || query.type === SearchQueryTypes.keyword) {
matchArrayField('media.metadata.keywords');
}
return q;

View File

@ -56,11 +56,12 @@
</div>
<app-gallery-navbar [searchResult]="galleryService.content.value.searchResult"></app-gallery-navbar>
<app-gallery-directories class="directories" [directories]="directories"></app-gallery-directories>
<app-gallery-map *ngIf="isPhotoWithLocation && mapEnabled"
[photos]="galleryService.content.value.searchResult.media"
[gpxFiles]="galleryService.content.value.searchResult.metaFile | gpxFiles"></app-gallery-map>
<app-gallery-directories class="directories" [directories]="directories"></app-gallery-directories>
<app-gallery-grid [media]="galleryService.content.value.searchResult.media"
[lightbox]="lightbox"></app-gallery-grid>

View File

@ -139,27 +139,10 @@ export class DBTestHelper {
}
}
private async initSQLite(): Promise<void> {
await this.resetSQLite();
Config.Server.Database.type = DatabaseType.sqlite;
Config.Server.Database.dbFolder = this.tempDir;
ProjectPath.reset();
}
private async initMySQL(): Promise<void> {
Config.Server.Database.type = DatabaseType.mysql;
Config.Server.Database.mysql.database = 'pigallery2_test';
await this.resetMySQL();
}
private async resetSQLite(): Promise<void> {
await ObjectManagers.reset();
// await SQLConnection.close();
await fs.promises.rmdir(this.tempDir, {recursive: true});
}
private async resetMySQL(): Promise<void> {
Config.Server.Database.type = DatabaseType.mysql;
Config.Server.Database.mysql.database = 'pigallery2_test';
@ -167,9 +150,11 @@ export class DBTestHelper {
await conn.query('DROP DATABASE IF EXISTS ' + conn.options.database);
await conn.query('CREATE DATABASE IF NOT EXISTS ' + conn.options.database);
await SQLConnection.close();
await ObjectManagers.InitSQLManagers();
}
private async clearUpMysql(): Promise<void> {
await ObjectManagers.reset();
Config.Server.Database.type = DatabaseType.mysql;
Config.Server.Database.mysql.database = 'pigallery2_test';
const conn = await SQLConnection.getConnection();
@ -177,8 +162,25 @@ export class DBTestHelper {
await SQLConnection.close();
}
private async initSQLite(): Promise<void> {
await this.resetSQLite();
}
private async resetSQLite(): Promise<void> {
Config.Server.Database.type = DatabaseType.sqlite;
Config.Server.Database.dbFolder = this.tempDir;
ProjectPath.reset();
await ObjectManagers.reset();
await fs.promises.rmdir(this.tempDir, {recursive: true});
await ObjectManagers.InitSQLManagers();
}
private async clearUpSQLite(): Promise<void> {
return this.resetSQLite();
Config.Server.Database.type = DatabaseType.sqlite;
Config.Server.Database.dbFolder = this.tempDir;
ProjectPath.reset();
await ObjectManagers.reset();
await fs.promises.rmdir(this.tempDir, {recursive: true});
}
private async clearUpMemory(): Promise<void> {

View File

@ -67,7 +67,6 @@ class GalleryManagerTest extends GalleryManager {
describe('SearchManager', (sqlHelper: DBTestHelper) => {
describe = tmpDescribe;
let dir: DirectoryDTO;
/**
* dir
* |- v
@ -79,6 +78,9 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
* |- p4
*/
let dir: DirectoryDTO;
let subDir: DirectoryDTO;
let subDir2: DirectoryDTO;
let v: VideoDTO;
let p: PhotoDTO;
let p2: PhotoDTO;
@ -88,8 +90,8 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
const setUpTestGallery = async (): Promise<void> => {
const directory: DirectoryDTO = TestHelper.getDirectoryEntry();
const subDir = TestHelper.getDirectoryEntry(directory, 'The Phantom Menace');
const subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi');
subDir = TestHelper.getDirectoryEntry(directory, 'The Phantom Menace');
subDir2 = TestHelper.getDirectoryEntry(directory, 'Return of the Jedi');
p = TestHelper.getPhotoEntry1(directory);
p2 = TestHelper.getPhotoEntry2(directory);
p4 = TestHelper.getPhotoEntry4(subDir2);
@ -98,6 +100,8 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
v = TestHelper.getVideoEntry1(directory);
dir = await DBTestHelper.persistTestDir(directory);
subDir = dir.directories[0];
subDir2 = dir.directories[1];
p = (dir.media.filter(m => m.name === p.name)[0] as any);
p2 = (dir.media.filter(m => m.name === p2.name)[0] as any);
v = (dir.media.filter(m => m.name === v.name)[0] as any);
@ -108,6 +112,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
const setUpSqlDB = async () => {
await sqlHelper.initDB();
await setUpTestGallery();
await ObjectManagers.InitSQLManagers();
};
@ -193,8 +198,27 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
return ret;
};
const searchifyDir = (d: DirectoryDTO): DirectoryDTO => {
const tmpM = d.media;
const tmpD = d.directories;
const tmpP = d.preview;
const tmpMT = d.metaFile;
delete d.directories;
delete d.media;
delete d.preview;
delete d.metaFile;
const ret = Utils.clone(d);
d.directories = tmpD;
d.media = tmpM;
d.preview = tmpP;
d.metaFile = tmpMT;
ret.isPartial = true;
return ret;
};
const removeDir = (result: SearchResultDTO) => {
result.media = result.media.map(m => searchifyMedia(m));
result.directories = result.directories.map(m => searchifyDir(m));
return Utils.clone(result);
};
@ -438,18 +462,18 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
media: [p, p2, pFaceLess, v, p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({text: 'sw', negate: true, type: SearchQueryTypes.any_text} as TextSearch);
expect(Utils.clone(await sm.search(query)))
expect(removeDir(await sm.search(query)))
.to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [dir, subDir, subDir2],
media: [],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({text: 'Boba', type: SearchQueryTypes.any_text} as TextSearch);
@ -460,17 +484,17 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
media: [p],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({text: 'Boba', negate: true, type: SearchQueryTypes.any_text} as TextSearch);
expect(Utils.clone(await sm.search(query)))
expect(removeDir(await sm.search(query)))
.to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [dir, subDir, subDir2],
media: [p2, pFaceLess, p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({text: 'Boba', negate: true, type: SearchQueryTypes.any_text} as TextSearch);
// all should have faces
@ -497,7 +521,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
media: [],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({
text: 'Boba Fett',
@ -512,7 +536,7 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
media: [p],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
});
@ -667,26 +691,26 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
type: SearchQueryTypes.directory
} as TextSearch;
expect(Utils.clone(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
expect(removeDir(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [subDir2],
media: [p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({
text: 'wars dir',
type: SearchQueryTypes.directory
} as TextSearch);
expect(Utils.clone(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
expect(removeDir(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [dir, subDir, subDir2],
media: [p, p2, v, pFaceLess, p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({
text: '/wars dir',
@ -695,46 +719,46 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
} as TextSearch);
expect(Utils.clone(await sm.search({
expect(removeDir(await sm.search({
text: '/wars dir',
matchType: TextSearchQueryMatchTypes.exact_match,
type: SearchQueryTypes.directory
} as TextSearch))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [dir],
media: [p, p2, v],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
query = ({
text: '/wars dir/Return of the Jedi',
// matchType: TextSearchQueryMatchTypes.like,
type: SearchQueryTypes.directory
} as TextSearch);
expect(removeDir(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [subDir2],
media: [p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO), JSON.stringify(query));
query = ({
text: '/wars dir/Return of the Jedi',
matchType: TextSearchQueryMatchTypes.exact_match,
type: SearchQueryTypes.directory
} as TextSearch);
expect(Utils.clone(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
expect(removeDir(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
directories: [subDir2],
media: [p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
query = ({
text: '/wars dir/Return of the Jedi',
matchType: TextSearchQueryMatchTypes.exact_match,
type: SearchQueryTypes.directory
} as TextSearch);
expect(Utils.clone(await sm.search(query))).to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [],
media: [p4],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
} as SearchResultDTO), JSON.stringify(query));
});
@ -1088,6 +1112,25 @@ describe('SearchManager', (sqlHelper: DBTestHelper) => {
});
it('search result should return directory', async () => {
const sm = new SearchManager();
let query = {
text: subDir.name,
type: SearchQueryTypes.any_text
} as TextSearch;
expect(removeDir(await sm.search(query)))
.to.deep.equalInAnyOrder(removeDir({
searchQuery: query,
directories: [subDir],
media: [pFaceLess],
metaFile: [],
resultOverflow: false
} as SearchResultDTO));
});
});