1
0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00

fixing search error

This commit is contained in:
Patrik J. Braun 2018-01-30 20:01:16 -05:00
parent 83698541be
commit 2761cbc507
6 changed files with 311 additions and 81 deletions

View File

@ -62,6 +62,7 @@ export class RenderingMWs {
if (err instanceof ErrorDTO) {
if (err.details) {
if (!(req.session.user && req.session.user.role >= UserRoles.Developer)) {
console.log(err);
Logger.warn("Handled error:", err.details.toString() || err.details);
delete (err.details);
} else {

View File

@ -10,6 +10,7 @@ import {DataBaseConfig, DatabaseType} from "../../../common/config/private/IPriv
import {PasswordHelper} from "../PasswordHelper";
import {ProjectPath} from "../../ProjectPath";
import {VersionEntity} from "./enitites/VersionEntity";
import {Logger} from "../../Logger";
export class SQLConnection {
@ -37,7 +38,7 @@ export class SQLConnection {
options.synchronize = false;
// options.logging = "all";
this.connection = await createConnection(options);
await SQLConnection.sync(this.connection);
await SQLConnection.schemeSync(this.connection);
}
return this.connection;
@ -60,12 +61,12 @@ export class SQLConnection {
options.synchronize = false;
// options.logging = "all";
const conn = await createConnection(options);
await SQLConnection.sync(conn);
await SQLConnection.schemeSync(conn);
await conn.close();
return true;
}
private static async sync(connection: Connection) {
private static async schemeSync(connection: Connection) {
let version = null;
try {
version = await connection.getRepository(VersionEntity).findOne();
@ -74,7 +75,7 @@ export class SQLConnection {
if (version && version.version == SQLConnection.VERSION) {
return;
}
Logger.info("Updating database scheme");
if (!version) {
version = new VersionEntity();
}

View File

@ -4,11 +4,22 @@ import {SearchResultDTO} from "../../../common/entities/SearchResultDTO";
import {SQLConnection} from "./SQLConnection";
import {PhotoEntity} from "./enitites/PhotoEntity";
import {DirectoryEntity} from "./enitites/DirectoryEntity";
import {PositionMetaData} from "../../../common/entities/PhotoDTO";
export class SearchManager implements ISearchManager {
async autocomplete(text: string) {
private static autoCompleteItemsUnique(array: Array<AutoCompleteItem>): Array<AutoCompleteItem> {
let a = array.concat();
for (let i = 0; i < a.length; ++i) {
for (let j = i + 1; j < a.length; ++j) {
if (a[i].equals(a[j]))
a.splice(j--, 1);
}
}
return a;
}
async autocomplete(text: string): Promise<Array<AutoCompleteItem>> {
const connection = await SQLConnection.getConnection();
@ -19,11 +30,11 @@ export class SearchManager implements ISearchManager {
(await photoRepository
.createQueryBuilder('photo')
.select('DISTINCT(photo.metadataKeywords)')
.select('DISTINCT(photo.metadata.keywords)')
.where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.limit(5)
.getRawMany())
.map(r => <Array<string>>JSON.parse(r.metadataKeywords))
.map(r => <Array<string>>r.metadataKeywords.split(","))
.forEach(keywords => {
result = result.concat(this.encapsulateAutoComplete(keywords.filter(k => k.toLowerCase().indexOf(text.toLowerCase()) != -1), SearchTypes.keyword));
});
@ -31,18 +42,19 @@ export class SearchManager implements ISearchManager {
(await photoRepository
.createQueryBuilder('photo')
.select('DISTINCT(photo.metadataPositionData)')
.where('photo.metadata.positionData LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.select('photo.metadata.positionData.country as country, photo.metadata.positionData.state as state, photo.metadata.positionData.city as city')
.where('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.groupBy('photo.metadata.positionData.country, photo.metadata.positionData.state, photo.metadata.positionData.city')
.limit(5)
.getRawMany())
.map(r => <PositionMetaData>JSON.parse(r.metadataPositionData))
.filter(pm => !!pm)
.map(pm => <Array<string>>[pm.city || "", pm.country || "", pm.state || ""])
.forEach(positions => {
result = result.concat(this.encapsulateAutoComplete(positions.filter(p => p.toLowerCase().indexOf(text.toLowerCase()) != -1), SearchTypes.position));
});
result = result.concat(this.encapsulateAutoComplete((await photoRepository
.createQueryBuilder('photo')
.select('DISTINCT(photo.name)')
@ -60,13 +72,13 @@ export class SearchManager implements ISearchManager {
.map(r => r.name), SearchTypes.directory));
return this.autoCompleteItemsUnique(result);
return SearchManager.autoCompleteItemsUnique(result);
}
async search(text: string, searchType: SearchTypes) {
async search(text: string, searchType: SearchTypes): Promise<SearchResultDTO> {
const connection = await SQLConnection.getConnection();
let result: SearchResultDTO = <SearchResultDTO>{
const result: SearchResultDTO = <SearchResultDTO>{
searchText: text,
searchType: searchType,
directories: [],
@ -90,27 +102,21 @@ export class SearchManager implements ISearchManager {
}
if (!searchType || searchType === SearchTypes.position) {
query.orWhere('photo.metadata.positionData LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"});
query.orWhere('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"});
}
if (!searchType || searchType === SearchTypes.keyword) {
query.orWhere('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"});
}
let photos = await query
result.photos = await query
.limit(2001)
.getMany();
if (photos) {
for (let i = 0; i < photos.length; i++) {
photos[i].metadata.keywords = <any>JSON.parse(<any>photos[i].metadata.keywords);
photos[i].metadata.cameraData = <any>JSON.parse(<any>photos[i].metadata.cameraData);
photos[i].metadata.positionData = <any>JSON.parse(<any>photos[i].metadata.positionData);
photos[i].metadata.size = <any>JSON.parse(<any>photos[i].metadata.size);
}
result.photos = photos;
if (result.photos.length > 2000) {
result.resultOverflow = true;
}
if (result.photos.length > 2000) {
result.resultOverflow = true;
}
result.directories = await connection
@ -127,68 +133,46 @@ export class SearchManager implements ISearchManager {
return result;
}
async instantSearch(text: string) {
async instantSearch(text: string): Promise<SearchResultDTO> {
const connection = await SQLConnection.getConnection();
let result: SearchResultDTO = <SearchResultDTO>{
searchText: text,
//searchType:undefined, not adding this
directories: [],
photos: [],
resultOverflow: false
};
let photos = await connection
result.photos = await connection
.getRepository(PhotoEntity)
.createQueryBuilder("photo")
.orderBy("photo.metadata.creationDate", "ASC")
.where('photo.metadata.keywords LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.country LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.state LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.metadata.positionData.city LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.orWhere('photo.name LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.innerJoinAndSelect("photo.directory", "directory")
.limit(10)
.getMany();
if (photos) {
for (let i = 0; i < photos.length; i++) {
photos[i].metadata.keywords = <any>JSON.parse(<any>photos[i].metadata.keywords);
photos[i].metadata.cameraData = <any>JSON.parse(<any>photos[i].metadata.cameraData);
photos[i].metadata.positionData = <any>JSON.parse(<any>photos[i].metadata.positionData);
photos[i].metadata.size = <any>JSON.parse(<any>photos[i].metadata.size);
}
result.photos = photos;
}
const directories = await connection
result.directories = await connection
.getRepository(DirectoryEntity)
.createQueryBuilder("dir")
.where('dir.name LIKE :text COLLATE utf8_general_ci', {text: "%" + text + "%"})
.limit(10)
.getMany();
result.directories = directories;
return result;
}
private encapsulateAutoComplete(values: Array<string>, type: SearchTypes) {
private encapsulateAutoComplete(values: Array<string>, type: SearchTypes): Array<AutoCompleteItem> {
let res = [];
values.forEach((value) => {
res.push(new AutoCompleteItem(value, type));
});
return res;
}
private autoCompleteItemsUnique(array: Array<AutoCompleteItem>) {
let a = array.concat();
for (let i = 0; i < a.length; ++i) {
for (let j = i + 1; j < a.length; ++j) {
if (a[i].equals(a[j]))
a.splice(j--, 1);
}
}
return a;
}
}

View File

@ -45,26 +45,26 @@
},
"devDependencies": {
"@agm/core": "^1.0.0-beta.2",
"@angular/animations": "^5.2.1",
"@angular/cli": "1.6.5",
"@angular/common": "~5.2.1",
"@angular/compiler": "~5.2.1",
"@angular/compiler-cli": "^5.2.1",
"@angular/core": "~5.2.1",
"@angular/forms": "~5.2.1",
"@angular/http": "~5.2.1",
"@angular/language-service": "^5.2.1",
"@angular/platform-browser": "~5.2.1",
"@angular/platform-browser-dynamic": "~5.2.1",
"@angular/router": "~5.2.1",
"@angular/animations": "^5.2.2",
"@angular/cli": "1.6.6",
"@angular/common": "~5.2.2",
"@angular/compiler": "~5.2.2",
"@angular/compiler-cli": "^5.2.2",
"@angular/core": "~5.2.2",
"@angular/forms": "~5.2.2",
"@angular/http": "~5.2.2",
"@angular/language-service": "^5.2.2",
"@angular/platform-browser": "~5.2.2",
"@angular/platform-browser-dynamic": "~5.2.2",
"@angular/router": "~5.2.2",
"@types/bcryptjs": "^2.4.1",
"@types/chai": "^4.1.1",
"@types/chai": "^4.1.2",
"@types/cookie-session": "^2.0.34",
"@types/express": "^4.11.0",
"@types/gm": "^1.17.33",
"@types/jasmine": "^2.8.4",
"@types/jasmine": "^2.8.6",
"@types/jimp": "^0.2.28",
"@types/node": "^9.3.0",
"@types/node": "^9.4.0",
"@types/sharp": "^0.17.6",
"@types/winston": "^2.3.7",
"bootstrap": "^3.3.7",
@ -83,7 +83,7 @@
"jw-bootstrap-switch-ng2": "1.0.10",
"karma": "^2.0.0",
"karma-cli": "^1.0.1",
"karma-coverage-istanbul-reporter": "^1.3.3",
"karma-coverage-istanbul-reporter": "^1.4.1",
"karma-jasmine": "^1.1.1",
"karma-jasmine-html-reporter": "^0.2.2",
"karma-phantomjs-launcher": "^1.0.4",
@ -94,11 +94,11 @@
"ng2-cookies": "^1.0.12",
"ng2-slim-loading-bar": "^4.0.0",
"ng2-toastr": "^4.1.2",
"ngx-bootstrap": "^2.0.0",
"ngx-clipboard": "^9.1.2",
"ngx-bootstrap": "^2.0.2",
"ngx-clipboard": "^9.1.3",
"phantomjs-prebuilt": "^2.1.16",
"protractor": "^5.2.2",
"remap-istanbul": "^0.10.0",
"protractor": "^5.3.0",
"remap-istanbul": "^0.10.1",
"rimraf": "^2.6.2",
"run-sequence": "^2.2.1",
"rxjs": "^5.5.6",

View File

@ -9,7 +9,11 @@ import {UserRoles} from "../../../../../common/entities/UserDTO";
import {PasswordHelper} from "../../../../../backend/model/PasswordHelper";
import {DirectoryEntity} from "../../../../../backend/model/sql/enitites/DirectoryEntity";
import {
CameraMetadataEntity, GPSMetadataEntity, ImageSizeEntity, PhotoEntity, PhotoMetadataEntity,
CameraMetadataEntity,
GPSMetadataEntity,
ImageSizeEntity,
PhotoEntity,
PhotoMetadataEntity,
PositionMetaDataEntity
} from "../../../../../backend/model/sql/enitites/PhotoEntity";
@ -189,7 +193,7 @@ describe('Typeorm integration', () => {
expect(photos.length).to.equal(0);
});
it('should open and close connection twice with photo added ', async () => {
it('should open and close connection twice with photo added ', async () => {
let conn = await SQLConnection.getConnection();
const dir = await conn.getRepository(DirectoryEntity).save(getDir());
let dir2 = getDir();

View File

@ -0,0 +1,240 @@
import {expect} from "chai";
import * as fs from "fs";
import * as path from "path";
import {Config} from "../../../../../common/config/private/Config";
import {DatabaseType} from "../../../../../common/config/private/IPrivateConfig";
import {SQLConnection} from "../../../../../backend/model/sql/SQLConnection";
import {
CameraMetadataEntity,
GPSMetadataEntity,
ImageSizeEntity,
PhotoEntity,
PhotoMetadataEntity,
PositionMetaDataEntity
} from "../../../../../backend/model/sql/enitites/PhotoEntity";
import {SearchManager} from "../../../../../backend/model/sql/SearchManager";
import {AutoCompleteItem, SearchTypes} from "../../../../../common/entities/AutoCompleteItem";
import {SearchResultDTO} from "../../../../../common/entities/SearchResultDTO";
import {DirectoryEntity} from "../../../../../backend/model/sql/enitites/DirectoryEntity";
describe('SearchManager', () => {
const tempDir = path.join(__dirname, "../../tmp");
const dbPath = path.join(tempDir, "test.db");
const dir = new DirectoryEntity();
dir.name = "wars dir";
dir.path = ".";
dir.lastModified = Date.now();
dir.lastScanned = null;
const getPhoto = () => {
const sd = new ImageSizeEntity();
sd.height = 200;
sd.width = 200;
const gps = new GPSMetadataEntity();
/* gps.altitude = 1;
gps.latitude = 1;
gps.longitude = 1;*/
const pd = new PositionMetaDataEntity();
/* pd.city = "New York";
pd.country = "Alderan";
pd.state = "Death star";*/
pd.GPSData = gps;
const cd = new CameraMetadataEntity();
/* cd.ISO = 100;
cd.model = "60D";
cd.maker = "Canon";
cd.fStop = 1;
cd.exposure = 1;
cd.focalLength = 1;*/
cd.lens = "Lens";
const m = new PhotoMetadataEntity();
m.keywords = ["apple"];
m.cameraData = cd;
m.positionData = pd;
m.size = sd;
m.creationDate = Date.now();
m.fileSize = 123456789;
const d = new PhotoEntity();
d.name = "test photo.jpg";
d.directory = dir;
d.metadata = m;
return d;
};
let p = getPhoto();
p.metadata.keywords = ["Boba Fett", "star wars", "Anakin", "death star"];
p.metadata.positionData.city = "Mos Eisley";
p.metadata.positionData.country = "Tatooine";
p.name = "sw1";
let p2 = getPhoto();
p2.metadata.keywords = ["Padmé Amidala", "star wars", "Natalie Portman", "death star"];
p2.metadata.positionData.city = "Derem City";
p2.metadata.positionData.state = "Research City";
p2.metadata.positionData.country = "Kamino";
p2.name = "sw2";
const setUpSqlDB = async () => {
if (fs.existsSync(dbPath)) {
fs.unlinkSync(dbPath);
}
if (!fs.existsSync(tempDir)) {
fs.mkdirSync(tempDir);
}
Config.Server.database.type = DatabaseType.sqlite;
Config.Server.database.sqlite.storage = dbPath;
const conn = await SQLConnection.getConnection();
const pr = conn.getRepository(PhotoEntity);
await conn.getRepository(DirectoryEntity).save(p.directory);
await pr.save(p);
await pr.save(p2);
await SQLConnection.close();
};
const teardownUpSqlDB = async () => {
await SQLConnection.close();
if (fs.existsSync(dbPath)) {
fs.unlinkSync(dbPath);
}
if (fs.existsSync(tempDir)) {
fs.rmdirSync(tempDir);
}
};
beforeEach(async () => {
await setUpSqlDB();
});
afterEach(async () => {
await teardownUpSqlDB();
});
it('should get autocomplete', async () => {
let sm = new SearchManager();
const cmp = (a: AutoCompleteItem, b: AutoCompleteItem) => {
return a.text.localeCompare(b.text);
};
expect((await sm.autocomplete("tat"))).to.deep.equal([new AutoCompleteItem("Tatooine", SearchTypes.position)]);
expect((await sm.autocomplete("star"))).to.deep.equal([new AutoCompleteItem("star wars", SearchTypes.keyword),
new AutoCompleteItem("death star", SearchTypes.keyword)]);
expect((await sm.autocomplete("wars"))).to.deep.equal([new AutoCompleteItem("star wars", SearchTypes.keyword),
new AutoCompleteItem("wars dir", SearchTypes.directory)]);
expect((await sm.autocomplete("arch"))).eql([new AutoCompleteItem("Research City", SearchTypes.position)]);
expect((await sm.autocomplete("a")).sort(cmp)).eql([
new AutoCompleteItem("Boba Fett", SearchTypes.keyword),
new AutoCompleteItem("star wars", SearchTypes.keyword),
new AutoCompleteItem("Anakin", SearchTypes.keyword),
new AutoCompleteItem("death star", SearchTypes.keyword),
new AutoCompleteItem("Padmé Amidala", SearchTypes.keyword),
new AutoCompleteItem("Natalie Portman", SearchTypes.keyword),
new AutoCompleteItem("Kamino", SearchTypes.position),
new AutoCompleteItem("Tatooine", SearchTypes.position),
new AutoCompleteItem("wars dir", SearchTypes.directory),
new AutoCompleteItem("Research City", SearchTypes.position)].sort(cmp));
expect((await sm.autocomplete("sw")).sort(cmp)).to.deep.equal([new AutoCompleteItem("sw1", SearchTypes.image),
new AutoCompleteItem("sw2", SearchTypes.image)].sort(cmp));
});
it('should search', async () => {
let sm = new SearchManager();
expect((await sm.search("sw", null))).to.deep.equal(<SearchResultDTO>{
searchText: "sw",
searchType: null,
directories: [],
photos: [p, p2],
resultOverflow: false
});
expect((await sm.search("Tatooine", SearchTypes.position))).to.deep.equal(<SearchResultDTO>{
searchText: "Tatooine",
searchType: SearchTypes.position,
directories: [],
photos: [p],
resultOverflow: false
});
expect((await sm.search("ortm", SearchTypes.keyword))).to.deep.equal(<SearchResultDTO>{
searchText: "ortm",
searchType: SearchTypes.keyword,
directories: [],
photos: [p2],
resultOverflow: false
});
expect((await sm.search("ortm", SearchTypes.keyword))).to.deep.equal(<SearchResultDTO>{
searchText: "ortm",
searchType: SearchTypes.keyword,
directories: [],
photos: [p2],
resultOverflow: false
});
expect((await sm.search("wa", SearchTypes.keyword))).to.deep.equal(<SearchResultDTO>{
searchText: "wa",
searchType: SearchTypes.keyword,
directories: [dir],
photos: [p, p2],
resultOverflow: false
});
});
it('should instant search', async () => {
let sm = new SearchManager();
expect((await sm.instantSearch("sw"))).to.deep.equal({
searchText: "sw",
directories: [],
photos: [p, p2],
resultOverflow: false
});
expect((await sm.instantSearch("Tatooine"))).to.deep.equal({
searchText: "Tatooine",
directories: [],
photos: [p],
resultOverflow: false
});
expect((await sm.instantSearch("ortm"))).to.deep.equal({
searchText: "ortm",
directories: [],
photos: [p2],
resultOverflow: false
});
expect((await sm.instantSearch("ortm"))).to.deep.equal({
searchText: "ortm",
directories: [],
photos: [p2],
resultOverflow: false
});
expect((await sm.instantSearch("wa"))).to.deep.equal({
searchText: "wa",
directories: [dir],
photos: [p, p2],
resultOverflow: false
});
});
});