xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2024-11-03 21:04:03 +08:00
Code Actions Projects Activity

improving duplicate finding and UI

Browse Source
This commit is contained in:
Patrik J. Braun 2019-01-19 00:18:20 +01:00
parent 41dc64f805
commit 54781ee667
8 changed files with 200 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
backend/model/sql/GalleryManager.ts
View File

@ -181,50 +181,67 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
.having('count(*)>1'),
'innerMedia',
'media.name=innerMedia.name AND media.metadata.fileSize = innerMedia.fileSize')
.innerJoinAndSelect('media.directory', 'directory').getMany();
.innerJoinAndSelect('media.directory', 'directory')
.orderBy('media.name, media.metadata.fileSize')
.limit(Config.Server.duplicates.listingLimit).getMany();
const duplicateParis: DuplicatesDTO[] = [];
let i = duplicates.length - 1;
while (i >= 0) {
const list = [duplicates[i]];
let j = i - 1;
while (j >= 0 && duplicates[i].name === duplicates[j].name && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) {
list.push(duplicates[j]);
j--;
const processDuplicates = (duplicateList: MediaEntity[],
equalFn: (a: MediaEntity, b: MediaEntity) => boolean,
checkDuplicates: boolean = false) => {
let i = duplicateList.length - 1;
while (i >= 0) {
const list = [duplicateList[i]];
let j = i - 1;
while (j >= 0 && equalFn(duplicateList[i], duplicateList[j])) {
list.push(duplicateList[j]);
j--;
}
i = j;
// if we cut the select list with the SQL LIMIT, filter unpaired media
if (list.length < 2) {
continue;
}
if (checkDuplicates) {
// ad to group if one already existed
const foundDuplicates = duplicateParis.find(dp =>
!!dp.media.find(m =>
!!list.find(lm => lm.id === m.id)));
if (foundDuplicates) {
list.forEach(lm => {
if (!!foundDuplicates.media.find(m => m.id === lm.id)) {
return;
}
foundDuplicates.media.push(lm);
});
continue;
}
}
duplicateParis.push({media: list});
}
i = j;
duplicateParis.push({media: list});
}
};
processDuplicates(duplicates,
(a, b) => a.name === b.name &&
a.metadata.fileSize === b.metadata.fileSize);
duplicates = await mediaRepository.createQueryBuilder('media')
.innerJoin(query => query.from(MediaEntity, 'innerMedia')
.select(['innerMedia.metadata.creationDate as creationDate', 'innerMedia.metadata.fileSize as fileSize', 'count(*)'])
.groupBy('innerMedia.name, innerMedia.metadata.fileSize')
.groupBy('innerMedia.metadata.creationDate, innerMedia.metadata.fileSize')
.having('count(*)>1'),
'innerMedia',
'media.metadata.creationDate=innerMedia.creationDate AND media.metadata.fileSize = innerMedia.fileSize')
.innerJoinAndSelect('media.directory', 'directory').getMany();
i = duplicates.length - 1;
while (i >= 0) {
const list = [duplicates[i]];
let j = i - 1;
while (j >= 0 && duplicates[i].metadata.creationDate === duplicates[j].metadata.creationDate && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) {
list.push(duplicates[j]);
j--;
}
i = j;
if (list.filter(paired =>
!!duplicateParis.find(dp =>
!!dp.media.find(m =>
m.id === paired.id))).length === list.length) {
continue;
}
duplicateParis.push({media: list});
}
.innerJoinAndSelect('media.directory', 'directory')
.orderBy('media.metadata.creationDate, media.metadata.fileSize')
.limit(Config.Server.duplicates.listingLimit).getMany();
processDuplicates(duplicates,
(a, b) => a.metadata.creationDate === b.metadata.creationDate &&
a.metadata.fileSize === b.metadata.fileSize, true);
return duplicateParis;

5
common/config/private/IPrivateConfig.ts
View File

@ -56,6 +56,10 @@ export interface ThreadingConfig {
thumbnailThreads: number;
}
export interface DuplicatesConfig {
listingLimit: number; // maximum number of duplicates to list
}
export interface ServerConfig {
port: number;
host: string;
@ -67,6 +71,7 @@ export interface ServerConfig {
sessionTimeout: number;
indexing: IndexingConfig;
photoMetadataSize: number;
duplicates: DuplicatesConfig;
}
export interface IPrivateConfig {

3
common/config/private/PrivateConfigClass.ts
View File

@ -45,6 +45,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon
folderPreviewSize: 2,
cachedFolderTimeout: 1000 * 60 * 60,
reIndexingSensitivity: ReIndexingSensitivity.low
},
duplicates: {
listingLimit: 1000
}
};
private ConfigLoader: any;

43
frontend/app/duplicates/duplicates.component.html
View File

@ -1,31 +1,34 @@
<app-frame>
<div body class="container">
<ng-template [ngIf]="_duplicateService.duplicates.value">
<div *ngFor="let pairs of _duplicateService.duplicates.value" class="card">
<div class="card-body">
<div *ngFor="let media of pairs.media"
class="row"
[routerLink]="['/gallery', getDirectoryPath(media)]"
[queryParams]="queryService.getParams()">
<app-duplicates-photo class="col-1" [media]="media"></app-duplicates-photo>
<div class="col-5">
/{{getDirectoryPath(media)}}/<span class="same-data">{{media.name}}</span>
</div>
<div class="col-2">
<span class="same-data">{{media.metadata.fileSize | fileSize}}</span>
</div>
<div class="col-2" [title]="media.metadata.creationDate">
{{media.metadata.creationDate | date}}
</div>
<div class="col-2">
{{media.metadata.size.width}}x{{media.metadata.size.height}}
<ng-template [ngIf]="renderedDirGroups">
<div *ngFor="let group of renderedDirGroups">
<strong>{{group.name}}</strong>
<div *ngFor="let pairs of group.duplicates" class="card">
<div class="card-body">
<div *ngFor="let media of pairs.media"
class="row"
[routerLink]="['/gallery', getDirectoryPath(media.directory)]"
[queryParams]="queryService.getParams()">
<app-duplicates-photo class="col-1" [media]="media"></app-duplicates-photo>
<div class="col-5">
/{{getDirectoryPath(media.directory)}}/<span class="same-data">{{media.name}}</span>
</div>
<div class="col-2">
<span class="same-data">{{media.metadata.fileSize | fileSize}}</span>
</div>
<div class="col-2" [title]="media.metadata.creationDate">
{{media.metadata.creationDate | date}}
</div>
<div class="col-2">
{{media.metadata.size.width}}x{{media.metadata.size.height}}
</div>
</div>
</div>
</div>
</div>
</ng-template>
<ng-template [ngIf]="!_duplicateService.duplicates.value">
<ng-template [ngIf]="!renderedDirGroups">
loading
</ng-template>
</div>

109
frontend/app/duplicates/duplicates.component.ts
View File

@ -1,22 +1,121 @@
import {Component} from '@angular/core';
import {Component, HostListener, OnDestroy} from '@angular/core';
import {DuplicateService} from './duplicates.service';
import {MediaDTO} from '../../../common/entities/MediaDTO';
import {Utils} from '../../../common/Utils';
import {QueryService} from '../model/query.service';
import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO';
import {DirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {Subscription} from 'rxjs';
import {Config} from '../../../common/config/public/Config';
import {PageHelper} from '../model/page.helper';
interface GroupedDuplicate {
name: string;
duplicates: DuplicatesDTO[];
}
@Component({
selector: 'app-duplicate',
templateUrl: './duplicates.component.html',
styleUrls: ['./duplicates.component.css']
})
export class DuplicateComponent {
export class DuplicateComponent implements OnDestroy {
directoryGroups: GroupedDuplicate[] = null;
renderedDirGroups: GroupedDuplicate[] = null;
renderedIndex = {
group: -1,
pairs: 0
};
subscription: Subscription;
renderTimer: number = null;
constructor(public _duplicateService: DuplicateService,
public queryService: QueryService) {
this._duplicateService.getDuplicates().catch(console.error);
this.subscription = this._duplicateService.duplicates.subscribe((duplicates: DuplicatesDTO[]) => {
this.directoryGroups = [];
this.renderedIndex = {group: -1, pairs: 0};
this.renderedDirGroups = [];
if (duplicates === null) {
return;
}
const getMostFrequentDir = (dupls: DuplicatesDTO[]) => {
if (dupls.length === 0) {
return null;
}
const dirFrequency: { [key: number]: { count: number, dir: DirectoryDTO } } = {};
dupls.forEach(d => d.media.forEach(m => {
dirFrequency[m.directory.id] = dirFrequency[m.directory.id] || {dir: m.directory, count: 0};
dirFrequency[m.directory.id].count++;
}));
let max: { count: number, dir: DirectoryDTO } = {count: -1, dir: null};
for (const freq of Object.values(dirFrequency)) {
if (max.count <= freq.count) {
max = freq;
}
}
return max.dir;
};
while (duplicates.length > 0) {
const dir = getMostFrequentDir(duplicates);
const group = duplicates.filter(d => d.media.find(m => m.directory.id === dir.id));
duplicates = duplicates.filter(d => !d.media.find(m => m.directory.id === dir.id));
this.directoryGroups.push({name: this.getDirectoryPath(dir) + ' (' + group.length + ')', duplicates: group});
}
this.renderMore();
});
}
getDirectoryPath(media: MediaDTO) {
return Utils.concatUrls(media.directory.path, media.directory.name);
ngOnDestroy(): void {
if (this.subscription) {
this.subscription.unsubscribe();
this.subscription = null;
}
}
getDirectoryPath(directory: DirectoryDTO) {
return Utils.concatUrls(directory.path, directory.name);
}
renderMore = () => {
if (this.renderTimer !== null) {
clearTimeout(this.renderTimer);
this.renderTimer = null;
}
if (this.renderedIndex.group === this.directoryGroups.length - 1 &&
this.renderedIndex.pairs >=
this.directoryGroups[this.renderedIndex.group].duplicates.length) {
return;
}
if (this.shouldRenderMore()) {
if (this.renderedDirGroups.length === 0 ||
this.renderedIndex.pairs >=
this.directoryGroups[this.renderedIndex.group].duplicates.length) {
this.renderedDirGroups.push({
name: this.directoryGroups[++this.renderedIndex.group].name,
duplicates: []
});
this.renderedIndex.pairs = 0;
}
this.renderedDirGroups[this.renderedDirGroups.length - 1].duplicates
.push(this.directoryGroups[this.renderedIndex.group].duplicates[this.renderedIndex.pairs++]);
this.renderTimer = window.setTimeout(this.renderMore, 0);
}
};
@HostListener('window:scroll')
onScroll() {
this.renderMore();
}
private shouldRenderMore(): boolean {
return Config.Client.Other.enableOnScrollRendering === false ||
PageHelper.ScrollY >= PageHelper.MaxScrollY * 0.7
|| (document.body.clientHeight) * 0.85 < window.innerHeight;
}
}

1
frontend/app/gallery/grid/grid.gallery.component.ts
View File

@ -292,7 +292,6 @@ export class GalleryGridComponent implements OnChanges, OnInit, AfterViewInit, O
return Config.Client.Other.enableOnScrollRendering === false ||
PageHelper.ScrollY >= (document.body.clientHeight + offset - window.innerHeight) * 0.7
|| (document.body.clientHeight + offset) * 0.85 < window.innerHeight;
}

14
frontend/app/model/page.helper.ts
View File

@ -11,14 +11,20 @@ export class PageHelper {
return this.supportPageOffset ? window.pageYOffset : this.isCSS1Compat ? document.documentElement.scrollTop : document.body.scrollTop;
}
public static get ScrollX(): number {
return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
}
public static set ScrollY(value: number) {
window.scrollTo(this.ScrollX, value);
}
public static get MaxScrollY(): number {
return Math.max(document.body.scrollHeight, document.body.offsetHeight,
document.documentElement.clientHeight, document.documentElement.scrollHeight,
document.documentElement.offsetHeight) - window.innerHeight;
}
public static get ScrollX(): number {
return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft;
}
public static showScrollY() {
PageHelper.body.style.overflowY = 'scroll';
}

9
frontend/app/settings/settings.service.ts
View File

@ -71,8 +71,8 @@ export class SettingsService {
updateTimeout: 2000
},
imagesFolder: '',
port: 80,
host: '0.0.0.0',
port: 80,
host: '0.0.0.0',
thumbnail: {
folder: '',
qualityPriority: true,
@ -88,7 +88,10 @@ export class SettingsService {
folderPreviewSize: 0,
reIndexingSensitivity: ReIndexingSensitivity.medium
},
photoMetadataSize: 512 * 1024
photoMetadataSize: 512 * 1024,
duplicates: {
listingLimit: 1000
}
}
});
}