From 54781ee667d13f9a9deeed216ad97c047e3ccba8 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sat, 19 Jan 2019 00:18:20 +0100 Subject: [PATCH] improving duplicate finding and UI --- backend/model/sql/GalleryManager.ts | 81 ++++++++----- common/config/private/IPrivateConfig.ts | 5 + common/config/private/PrivateConfigClass.ts | 3 + .../app/duplicates/duplicates.component.html | 43 +++---- .../app/duplicates/duplicates.component.ts | 109 +++++++++++++++++- .../gallery/grid/grid.gallery.component.ts | 1 - frontend/app/model/page.helper.ts | 14 ++- frontend/app/settings/settings.service.ts | 9 +- 8 files changed, 200 insertions(+), 65 deletions(-) diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index f3e9bb64..5adc3614 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -181,50 +181,67 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { .having('count(*)>1'), 'innerMedia', 'media.name=innerMedia.name AND media.metadata.fileSize = innerMedia.fileSize') - .innerJoinAndSelect('media.directory', 'directory').getMany(); + .innerJoinAndSelect('media.directory', 'directory') + .orderBy('media.name, media.metadata.fileSize') + .limit(Config.Server.duplicates.listingLimit).getMany(); + const duplicateParis: DuplicatesDTO[] = []; - let i = duplicates.length - 1; - while (i >= 0) { - const list = [duplicates[i]]; - let j = i - 1; - while (j >= 0 && duplicates[i].name === duplicates[j].name && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) { - list.push(duplicates[j]); - j--; + const processDuplicates = (duplicateList: MediaEntity[], + equalFn: (a: MediaEntity, b: MediaEntity) => boolean, + checkDuplicates: boolean = false) => { + let i = duplicateList.length - 1; + while (i >= 0) { + const list = [duplicateList[i]]; + let j = i - 1; + while (j >= 0 && equalFn(duplicateList[i], duplicateList[j])) { + list.push(duplicateList[j]); + j--; + } + i = j; + // if we cut the select list with the SQL LIMIT, filter unpaired media + if (list.length < 2) { + continue; + } + if (checkDuplicates) { + // ad to group if one already existed + const foundDuplicates = duplicateParis.find(dp => + !!dp.media.find(m => + !!list.find(lm => lm.id === m.id))); + if (foundDuplicates) { + list.forEach(lm => { + if (!!foundDuplicates.media.find(m => m.id === lm.id)) { + return; + } + foundDuplicates.media.push(lm); + }); + continue; + } + } + + duplicateParis.push({media: list}); } - i = j; - duplicateParis.push({media: list}); - } + }; + + processDuplicates(duplicates, + (a, b) => a.name === b.name && + a.metadata.fileSize === b.metadata.fileSize); duplicates = await mediaRepository.createQueryBuilder('media') .innerJoin(query => query.from(MediaEntity, 'innerMedia') .select(['innerMedia.metadata.creationDate as creationDate', 'innerMedia.metadata.fileSize as fileSize', 'count(*)']) - .groupBy('innerMedia.name, innerMedia.metadata.fileSize') + .groupBy('innerMedia.metadata.creationDate, innerMedia.metadata.fileSize') .having('count(*)>1'), 'innerMedia', 'media.metadata.creationDate=innerMedia.creationDate AND media.metadata.fileSize = innerMedia.fileSize') - .innerJoinAndSelect('media.directory', 'directory').getMany(); - - i = duplicates.length - 1; - while (i >= 0) { - const list = [duplicates[i]]; - let j = i - 1; - while (j >= 0 && duplicates[i].metadata.creationDate === duplicates[j].metadata.creationDate && duplicates[i].metadata.fileSize === duplicates[j].metadata.fileSize) { - list.push(duplicates[j]); - j--; - } - i = j; - if (list.filter(paired => - !!duplicateParis.find(dp => - !!dp.media.find(m => - m.id === paired.id))).length === list.length) { - continue; - } - - duplicateParis.push({media: list}); - } + .innerJoinAndSelect('media.directory', 'directory') + .orderBy('media.metadata.creationDate, media.metadata.fileSize') + .limit(Config.Server.duplicates.listingLimit).getMany(); + processDuplicates(duplicates, + (a, b) => a.metadata.creationDate === b.metadata.creationDate && + a.metadata.fileSize === b.metadata.fileSize, true); return duplicateParis; diff --git a/common/config/private/IPrivateConfig.ts b/common/config/private/IPrivateConfig.ts index c1356693..7c233609 100644 --- a/common/config/private/IPrivateConfig.ts +++ b/common/config/private/IPrivateConfig.ts @@ -56,6 +56,10 @@ export interface ThreadingConfig { thumbnailThreads: number; } +export interface DuplicatesConfig { + listingLimit: number; // maximum number of duplicates to list +} + export interface ServerConfig { port: number; host: string; @@ -67,6 +71,7 @@ export interface ServerConfig { sessionTimeout: number; indexing: IndexingConfig; photoMetadataSize: number; + duplicates: DuplicatesConfig; } export interface IPrivateConfig { diff --git a/common/config/private/PrivateConfigClass.ts b/common/config/private/PrivateConfigClass.ts index ca3e908a..585daa0a 100644 --- a/common/config/private/PrivateConfigClass.ts +++ b/common/config/private/PrivateConfigClass.ts @@ -45,6 +45,9 @@ export class PrivateConfigClass extends PublicConfigClass implements IPrivateCon folderPreviewSize: 2, cachedFolderTimeout: 1000 * 60 * 60, reIndexingSensitivity: ReIndexingSensitivity.low + }, + duplicates: { + listingLimit: 1000 } }; private ConfigLoader: any; diff --git a/frontend/app/duplicates/duplicates.component.html b/frontend/app/duplicates/duplicates.component.html index 7059a32f..667cb643 100644 --- a/frontend/app/duplicates/duplicates.component.html +++ b/frontend/app/duplicates/duplicates.component.html @@ -1,31 +1,34 @@
- -
-
-
- -
- /{{getDirectoryPath(media)}}/{{media.name}} -
-
- {{media.metadata.fileSize | fileSize}} -
-
- {{media.metadata.creationDate | date}} -
-
- {{media.metadata.size.width}}x{{media.metadata.size.height}} + +
+ {{group.name}} +
+
+
+ +
+ /{{getDirectoryPath(media.directory)}}/{{media.name}} +
+
+ {{media.metadata.fileSize | fileSize}} +
+
+ {{media.metadata.creationDate | date}} +
+
+ {{media.metadata.size.width}}x{{media.metadata.size.height}} +
- + loading
diff --git a/frontend/app/duplicates/duplicates.component.ts b/frontend/app/duplicates/duplicates.component.ts index 8e7018c0..91b7519e 100644 --- a/frontend/app/duplicates/duplicates.component.ts +++ b/frontend/app/duplicates/duplicates.component.ts @@ -1,22 +1,121 @@ -import {Component} from '@angular/core'; +import {Component, HostListener, OnDestroy} from '@angular/core'; import {DuplicateService} from './duplicates.service'; -import {MediaDTO} from '../../../common/entities/MediaDTO'; import {Utils} from '../../../common/Utils'; import {QueryService} from '../model/query.service'; +import {DuplicatesDTO} from '../../../common/entities/DuplicatesDTO'; +import {DirectoryDTO} from '../../../common/entities/DirectoryDTO'; +import {Subscription} from 'rxjs'; +import {Config} from '../../../common/config/public/Config'; +import {PageHelper} from '../model/page.helper'; + +interface GroupedDuplicate { + name: string; + duplicates: DuplicatesDTO[]; +} @Component({ selector: 'app-duplicate', templateUrl: './duplicates.component.html', styleUrls: ['./duplicates.component.css'] }) -export class DuplicateComponent { +export class DuplicateComponent implements OnDestroy { + + directoryGroups: GroupedDuplicate[] = null; + renderedDirGroups: GroupedDuplicate[] = null; + renderedIndex = { + group: -1, + pairs: 0 + }; + subscription: Subscription; + renderTimer: number = null; + constructor(public _duplicateService: DuplicateService, public queryService: QueryService) { this._duplicateService.getDuplicates().catch(console.error); + this.subscription = this._duplicateService.duplicates.subscribe((duplicates: DuplicatesDTO[]) => { + this.directoryGroups = []; + this.renderedIndex = {group: -1, pairs: 0}; + this.renderedDirGroups = []; + if (duplicates === null) { + return; + } + const getMostFrequentDir = (dupls: DuplicatesDTO[]) => { + if (dupls.length === 0) { + return null; + } + const dirFrequency: { [key: number]: { count: number, dir: DirectoryDTO } } = {}; + dupls.forEach(d => d.media.forEach(m => { + dirFrequency[m.directory.id] = dirFrequency[m.directory.id] || {dir: m.directory, count: 0}; + dirFrequency[m.directory.id].count++; + })); + let max: { count: number, dir: DirectoryDTO } = {count: -1, dir: null}; + for (const freq of Object.values(dirFrequency)) { + if (max.count <= freq.count) { + max = freq; + } + } + return max.dir; + }; + + while (duplicates.length > 0) { + const dir = getMostFrequentDir(duplicates); + const group = duplicates.filter(d => d.media.find(m => m.directory.id === dir.id)); + duplicates = duplicates.filter(d => !d.media.find(m => m.directory.id === dir.id)); + this.directoryGroups.push({name: this.getDirectoryPath(dir) + ' (' + group.length + ')', duplicates: group}); + } + this.renderMore(); + }); } - getDirectoryPath(media: MediaDTO) { - return Utils.concatUrls(media.directory.path, media.directory.name); + ngOnDestroy(): void { + if (this.subscription) { + this.subscription.unsubscribe(); + this.subscription = null; + } + } + + getDirectoryPath(directory: DirectoryDTO) { + return Utils.concatUrls(directory.path, directory.name); + } + + renderMore = () => { + if (this.renderTimer !== null) { + clearTimeout(this.renderTimer); + this.renderTimer = null; + } + + if (this.renderedIndex.group === this.directoryGroups.length - 1 && + this.renderedIndex.pairs >= + this.directoryGroups[this.renderedIndex.group].duplicates.length) { + return; + } + if (this.shouldRenderMore()) { + if (this.renderedDirGroups.length === 0 || + this.renderedIndex.pairs >= + this.directoryGroups[this.renderedIndex.group].duplicates.length) { + this.renderedDirGroups.push({ + name: this.directoryGroups[++this.renderedIndex.group].name, + duplicates: [] + }); + this.renderedIndex.pairs = 0; + } + this.renderedDirGroups[this.renderedDirGroups.length - 1].duplicates + .push(this.directoryGroups[this.renderedIndex.group].duplicates[this.renderedIndex.pairs++]); + + this.renderTimer = window.setTimeout(this.renderMore, 0); + } + }; + + + @HostListener('window:scroll') + onScroll() { + this.renderMore(); + } + + private shouldRenderMore(): boolean { + return Config.Client.Other.enableOnScrollRendering === false || + PageHelper.ScrollY >= PageHelper.MaxScrollY * 0.7 + || (document.body.clientHeight) * 0.85 < window.innerHeight; } } diff --git a/frontend/app/gallery/grid/grid.gallery.component.ts b/frontend/app/gallery/grid/grid.gallery.component.ts index 084e868d..7c91882a 100644 --- a/frontend/app/gallery/grid/grid.gallery.component.ts +++ b/frontend/app/gallery/grid/grid.gallery.component.ts @@ -292,7 +292,6 @@ export class GalleryGridComponent implements OnChanges, OnInit, AfterViewInit, O return Config.Client.Other.enableOnScrollRendering === false || PageHelper.ScrollY >= (document.body.clientHeight + offset - window.innerHeight) * 0.7 || (document.body.clientHeight + offset) * 0.85 < window.innerHeight; - } diff --git a/frontend/app/model/page.helper.ts b/frontend/app/model/page.helper.ts index 1decc2ac..9f3da064 100644 --- a/frontend/app/model/page.helper.ts +++ b/frontend/app/model/page.helper.ts @@ -11,14 +11,20 @@ export class PageHelper { return this.supportPageOffset ? window.pageYOffset : this.isCSS1Compat ? document.documentElement.scrollTop : document.body.scrollTop; } - public static get ScrollX(): number { - return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft; - } - public static set ScrollY(value: number) { window.scrollTo(this.ScrollX, value); } + public static get MaxScrollY(): number { + return Math.max(document.body.scrollHeight, document.body.offsetHeight, + document.documentElement.clientHeight, document.documentElement.scrollHeight, + document.documentElement.offsetHeight) - window.innerHeight; + } + + public static get ScrollX(): number { + return this.supportPageOffset ? window.pageXOffset : this.isCSS1Compat ? document.documentElement.scrollLeft : document.body.scrollLeft; + } + public static showScrollY() { PageHelper.body.style.overflowY = 'scroll'; } diff --git a/frontend/app/settings/settings.service.ts b/frontend/app/settings/settings.service.ts index a2e85cfa..7cecaf16 100644 --- a/frontend/app/settings/settings.service.ts +++ b/frontend/app/settings/settings.service.ts @@ -71,8 +71,8 @@ export class SettingsService { updateTimeout: 2000 }, imagesFolder: '', - port: 80, - host: '0.0.0.0', + port: 80, + host: '0.0.0.0', thumbnail: { folder: '', qualityPriority: true, @@ -88,7 +88,10 @@ export class SettingsService { folderPreviewSize: 0, reIndexingSensitivity: ReIndexingSensitivity.medium }, - photoMetadataSize: 512 * 1024 + photoMetadataSize: 512 * 1024, + duplicates: { + listingLimit: 1000 + } } }); }