xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00
Code Actions Projects Activity

Implementing changes only indexing for indexing job. Making that to default

Browse Source
This commit is contained in:
Patrik J. Braun 2021-05-12 13:56:10 +02:00
parent 392f111cfc
commit 1d3f9df8f4
20 changed files with 282 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/BenchmarkRunner.ts
View File

@ -313,7 +313,7 @@ export class BenchmarkRunner {
onProgressUpdate: (progress: JobProgress): void => {
}
};
indexingJob.start().catch(console.error);
indexingJob.start({indexChangesOnly: false}).catch(console.error);
} catch (e) {
console.error(e);
reject(e);

18
src/backend/model/database/sql/GalleryManager.ts
View File

@ -198,6 +198,24 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager {
}
/**
* Returns with the directories only, does not include media or metafiles
*/
public async selectDirStructure(relativeDirectoryName: string): Promise<DirectoryEntity> {
const directoryPath = GalleryManager.parseRelativeDirePath(relativeDirectoryName);
const connection = await SQLConnection.getConnection();
const query = connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.where('directory.name = :name AND directory.path = :path', {
name: directoryPath.name,
path: directoryPath.parent
})
.leftJoinAndSelect('directory.directories', 'directories');
return await query.getOne();
}
protected async selectParentDir(connection: Connection, directoryName: string, directoryParent: string): Promise<DirectoryEntity> {
const query = connection
.getRepository(DirectoryEntity)

2
src/backend/model/database/sql/IGalleryManager.ts
View File

@ -16,4 +16,6 @@ export interface ISQLGalleryManager extends IGalleryManager {
countMediaSize(): Promise<number>;
getPossibleDuplicates(): Promise<DuplicatesDTO[]>;
selectDirStructure(directory: string): Promise<DirectoryDTO>;
}

5
src/backend/model/database/sql/IndexingManager.ts
View File

@ -57,12 +57,11 @@ export class IndexingManager implements IIndexingManager {
async resetDB(): Promise<void> {
Logger.info(LOG_TAG, 'Resetting DB');
const connection = await SQLConnection.getConnection();
return connection
await connection
.getRepository(DirectoryEntity)
.createQueryBuilder('directory')
.delete()
.execute().then((): void => {
});
.execute();
}
// Todo fix it, once typeorm support connection pools for sqlite

37
src/backend/model/database/sql/enitites/PhotoEntity.ts
View File

@ -8,12 +8,20 @@ export class CameraMetadataEntity implements CameraMetadata {
@Column('int', {nullable: true, unsigned: true})
ISO: number;
@Column('text', {nullable: true})
@Column(columnCharsetCS)
@Column({
type: 'text', nullable: true,
charset: columnCharsetCS.charset,
collation: columnCharsetCS.collation
})
model: string;
@Column('text', {nullable: true})
@Column(columnCharsetCS)
@Column({
type: 'text', nullable: true,
charset: columnCharsetCS.charset,
collation: columnCharsetCS.collation
})
make: string;
@Column('float', {nullable: true})
@ -47,18 +55,27 @@ export class PositionMetaDataEntity implements PositionMetaData {
GPSData: GPSMetadataEntity;
@Index()
@Column('text', {nullable: true})
@Column(columnCharsetCS)
@Column({
type: 'text', nullable: true,
charset: columnCharsetCS.charset,
collation: columnCharsetCS.collation
})
country: string;
@Index()
@Column('text', {nullable: true})
@Column(columnCharsetCS)
@Column({
type: 'text', nullable: true,
charset: columnCharsetCS.charset,
collation: columnCharsetCS.collation
})
state: string;
@Index()
@Column('text', {nullable: true})
@Column(columnCharsetCS)
@Column({
type: 'text', nullable: true,
charset: columnCharsetCS.charset,
collation: columnCharsetCS.collation
})
city: string;
}

48
src/backend/model/jobs/jobs/IndexingJob.ts
View File

@ -1,16 +1,29 @@
import {ObjectManagers} from '../../ObjectManagers';
import * as path from 'path';
import * as fs from 'fs';
import {Config} from '../../../../common/config/private/Config';
import {Job} from './Job';
import {ConfigTemplateEntry, DefaultsJobs} from '../../../../common/entities/job/JobDTO';
import {JobProgressStates} from '../../../../common/entities/job/JobProgressDTO';
import {DatabaseType, ServerConfig} from '../../../../common/config/private/PrivateConfig';
import {DatabaseType} from '../../../../common/config/private/PrivateConfig';
import {DiskMangerWorker} from '../../threading/DiskMangerWorker';
import {ProjectPath} from '../../../ProjectPath';
import {backendTexts} from '../../../../common/BackendTexts';
import {DirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import {ISQLGalleryManager} from '../../database/sql/IGalleryManager';
import {Logger} from '../../../Logger';
export class IndexingJob extends Job {
export class IndexingJob<S extends { indexChangesOnly: boolean } = { indexChangesOnly: boolean }> extends Job<S> {
public readonly Name = DefaultsJobs[DefaultsJobs.Indexing];
directoriesToIndex: string[] = [];
public readonly ConfigTemplate: ConfigTemplateEntry[] = null;
public readonly ConfigTemplate: ConfigTemplateEntry[] = [{
id: 'indexChangesOnly',
type: 'boolean',
name: backendTexts.indexChangesOnly.name,
description: backendTexts.indexChangesOnly.description,
defaultValue: true
}];
public get Supported(): boolean {
return Config.Server.Database.type !== DatabaseType.memory;
@ -26,16 +39,39 @@ export class IndexingJob extends Job {
if (ObjectManagers.getInstance().IndexingManager.IsSavingInProgress) {
await ObjectManagers.getInstance().IndexingManager.SavingReady;
}
this.Progress.Left = 0;
return false;
}
const directory = this.directoriesToIndex.shift();
this.Progress.log(directory);
this.Progress.Left = this.directoriesToIndex.length;
const scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory);
let scanned: DirectoryDTO;
let dirChanged = true;
// check if the folder got modified if only changes need to be indexed
if (this.config.indexChangesOnly) {
const stat = fs.statSync(path.join(ProjectPath.ImageFolder, directory));
const lastModified = DiskMangerWorker.calcLastModified(stat);
scanned = await (ObjectManagers.getInstance().GalleryManager as ISQLGalleryManager).selectDirStructure(directory);
// If not modified and it was scanned before, dir is up-to-date
if (scanned && scanned.lastModified === lastModified && scanned.lastScanned != null) {
dirChanged = false;
}
}
// reindex
if (dirChanged || !this.config.indexChangesOnly) {
this.Progress.log('Indexing: ' + directory);
this.Progress.Processed++;
scanned = await ObjectManagers.getInstance().IndexingManager.indexDirectory(directory);
} else {
this.Progress.log('Skipped: ' + directory);
this.Progress.Skipped++;
Logger.silly('Skipping reindexing, no change for: ' + directory);
}
if (this.Progress.State !== JobProgressStates.running) {
return false;
}
this.Progress.Processed++;
for (const item of scanned.directories) {
this.directoriesToIndex.push(path.join(item.path, item.name));
}

1
src/backend/model/jobs/jobs/TempFolderCleaningJob.ts
View File

@ -105,6 +105,7 @@ export class TempFolderCleaningJob extends Job {
protected async step(): Promise<boolean> {
if (this.directoryQueue.length === 0) {
this.Progress.Left = 0;
return false;
}
if (this.tempRootCleaned === false) {

3
src/common/BackendTexts.ts
View File

@ -1,5 +1,6 @@
export type backendText = number;
export const backendTexts = {
indexedFilesOnly: {name: 10, description: 12},
sizeToGenerate: {name: 20, description: 22}
sizeToGenerate: {name: 20, description: 22},
indexChangesOnly: {name: 30, description: 32}
};

2
src/common/config/private/PrivateConfig.ts
View File

@ -221,7 +221,7 @@ export class ServerJobConfig {
new JobScheduleConfig(DefaultsJobs[DefaultsJobs.Indexing],
DefaultsJobs[DefaultsJobs.Indexing],
false,
new NeverJobTrigger(), {}
new NeverJobTrigger(), {indexChangesOnly: true}
),
new JobScheduleConfig(DefaultsJobs[DefaultsJobs['Thumbnail Generation']],
DefaultsJobs[DefaultsJobs['Thumbnail Generation']],

4
src/frontend/app/model/backendtext.service.ts
View File

@ -19,6 +19,10 @@ export class BackendtextService {
return $localize`Indexed only`;
case backendTexts.indexedFilesOnly.description:
return $localize`Only checks indexed files.`;
case backendTexts.indexChangesOnly.name:
return $localize`Index changes only`;
case backendTexts.indexChangesOnly.description:
return $localize`Only indexes a folder if it got changed.`;
default:
return null;
}

2
src/frontend/app/ui/settings/indexing/indexing.settings.component.ts
View File

@ -77,7 +77,7 @@ export class IndexingSettingsComponent extends SettingsComponentDirective<Server
this.inProgress = true;
this.error = '';
try {
await this.jobsService.start(DefaultsJobs[DefaultsJobs.Indexing]);
await this.jobsService.start(DefaultsJobs[DefaultsJobs.Indexing], {indexChangesOnly: true});
this.notification.info($localize`Folder indexing started`);
this.inProgress = false;
return true;

20
src/frontend/translate/messages.en.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.es.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.fr.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.hu.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum ennyi képet is videót fog listázni az alkalmazás egy keresési eredményben</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>ZIP-elt letöltés</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Engedélyezi a mappa letöltését ZIP-elve</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Csak változások indexelése</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Csak akkor indexel újra egy mappát, ha az változott</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximálisan ennyi képet és mappát listáz ki egy keresési eredménnyel</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.it.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

22
src/frontend/translate/messages.pl.xlf
View File

@ -1279,7 +1279,7 @@
<context context-type="sourcefile">src/frontend/app/ui/settings/jobs/jobs.settings.component.html</context>
<context context-type="linenumber">228,229</context>
</context-group>
<target>+ Dodaj zadanie </target>
<target>+ Dodaj zadanie</target>
</trans-unit>
<trans-unit id="8500119868694688450" datatype="html">
<source>Add new job</source>
@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maksymalna liczba zdjęć i plików wideo wyświetlanych w wynikach wyszukiwania</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.ro.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.ru.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>

20
src/frontend/translate/messages.sv.xlf
View File

@ -2752,6 +2752,26 @@
<source>Maximum number of photos and videos that listed in one search result</source>
<target>Maximum number of photos and videos that listed in one search result</target>
</trans-unit>
<trans-unit id="1524055079900109760" datatype="html">
<source>Download Zip</source>
<target>Download Zip</target>
</trans-unit>
<trans-unit id="8061493549899716713" datatype="html">
<source>Enable download zip of a directory contents</source>
<target>Enable download zip of a directory contents</target>
</trans-unit>
<trans-unit id="8335094363935357421" datatype="html">
<source>Index changes only</source>
<target>Index changes only</target>
</trans-unit>
<trans-unit id="2887355625408505193" datatype="html">
<source>Only indexes a folder if it got changed.</source>
<target>Only indexes a folder if it got changed.</target>
</trans-unit>
<trans-unit id="759454492324260485" datatype="html">
<source>Maximum number of photos and videos that listed in one search result.</source>
<target>Maximum number of photos and videos that listed in one search result.</target>
</trans-unit>
</body>
</file>
</xliff>