xuthus/pigallery2
1
0
Fork 0
mirror of https://github.com/xuthus83/pigallery2.git synced 2025-01-14 14:43:17 +08:00
Code Actions Projects Activity

Implementing media fail check.

Browse Source 
Gallery now will stop indexing if the root folder is empty. That is probably unintentional and would erase the whole database.
This commit is contained in:
Patrik J. Braun 2022-05-15 22:07:46 +02:00
parent e3bc01e41b
commit 5778ef00f7
2 changed files with 88 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/backend/model/database/sql/IndexingManager.ts
View File

@ -1,34 +1,34 @@
import { ParentDirectoryDTO } from '../../../../common/entities/DirectoryDTO'; import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO';
import { DirectoryEntity } from './enitites/DirectoryEntity'; import {DirectoryEntity} from './enitites/DirectoryEntity';
import { SQLConnection } from './SQLConnection'; import {SQLConnection} from './SQLConnection';
import { DiskManager } from '../../DiskManger'; import {DiskManager} from '../../DiskManger';
import { PhotoEntity, PhotoMetadataEntity } from './enitites/PhotoEntity'; import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
import { Utils } from '../../../../common/Utils'; import {Utils} from '../../../../common/Utils';
import { import {
FaceRegion, FaceRegion,
PhotoMetadata, PhotoMetadata,
} from '../../../../common/entities/PhotoDTO'; } from '../../../../common/entities/PhotoDTO';
import { Connection, Repository } from 'typeorm'; import {Connection, Repository} from 'typeorm';
import { MediaEntity } from './enitites/MediaEntity'; import {MediaEntity} from './enitites/MediaEntity';
import { MediaDTO, MediaDTOUtils } from '../../../../common/entities/MediaDTO'; import {MediaDTO, MediaDTOUtils} from '../../../../common/entities/MediaDTO';
import { VideoEntity } from './enitites/VideoEntity'; import {VideoEntity} from './enitites/VideoEntity';
import { FileEntity } from './enitites/FileEntity'; import {FileEntity} from './enitites/FileEntity';
import { FileDTO } from '../../../../common/entities/FileDTO'; import {FileDTO} from '../../../../common/entities/FileDTO';
import { NotificationManager } from '../../NotifocationManager'; import {NotificationManager} from '../../NotifocationManager';
import { FaceRegionEntry } from './enitites/FaceRegionEntry'; import {FaceRegionEntry} from './enitites/FaceRegionEntry';
import { ObjectManagers } from '../../ObjectManagers'; import {ObjectManagers} from '../../ObjectManagers';
import { IIndexingManager } from '../interfaces/IIndexingManager'; import {IIndexingManager} from '../interfaces/IIndexingManager';
import { DiskMangerWorker } from '../../threading/DiskMangerWorker'; import {DiskMangerWorker} from '../../threading/DiskMangerWorker';
import { Logger } from '../../../Logger'; import {Logger} from '../../../Logger';
import { import {
ServerPG2ConfMap, ServerPG2ConfMap,
ServerSidePG2ConfAction, ServerSidePG2ConfAction,
} from '../../../../common/PG2ConfMap'; } from '../../../../common/PG2ConfMap';
import { ProjectPath } from '../../../ProjectPath'; import {ProjectPath} from '../../../ProjectPath';
import * as path from 'path'; import * as path from 'path';
import * as fs from 'fs'; import * as fs from 'fs';
import { SearchQueryDTO } from '../../../../common/entities/SearchQueryDTO'; import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO';
import { PersonEntry } from './enitites/PersonEntry'; import {PersonEntry} from './enitites/PersonEntry';
const LOG_TAG = '[IndexingManager]'; const LOG_TAG = '[IndexingManager]';
@ -82,6 +82,14 @@ export class IndexingManager implements IIndexingManager {
// eslint-disable-next-line no-async-promise-executor // eslint-disable-next-line no-async-promise-executor
return new Promise(async (resolve, reject): Promise<void> => { return new Promise(async (resolve, reject): Promise<void> => {
try { try {
// Check if root is still a valid (non-empty) folder
// With weak devices it is possible that the media that stores
// the galley gets unmounted that triggers a full gallery wipe.
// Prevent it by stopping indexing on an empty folder.
if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {
return reject(new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.'));
}
const scannedDirectory = await DiskManager.scanDirectory( const scannedDirectory = await DiskManager.scanDirectory(
relativeDirectoryName relativeDirectoryName
); );
@ -157,9 +165,9 @@ export class IndexingManager implements IIndexingManager {
dir.lastModified === scannedDirectory.lastModified && dir.lastModified === scannedDirectory.lastModified &&
dir.lastScanned === scannedDirectory.lastScanned && dir.lastScanned === scannedDirectory.lastScanned &&
(dir.media || dir.media.length) === (dir.media || dir.media.length) ===
(scannedDirectory.media || scannedDirectory.media.length) && (scannedDirectory.media || scannedDirectory.media.length) &&
(dir.metaFile || dir.metaFile.length) === (dir.metaFile || dir.metaFile.length) ===
(scannedDirectory.metaFile || scannedDirectory.metaFile.length) (scannedDirectory.metaFile || scannedDirectory.metaFile.length)
) !== -1 ) !== -1
) { ) {
return; return;
@ -230,11 +238,11 @@ export class IndexingManager implements IIndexingManager {
await directoryRepository await directoryRepository
.createQueryBuilder() .createQueryBuilder()
.update(DirectoryEntity) .update(DirectoryEntity)
.set({ parent: currentDirId as any }) .set({parent: currentDirId as any})
.where('path = :path', { .where('path = :path', {
path: DiskMangerWorker.pathFromParent(scannedDirectory), path: DiskMangerWorker.pathFromParent(scannedDirectory),
}) })
.andWhere('name NOT LIKE :root', { root: DiskMangerWorker.dirName('.') }) .andWhere('name NOT LIKE :root', {root: DiskMangerWorker.dirName('.')})
.andWhere('parent IS NULL') .andWhere('parent IS NULL')
.execute(); .execute();
@ -258,7 +266,7 @@ export class IndexingManager implements IIndexingManager {
childDirectories.splice(dirIndex, 1); childDirectories.splice(dirIndex, 1);
} else { } else {
// dir does not exists yet // dir does not exists yet
directory.parent = { id: currentDirId } as any; directory.parent = {id: currentDirId} as any;
(directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet (directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet
const d = await directoryRepository.insert( const d = await directoryRepository.insert(
directory as DirectoryEntity directory as DirectoryEntity
@ -307,7 +315,7 @@ export class IndexingManager implements IIndexingManager {
item.directory = null; item.directory = null;
metaFile = Utils.clone(item); metaFile = Utils.clone(item);
item.directory = scannedDirectory; item.directory = scannedDirectory;
metaFile.directory = { id: currentDirID } as any; metaFile.directory = {id: currentDirID} as any;
metaFilesToSave.push(metaFile); metaFilesToSave.push(metaFile);
} }
} }
@ -369,10 +377,10 @@ export class IndexingManager implements IIndexingManager {
// not in DB yet // not in DB yet
media[i].directory = null; media[i].directory = null;
mediaItem = Utils.clone(media[i]) as any; mediaItem = Utils.clone(media[i]) as any;
mediaItem.directory = { id: parentDirId } as any; mediaItem.directory = {id: parentDirId} as any;
(MediaDTOUtils.isPhoto(mediaItem) (MediaDTOUtils.isPhoto(mediaItem)
? mediaChange.insertP ? mediaChange.insertP
: mediaChange.insertV : mediaChange.insertV
).push(mediaItem); ).push(mediaItem);
} else { } else {
// already in the DB, only needs to be updated // already in the DB, only needs to be updated
@ -380,8 +388,8 @@ export class IndexingManager implements IIndexingManager {
if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) { if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
mediaItem.metadata = media[i].metadata as any; mediaItem.metadata = media[i].metadata as any;
(MediaDTOUtils.isPhoto(mediaItem) (MediaDTOUtils.isPhoto(mediaItem)
? mediaChange.saveP ? mediaChange.saveP
: mediaChange.saveV : mediaChange.saveV
).push(mediaItem); ).push(mediaItem);
} }
} }
@ -412,7 +420,7 @@ export class IndexingManager implements IIndexingManager {
); );
group.faces.forEach( group.faces.forEach(
(sf: FaceRegionEntry): any => (sf: FaceRegionEntry): any =>
(sf.media = { id: indexedMedia[mIndex].id } as any) (sf.media = {id: indexedMedia[mIndex].id} as any)
); );
faces.push(...group.faces); faces.push(...group.faces);
@ -435,7 +443,7 @@ export class IndexingManager implements IIndexingManager {
for (const face of scannedFaces) { for (const face of scannedFaces) {
if (persons.findIndex((f) => f.name === face.name) === -1) { if (persons.findIndex((f) => f.name === face.name) === -1) {
persons.push({ name: face.name, faceRegion: face }); persons.push({name: face.name, faceRegion: face});
} }
} }
await ObjectManagers.getInstance().PersonManager.saveAll(persons); await ObjectManagers.getInstance().PersonManager.saveAll(persons);

46
test/backend/unit/model/sql/IndexingManager.spec.ts
View File

@ -21,7 +21,9 @@ import {DiskManager} from '../../../../../src/backend/model/DiskManger';
import {AlbumManager} from '../../../../../src/backend/model/database/sql/AlbumManager'; import {AlbumManager} from '../../../../../src/backend/model/database/sql/AlbumManager';
import {SortingMethods} from '../../../../../src/common/entities/SortingMethods'; import {SortingMethods} from '../../../../../src/common/entities/SortingMethods';
// eslint-disable-next-line @typescript-eslint/no-var-requires
const deepEqualInAnyOrder = require('deep-equal-in-any-order'); const deepEqualInAnyOrder = require('deep-equal-in-any-order');
// eslint-disable-next-line @typescript-eslint/no-var-requires
const chai = require('chai'); const chai = require('chai');
chai.use(deepEqualInAnyOrder); chai.use(deepEqualInAnyOrder);
@ -53,9 +55,11 @@ class IndexingManagerTest extends IndexingManager {
} }
// to help WebStorm to handle the test cases // to help WebStorm to handle the test cases
// eslint-disable-next-line @typescript-eslint/no-explicit-any
declare let describe: any; declare let describe: any;
declare const after: any; declare const after: any;
declare const it: any; declare const it: any;
// eslint-disable-next-line prefer-const
describe = DBTestHelper.describe(); describe = DBTestHelper.describe();
describe('IndexingManager', (sqlHelper: DBTestHelper) => { describe('IndexingManager', (sqlHelper: DBTestHelper) => {
@ -155,6 +159,46 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
}); });
it('should stop indexing on empty folder', async () => {
const gm = new GalleryManagerTest();
ProjectPath.reset();
ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets');
Config.Server.Threading.enabled = false;
await ObjectManagers.getInstance().IndexingManager.indexDirectory('.');
if (ObjectManagers.getInstance().IndexingManager.IsSavingInProgress) {
await ObjectManagers.getInstance().IndexingManager.SavingReady;
}
const directoryPath = GalleryManager.parseRelativeDirePath(
'.'
);
const conn = await SQLConnection.getConnection();
const selected = await gm.selectParentDir(conn, directoryPath.name,
directoryPath.parent);
await gm.fillParentDir(conn, selected);
expect(selected?.media?.length)
.to.be.greaterThan(0);
const tmpDir = path.join(__dirname, '/../../../tmp/rnd5sdf_emptyDir');
fs.mkdirSync(tmpDir);
ProjectPath.ImageFolder = tmpDir;
let notFailed = false;
try {
await ObjectManagers.getInstance().IndexingManager.indexDirectory('.');
notFailed = true;
} catch (e) {
// it expected to fail
}
if (notFailed) {
expect(true).to.equal(false, 'indexDirectory is expected to fail');
}
});
it('should support case sensitive directory', async () => { it('should support case sensitive directory', async () => {
const gm = new GalleryManagerTest(); const gm = new GalleryManagerTest();
const im = new IndexingManagerTest(); const im = new IndexingManagerTest();
@ -605,6 +649,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
}); });
afterEach(() => { afterEach(() => {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore // @ts-ignore
fs.statSync = statSync; fs.statSync = statSync;
}); });
@ -612,6 +657,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => {
it('with re indexing severity low', async () => { it('with re indexing severity low', async () => {
Config.Server.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low; Config.Server.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low;
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore // @ts-ignore
fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)});
const gm = new GalleryManagerTest(); const gm = new GalleryManagerTest();