From 5778ef00f7b169d51adec734c703be165f944662 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sun, 15 May 2022 22:07:46 +0200 Subject: [PATCH] Implementing media fail check. Gallery now will stop indexing if the root folder is empty. That is probably unintentional and would erase the whole database. --- .../model/database/sql/IndexingManager.ts | 76 ++++++++++--------- .../unit/model/sql/IndexingManager.spec.ts | 46 +++++++++++ 2 files changed, 88 insertions(+), 34 deletions(-) diff --git a/src/backend/model/database/sql/IndexingManager.ts b/src/backend/model/database/sql/IndexingManager.ts index f7ec251e..18c2f178 100644 --- a/src/backend/model/database/sql/IndexingManager.ts +++ b/src/backend/model/database/sql/IndexingManager.ts @@ -1,34 +1,34 @@ -import { ParentDirectoryDTO } from '../../../../common/entities/DirectoryDTO'; -import { DirectoryEntity } from './enitites/DirectoryEntity'; -import { SQLConnection } from './SQLConnection'; -import { DiskManager } from '../../DiskManger'; -import { PhotoEntity, PhotoMetadataEntity } from './enitites/PhotoEntity'; -import { Utils } from '../../../../common/Utils'; +import {ParentDirectoryDTO} from '../../../../common/entities/DirectoryDTO'; +import {DirectoryEntity} from './enitites/DirectoryEntity'; +import {SQLConnection} from './SQLConnection'; +import {DiskManager} from '../../DiskManger'; +import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity'; +import {Utils} from '../../../../common/Utils'; import { FaceRegion, PhotoMetadata, } from '../../../../common/entities/PhotoDTO'; -import { Connection, Repository } from 'typeorm'; -import { MediaEntity } from './enitites/MediaEntity'; -import { MediaDTO, MediaDTOUtils } from '../../../../common/entities/MediaDTO'; -import { VideoEntity } from './enitites/VideoEntity'; -import { FileEntity } from './enitites/FileEntity'; -import { FileDTO } from '../../../../common/entities/FileDTO'; -import { NotificationManager } from '../../NotifocationManager'; -import { FaceRegionEntry } from './enitites/FaceRegionEntry'; -import { ObjectManagers } from '../../ObjectManagers'; -import { IIndexingManager } from '../interfaces/IIndexingManager'; -import { DiskMangerWorker } from '../../threading/DiskMangerWorker'; -import { Logger } from '../../../Logger'; +import {Connection, Repository} from 'typeorm'; +import {MediaEntity} from './enitites/MediaEntity'; +import {MediaDTO, MediaDTOUtils} from '../../../../common/entities/MediaDTO'; +import {VideoEntity} from './enitites/VideoEntity'; +import {FileEntity} from './enitites/FileEntity'; +import {FileDTO} from '../../../../common/entities/FileDTO'; +import {NotificationManager} from '../../NotifocationManager'; +import {FaceRegionEntry} from './enitites/FaceRegionEntry'; +import {ObjectManagers} from '../../ObjectManagers'; +import {IIndexingManager} from '../interfaces/IIndexingManager'; +import {DiskMangerWorker} from '../../threading/DiskMangerWorker'; +import {Logger} from '../../../Logger'; import { ServerPG2ConfMap, ServerSidePG2ConfAction, } from '../../../../common/PG2ConfMap'; -import { ProjectPath } from '../../../ProjectPath'; +import {ProjectPath} from '../../../ProjectPath'; import * as path from 'path'; import * as fs from 'fs'; -import { SearchQueryDTO } from '../../../../common/entities/SearchQueryDTO'; -import { PersonEntry } from './enitites/PersonEntry'; +import {SearchQueryDTO} from '../../../../common/entities/SearchQueryDTO'; +import {PersonEntry} from './enitites/PersonEntry'; const LOG_TAG = '[IndexingManager]'; @@ -82,6 +82,14 @@ export class IndexingManager implements IIndexingManager { // eslint-disable-next-line no-async-promise-executor return new Promise(async (resolve, reject): Promise => { try { + // Check if root is still a valid (non-empty) folder + // With weak devices it is possible that the media that stores + // the galley gets unmounted that triggers a full gallery wipe. + // Prevent it by stopping indexing on an empty folder. + if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) { + return reject(new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.')); + } + const scannedDirectory = await DiskManager.scanDirectory( relativeDirectoryName ); @@ -157,9 +165,9 @@ export class IndexingManager implements IIndexingManager { dir.lastModified === scannedDirectory.lastModified && dir.lastScanned === scannedDirectory.lastScanned && (dir.media || dir.media.length) === - (scannedDirectory.media || scannedDirectory.media.length) && + (scannedDirectory.media || scannedDirectory.media.length) && (dir.metaFile || dir.metaFile.length) === - (scannedDirectory.metaFile || scannedDirectory.metaFile.length) + (scannedDirectory.metaFile || scannedDirectory.metaFile.length) ) !== -1 ) { return; @@ -230,11 +238,11 @@ export class IndexingManager implements IIndexingManager { await directoryRepository .createQueryBuilder() .update(DirectoryEntity) - .set({ parent: currentDirId as any }) + .set({parent: currentDirId as any}) .where('path = :path', { path: DiskMangerWorker.pathFromParent(scannedDirectory), }) - .andWhere('name NOT LIKE :root', { root: DiskMangerWorker.dirName('.') }) + .andWhere('name NOT LIKE :root', {root: DiskMangerWorker.dirName('.')}) .andWhere('parent IS NULL') .execute(); @@ -258,7 +266,7 @@ export class IndexingManager implements IIndexingManager { childDirectories.splice(dirIndex, 1); } else { // dir does not exists yet - directory.parent = { id: currentDirId } as any; + directory.parent = {id: currentDirId} as any; (directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet const d = await directoryRepository.insert( directory as DirectoryEntity @@ -307,7 +315,7 @@ export class IndexingManager implements IIndexingManager { item.directory = null; metaFile = Utils.clone(item); item.directory = scannedDirectory; - metaFile.directory = { id: currentDirID } as any; + metaFile.directory = {id: currentDirID} as any; metaFilesToSave.push(metaFile); } } @@ -369,10 +377,10 @@ export class IndexingManager implements IIndexingManager { // not in DB yet media[i].directory = null; mediaItem = Utils.clone(media[i]) as any; - mediaItem.directory = { id: parentDirId } as any; + mediaItem.directory = {id: parentDirId} as any; (MediaDTOUtils.isPhoto(mediaItem) - ? mediaChange.insertP - : mediaChange.insertV + ? mediaChange.insertP + : mediaChange.insertV ).push(mediaItem); } else { // already in the DB, only needs to be updated @@ -380,8 +388,8 @@ export class IndexingManager implements IIndexingManager { if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) { mediaItem.metadata = media[i].metadata as any; (MediaDTOUtils.isPhoto(mediaItem) - ? mediaChange.saveP - : mediaChange.saveV + ? mediaChange.saveP + : mediaChange.saveV ).push(mediaItem); } } @@ -412,7 +420,7 @@ export class IndexingManager implements IIndexingManager { ); group.faces.forEach( (sf: FaceRegionEntry): any => - (sf.media = { id: indexedMedia[mIndex].id } as any) + (sf.media = {id: indexedMedia[mIndex].id} as any) ); faces.push(...group.faces); @@ -435,7 +443,7 @@ export class IndexingManager implements IIndexingManager { for (const face of scannedFaces) { if (persons.findIndex((f) => f.name === face.name) === -1) { - persons.push({ name: face.name, faceRegion: face }); + persons.push({name: face.name, faceRegion: face}); } } await ObjectManagers.getInstance().PersonManager.saveAll(persons); diff --git a/test/backend/unit/model/sql/IndexingManager.spec.ts b/test/backend/unit/model/sql/IndexingManager.spec.ts index 3cb3adbe..ca873589 100644 --- a/test/backend/unit/model/sql/IndexingManager.spec.ts +++ b/test/backend/unit/model/sql/IndexingManager.spec.ts @@ -21,7 +21,9 @@ import {DiskManager} from '../../../../../src/backend/model/DiskManger'; import {AlbumManager} from '../../../../../src/backend/model/database/sql/AlbumManager'; import {SortingMethods} from '../../../../../src/common/entities/SortingMethods'; +// eslint-disable-next-line @typescript-eslint/no-var-requires const deepEqualInAnyOrder = require('deep-equal-in-any-order'); +// eslint-disable-next-line @typescript-eslint/no-var-requires const chai = require('chai'); chai.use(deepEqualInAnyOrder); @@ -53,9 +55,11 @@ class IndexingManagerTest extends IndexingManager { } // to help WebStorm to handle the test cases +// eslint-disable-next-line @typescript-eslint/no-explicit-any declare let describe: any; declare const after: any; declare const it: any; +// eslint-disable-next-line prefer-const describe = DBTestHelper.describe(); describe('IndexingManager', (sqlHelper: DBTestHelper) => { @@ -155,6 +159,46 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => { }); + it('should stop indexing on empty folder', async () => { + const gm = new GalleryManagerTest(); + + ProjectPath.reset(); + ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets'); + Config.Server.Threading.enabled = false; + + await ObjectManagers.getInstance().IndexingManager.indexDirectory('.'); + if (ObjectManagers.getInstance().IndexingManager.IsSavingInProgress) { + await ObjectManagers.getInstance().IndexingManager.SavingReady; + } + + const directoryPath = GalleryManager.parseRelativeDirePath( + '.' + ); + const conn = await SQLConnection.getConnection(); + const selected = await gm.selectParentDir(conn, directoryPath.name, + directoryPath.parent); + await gm.fillParentDir(conn, selected); + + + expect(selected?.media?.length) + .to.be.greaterThan(0); + const tmpDir = path.join(__dirname, '/../../../tmp/rnd5sdf_emptyDir'); + fs.mkdirSync(tmpDir); + ProjectPath.ImageFolder = tmpDir; + let notFailed = false; + try { + await ObjectManagers.getInstance().IndexingManager.indexDirectory('.'); + notFailed = true; + } catch (e) { + // it expected to fail + } + if (notFailed) { + expect(true).to.equal(false, 'indexDirectory is expected to fail'); + } + + }); + + it('should support case sensitive directory', async () => { const gm = new GalleryManagerTest(); const im = new IndexingManagerTest(); @@ -605,6 +649,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => { }); afterEach(() => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore fs.statSync = statSync; }); @@ -612,6 +657,7 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => { it('with re indexing severity low', async () => { Config.Server.Indexing.reIndexingSensitivity = ReIndexingSensitivity.low; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); const gm = new GalleryManagerTest();