From a7be82a59aa3e6c34873e8c5f00a2fcf63ce1178 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Sat, 22 Dec 2018 00:09:07 +0100 Subject: [PATCH] fixing taskQue --- backend/model/sql/GalleryManager.ts | 11 ++-- backend/model/threading/DiskMangerWorker.ts | 1 - backend/model/threading/MetadataLoader.ts | 4 +- backend/model/threading/TaskQue.ts | 6 ++- backend/model/threading/ThumbnailWorker.ts | 1 + test/backend/unit/model/sql/GalleryManager.ts | 54 ++++++++++++++++++- .../unit/model/threading/TaskQue.spec.ts | 8 +-- 7 files changed, 67 insertions(+), 18 deletions(-) diff --git a/backend/model/sql/GalleryManager.ts b/backend/model/sql/GalleryManager.ts index 88462366..7096da90 100644 --- a/backend/model/sql/GalleryManager.ts +++ b/backend/model/sql/GalleryManager.ts @@ -88,7 +88,6 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { const dir = await this.selectParentDir(connection, directoryName, directoryParent); - if (dir && dir.lastScanned != null) { // If it seems that the content did not changed, do not work on it if (knownLastModified && knownLastScanned @@ -102,25 +101,25 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { return null; } } - await this.fillParentDir(connection, dir); if (dir.lastModified !== lastModified) { return this.indexDirectory(relativeDirectoryName); } + // not indexed since a while, index it in a lazy manner if ((Date.now() - dir.lastScanned > Config.Server.indexing.cachedFolderTimeout && Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.medium) || Config.Server.indexing.reIndexingSensitivity >= ReIndexingSensitivity.high) { // on the fly reindexing + this.indexDirectory(relativeDirectoryName).catch((err) => { console.error(err); }); } + await this.fillParentDir(connection, dir); return dir; - - } // never scanned (deep indexed), do it and return with it @@ -275,7 +274,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { delete directory.media; await directoryRepository.save(directory); } - } else { //dir does not exists yet + } else { // dir does not exists yet scannedDirectory.directories[i].parent = currentDir; (scannedDirectory.directories[i]).lastScanned = null; // new child dir, not fully scanned yet const d = await directoryRepository.save(scannedDirectory.directories[i]); @@ -375,7 +374,7 @@ export class GalleryManager implements IGalleryManager, ISQLGalleryManager { async countMediaSize(): Promise { const connection = await SQLConnection.getConnection(); - let {sum} = await connection.getRepository(MediaEntity) + const {sum} = await connection.getRepository(MediaEntity) .createQueryBuilder('media') .select('SUM(media.metadata.fileSize)', 'sum') .getRawOne(); diff --git a/backend/model/threading/DiskMangerWorker.ts b/backend/model/threading/DiskMangerWorker.ts index 353eaa8e..94cc1f0c 100644 --- a/backend/model/threading/DiskMangerWorker.ts +++ b/backend/model/threading/DiskMangerWorker.ts @@ -72,7 +72,6 @@ export class DiskMangerWorker { if (err) { return reject(err); } - try { for (let i = 0; i < list.length; i++) { const file = list[i]; diff --git a/backend/model/threading/MetadataLoader.ts b/backend/model/threading/MetadataLoader.ts index 1aa25e6d..16e506e0 100644 --- a/backend/model/threading/MetadataLoader.ts +++ b/backend/model/threading/MetadataLoader.ts @@ -67,8 +67,8 @@ export class MetadataLoader { const data = Buffer.allocUnsafe(Config.Server.photoMetadataSize); fs.read(fd, data, 0, Config.Server.photoMetadataSize, 0, (err) => { + fs.closeSync(fd); if (err) { - fs.closeSync(fd); return reject({file: fullPath, error: err}); } const metadata: PhotoMetadata = { @@ -157,10 +157,8 @@ export class MetadataLoader { metadata.creationDate = metadata.creationDate || 0; - fs.closeSync(fd); return resolve(metadata); } catch (err) { - fs.closeSync(fd); return reject({file: fullPath, error: err}); } }); diff --git a/backend/model/threading/TaskQue.ts b/backend/model/threading/TaskQue.ts index 6be0da10..c811966d 100644 --- a/backend/model/threading/TaskQue.ts +++ b/backend/model/threading/TaskQue.ts @@ -53,6 +53,10 @@ export class TaskQue { } public ready(task: TaskQueEntry): void { - this.processing.slice(this.processing.indexOf(task), 1); + const index = this.processing.indexOf(task); + if (index === -1) { + throw new Error('Task does not exist'); + } + this.processing.splice(index, 1); } } diff --git a/backend/model/threading/ThumbnailWorker.ts b/backend/model/threading/ThumbnailWorker.ts index eeb15264..80456b14 100644 --- a/backend/model/threading/ThumbnailWorker.ts +++ b/backend/model/threading/ThumbnailWorker.ts @@ -165,6 +165,7 @@ export class ImageRendererFactory { public static Sharp() { const sharp = require('sharp'); + sharp.cache(false); return async (input: RendererInput): Promise => { Logger.silly('[SharpThRenderer] rendering thumbnail:' + input.mediaPath); diff --git a/test/backend/unit/model/sql/GalleryManager.ts b/test/backend/unit/model/sql/GalleryManager.ts index 40aa5f65..5a4e6b08 100644 --- a/test/backend/unit/model/sql/GalleryManager.ts +++ b/test/backend/unit/model/sql/GalleryManager.ts @@ -2,7 +2,7 @@ import {expect} from 'chai'; import * as fs from 'fs'; import * as path from 'path'; import {Config} from '../../../../../common/config/private/Config'; -import {DatabaseType} from '../../../../../common/config/private/IPrivateConfig'; +import {DatabaseType, ReIndexingSensitivity} from '../../../../../common/config/private/IPrivateConfig'; import {SQLConnection} from '../../../../../backend/model/sql/SQLConnection'; import {GalleryManager} from '../../../../../backend/model/sql/GalleryManager'; import {DirectoryDTO} from '../../../../../common/entities/DirectoryDTO'; @@ -229,4 +229,56 @@ describe('GalleryManager', () => { expect(selected.media.length).to.deep.equal(subDir.media.length); })).timeout(20000); + describe('Test listDirectory', () => { + const statSync = fs.statSync; + let dirTime = 0; + const indexedTime = { + lastScanned: 0, + lastModified: 0 + }; + + beforeEach(() => { + dirTime = 0; + indexedTime.lastModified = 0; + indexedTime.lastScanned = 0; + }); + + afterEach(() => { + // @ts-ignore + fs.statSync = statSync; + }); + + it('with re indexing severity low', async () => { + Config.Server.indexing.reIndexingSensitivity = ReIndexingSensitivity.low; + + // @ts-ignore + fs.statSync = () => ({ctime: new Date(dirTime), mtime: new Date(dirTime)}); + const gm = new GalleryManagerTest(); + gm.selectParentDir = (connection: Connection, directoryName: string, directoryParent: string) => { + return Promise.resolve(indexedTime); + }; + gm.fillParentDir = (connection: Connection, dir: DirectoryEntity) => { + return Promise.resolve(); + }; + + gm.indexDirectory = (...args) => { + return Promise.resolve('indexing'); + }; + + indexedTime.lastScanned = null; + expect(await gm.listDirectory('./')).to.be.equal('indexing'); + indexedTime.lastModified = 0; + dirTime = 1; + expect(await gm.listDirectory('./')).to.be.equal('indexing'); + indexedTime.lastScanned = 10; + indexedTime.lastModified = 1; + dirTime = 1; + expect(await gm.listDirectory('./')).to.be.equal(indexedTime); + expect(await gm.listDirectory('./', 1, 10)) + .to.be.equal(null); + + + }); + }); + }); diff --git a/test/backend/unit/model/threading/TaskQue.spec.ts b/test/backend/unit/model/threading/TaskQue.spec.ts index 92985588..ff1c7cd7 100644 --- a/test/backend/unit/model/threading/TaskQue.spec.ts +++ b/test/backend/unit/model/threading/TaskQue.spec.ts @@ -23,11 +23,7 @@ describe('TaskQue', () => { tq.add(2); const task = tq.get(); tq.ready(task); - try { - tq.ready(task); - expect(false).to.be.equal(true); // should not reach - } catch (e) { - expect(e).not.to.be.equal(null); - } + expect(tq.isEmpty()).to.be.equal(true); + expect(tq.ready.bind(tq, task)).to.be.throw('Task does not exist'); }); });