improving fileJob directory scanning

src/backend/model/DiskManger.ts

@@ -3,6 +3,7 @@ import {Logger} from '../Logger';
 import {Config} from '../../common/config/private/Config';
 import {DiskManagerTH} from './threading/ThreadPool';
 import {DiskMangerWorker} from './threading/DiskMangerWorker';
+import {FileDTO} from '../../common/entities/FileDTO';
 
 
 const LOG_TAG = '[DiskManager]';
@@ -16,6 +17,13 @@ export class DiskManager {
     }
   }
 
+
+  public static async scanDirectoryNoMetadata(relativeDirectoryName: string,
+                                              settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise<DirectoryDTO<FileDTO>> {
+    settings.noMetadata = true;
+    return this.scanDirectory(relativeDirectoryName, settings);
+  }
+
   public static async scanDirectory(relativeDirectoryName: string,
                                     settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise<DirectoryDTO> {
 

src/backend/model/jobs/jobs/FileJob.ts

@@ -28,6 +28,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
 
   protected constructor(private scanFilter: DiskMangerWorker.DirectoryScanSettings) {
     super();
+    this.scanFilter.noChildDirPhotos = true;
     if (Config.Server.Database.type !== ServerConfig.DatabaseType.memory) {
       this.ConfigTemplate.push({
         id: 'indexedOnly',
@@ -96,7 +97,7 @@ export abstract class FileJob<S extends { indexedOnly: boolean } = { indexedOnly
   private async loadADirectoryFromDisk() {
     const directory = this.directoryQueue.shift();
     this.Progress.log('scanning directory: ' + directory);
-    const scanned = await DiskManager.scanDirectory(directory, this.scanFilter);
+    const scanned = await DiskManager.scanDirectoryNoMetadata(directory, this.scanFilter);
     for (let i = 0; i < scanned.directories.length; i++) {
       this.directoryQueue.push(path.join(scanned.directories[i].path, scanned.directories[i].name));
     }

src/backend/model/threading/DiskMangerWorker.ts

@@ -81,6 +81,12 @@ export class DiskMangerWorker {
     return false;
   }
 
+  public static async scanDirectoryNoMetadata(relativeDirectoryName: string,
+                                              settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise<DirectoryDTO<FileDTO>> {
+    settings.noMetadata = true;
+    return this.scanDirectory(relativeDirectoryName, settings);
+  }
+
   public static async scanDirectory(relativeDirectoryName: string,
                                     settings: DiskMangerWorker.DirectoryScanSettings = {}): Promise<DirectoryDTO> {
 
@@ -103,6 +109,11 @@ export class DiskMangerWorker {
       media: [],
       metaFile: []
     };
+
+    // nothing to scan, we are here for the empty dir
+    if (settings.noPhoto === true && settings.noMetadata === true && settings.noVideo === true) {
+      return directory;
+    }
     const list = await fsp.readdir(absoluteDirectoryName);
     for (let i = 0; i < list.length; i++) {
       const file = list[i];
@@ -119,12 +130,15 @@ export class DiskMangerWorker {
             maxPhotos: Config.Server.Indexing.folderPreviewSize,
             noMetaFile: true,
             noVideo: true,
-            noDirectory: true
+            noDirectory: true,
+            noPhoto: settings.noChildDirPhotos || settings.noPhoto
           }
         );
+
         d.lastScanned = 0; // it was not a fully scan
         d.isPartial = true;
         directory.directories.push(d);
+
       } else if (PhotoProcessing.isPhoto(fullFilePath)) {
         if (settings.noPhoto === true) {
           continue;
@@ -132,9 +146,10 @@ export class DiskMangerWorker {
         directory.media.push(<PhotoDTO>{
           name: file,
           directory: null,
-          metadata: await MetadataLoader.loadPhotoMetadata(fullFilePath)
+          metadata: settings.noMetadata === true ? null : await MetadataLoader.loadPhotoMetadata(fullFilePath)
         });
 
+
         if (settings.maxPhotos && directory.media.length > settings.maxPhotos) {
           break;
         }
@@ -146,7 +161,7 @@ export class DiskMangerWorker {
           directory.media.push(<VideoDTO>{
             name: file,
             directory: null,
-            metadata: await MetadataLoader.loadVideoMetadata(fullFilePath)
+            metadata: settings.noMetadata === true ? null : await MetadataLoader.loadVideoMetadata(fullFilePath)
           });
         } catch (e) {
           Logger.warn('Media loading error, skipping: ' + file + ', reason: ' + e.toString());
@@ -185,5 +200,7 @@ export namespace DiskMangerWorker {
     noVideo?: boolean;
     noPhoto?: boolean;
     noDirectory?: boolean;
+    noMetadata?: boolean;
+    noChildDirPhotos?: boolean;
   }
 }

src/common/entities/DirectoryDTO.ts

@@ -1,17 +1,17 @@
 import {MediaDTO} from './MediaDTO';
 import {FileDTO} from './FileDTO';
 
-export interface DirectoryDTO {
+export interface DirectoryDTO<S extends FileDTO = MediaDTO> {
   id: number;
   name: string;
   path: string;
   lastModified: number;
   lastScanned: number;
   isPartial?: boolean;
-  parent: DirectoryDTO;
+  parent: DirectoryDTO<S>;
   mediaCount: number;
-  directories: DirectoryDTO[];
+  directories: DirectoryDTO<S>[];
-  media: MediaDTO[];
+  media: S[];
   metaFile: FileDTO[];
 }