From c6ba5051290c7ae277ab84294aa14b6e540f7126 Mon Sep 17 00:00:00 2001 From: "Patrik J. Braun" Date: Mon, 11 Sep 2023 13:54:42 +0200 Subject: [PATCH] Add date to markdowns #711 --- src/backend/model/database/IndexingManager.ts | 13 +++++++-- src/backend/model/database/SQLConnection.ts | 3 +++ .../model/database/enitites/FileEntity.ts | 17 +++++------- .../model/database/enitites/MDFileEntity.ts | 16 +++++++++++ .../model/threading/DiskMangerWorker.ts | 14 ++++++++++ src/common/DataStructureVersion.ts | 2 +- src/common/entities/MDFileDTO.ts | 10 +++++++ test/backend/assets/index.md | 1 + .../unit/model/sql/IndexingManager.spec.ts | 27 ++++++++++++++++++- 9 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 src/backend/model/database/enitites/MDFileEntity.ts create mode 100644 src/common/entities/MDFileDTO.ts create mode 100644 test/backend/assets/index.md diff --git a/src/backend/model/database/IndexingManager.ts b/src/backend/model/database/IndexingManager.ts index 051b2980..a32d3f91 100644 --- a/src/backend/model/database/IndexingManager.ts +++ b/src/backend/model/database/IndexingManager.ts @@ -22,6 +22,8 @@ import * as fs from 'fs'; import {SearchQueryDTO} from '../../../common/entities/SearchQueryDTO'; import {PersonEntry} from './enitites/PersonEntry'; import {PersonJunctionTable} from './enitites/PersonJunctionTable'; +import {MDFileEntity} from './enitites/MDFileEntity'; +import {MDFileDTO} from '../../../common/entities/MDFileDTO'; const LOG_TAG = '[IndexingManager]'; @@ -292,6 +294,7 @@ export class IndexingManager { scannedDirectory: ParentDirectoryDTO ): Promise { const fileRepository = connection.getRepository(FileEntity); + const MDfileRepository = connection.getRepository(MDFileEntity); // save files const indexedMetaFiles = await fileRepository .createQueryBuilder('file') @@ -319,8 +322,14 @@ export class IndexingManager { metaFilesToSave.push(metaFile); } } - await fileRepository.save(metaFilesToSave, { - chunk: Math.max(Math.ceil(metaFilesToSave.length / 500), 1), + + const MDFiles = metaFilesToSave.filter(f => !isNaN((f as MDFileDTO).date)); + const generalFiles = metaFilesToSave.filter(f => isNaN((f as MDFileDTO).date)); + await fileRepository.save(generalFiles, { + chunk: Math.max(Math.ceil(generalFiles.length / 500), 1), + }); + await MDfileRepository.save(MDFiles, { + chunk: Math.max(Math.ceil(MDFiles.length / 500), 1), }); await fileRepository.remove(indexedMetaFiles, { chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1), diff --git a/src/backend/model/database/SQLConnection.ts b/src/backend/model/database/SQLConnection.ts index 1da14657..84300d57 100644 --- a/src/backend/model/database/SQLConnection.ts +++ b/src/backend/model/database/SQLConnection.ts @@ -22,6 +22,7 @@ import {AlbumBaseEntity} from './enitites/album/AlbumBaseEntity'; import {SavedSearchEntity} from './enitites/album/SavedSearchEntity'; import {NotificationManager} from '../NotifocationManager'; import {PersonJunctionTable} from './enitites/PersonJunctionTable'; +import {MDFileEntity} from './enitites/MDFileEntity'; const LOG_TAG = '[SQLConnection]'; @@ -36,6 +37,7 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + MDFileEntity, PersonJunctionTable, PersonEntry, MediaEntity, @@ -76,6 +78,7 @@ export class SQLConnection { options.entities = [ UserEntity, FileEntity, + MDFileEntity, PersonJunctionTable, PersonEntry, MediaEntity, diff --git a/src/backend/model/database/enitites/FileEntity.ts b/src/backend/model/database/enitites/FileEntity.ts index a41ce8ff..00fd6082 100644 --- a/src/backend/model/database/enitites/FileEntity.ts +++ b/src/backend/model/database/enitites/FileEntity.ts @@ -1,18 +1,13 @@ -import { - Column, - Entity, - Index, - ManyToOne, - PrimaryGeneratedColumn, -} from 'typeorm'; -import { DirectoryEntity } from './DirectoryEntity'; -import { FileDTO } from '../../../../common/entities/FileDTO'; -import { columnCharsetCS } from './EntityUtils'; +import {Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn, TableInheritance,} from 'typeorm'; +import {DirectoryEntity} from './DirectoryEntity'; +import {FileDTO} from '../../../../common/entities/FileDTO'; +import {columnCharsetCS} from './EntityUtils'; @Entity() +@TableInheritance({column: {type: 'varchar', name: 'type', length: 16}}) export class FileEntity implements FileDTO { @Index() - @PrimaryGeneratedColumn({ unsigned: true }) + @PrimaryGeneratedColumn({unsigned: true}) id: number; @Column(columnCharsetCS) diff --git a/src/backend/model/database/enitites/MDFileEntity.ts b/src/backend/model/database/enitites/MDFileEntity.ts new file mode 100644 index 00000000..f67b19bf --- /dev/null +++ b/src/backend/model/database/enitites/MDFileEntity.ts @@ -0,0 +1,16 @@ +import {ChildEntity, Column,} from 'typeorm'; +import {MDFileDTO} from '../../../../common/entities/MDFileDTO'; +import {FileEntity} from './FileEntity'; + +@ChildEntity() +export class MDFileEntity extends FileEntity implements MDFileDTO { + + @Column('bigint', { + transformer: { + from: (v) => parseInt(v, 10), + to: (v) => v, + }, + }) + date: number; // same date as the youngest photo in a folder + +} diff --git a/src/backend/model/threading/DiskMangerWorker.ts b/src/backend/model/threading/DiskMangerWorker.ts index 477031c1..7959ad90 100644 --- a/src/backend/model/threading/DiskMangerWorker.ts +++ b/src/backend/model/threading/DiskMangerWorker.ts @@ -12,6 +12,7 @@ import {VideoProcessing} from '../fileprocessing/VideoProcessing'; import {PhotoProcessing} from '../fileprocessing/PhotoProcessing'; import {Utils} from '../../../common/Utils'; import {GPXProcessing} from '../fileprocessing/GPXProcessing'; +import {MDFileDTO} from '../../../common/entities/MDFileDTO'; export class DiskMangerWorker { public static calcLastModified(stat: Stats): number { @@ -227,6 +228,7 @@ export class DiskMangerWorker { ) { continue; } + directory.metaFile.push({ name: file, directory: null, @@ -244,6 +246,12 @@ export class DiskMangerWorker { directory.oldestMedia = Math.max(m.metadata.creationDate, directory.oldestMedia); } ); + + directory.metaFile.forEach(mf => { + if (DiskMangerWorker.isMarkdown(mf.name)) { + (mf as MDFileDTO).date = directory.youngestMedia; + } + }); } return directory; @@ -264,6 +272,12 @@ export class DiskMangerWorker { return false; } + + private static isMarkdown(fullPath: string): boolean { + const extension = path.extname(fullPath).toLowerCase(); + + return extension == '.md'; + } } export interface DirectoryScanSettings { diff --git a/src/common/DataStructureVersion.ts b/src/common/DataStructureVersion.ts index e55817a2..d7b968fe 100644 --- a/src/common/DataStructureVersion.ts +++ b/src/common/DataStructureVersion.ts @@ -1,4 +1,4 @@ /** * This version indicates that the sql/entities/*Entity.ts files got changed and the db needs to be recreated */ -export const DataStructureVersion = 33; +export const DataStructureVersion = 34; diff --git a/src/common/entities/MDFileDTO.ts b/src/common/entities/MDFileDTO.ts new file mode 100644 index 00000000..265bdfb5 --- /dev/null +++ b/src/common/entities/MDFileDTO.ts @@ -0,0 +1,10 @@ +import {DirectoryPathDTO} from './DirectoryDTO'; +import {FileDTO} from './FileDTO'; + +export interface MDFileDTO extends FileDTO { + id: number; + name: string; + directory: DirectoryPathDTO; + date: number; +} + diff --git a/test/backend/assets/index.md b/test/backend/assets/index.md new file mode 100644 index 00000000..8b99cb22 --- /dev/null +++ b/test/backend/assets/index.md @@ -0,0 +1 @@ +This is a test md. diff --git a/test/backend/unit/model/sql/IndexingManager.spec.ts b/test/backend/unit/model/sql/IndexingManager.spec.ts index 73362710..9fd32c64 100644 --- a/test/backend/unit/model/sql/IndexingManager.spec.ts +++ b/test/backend/unit/model/sql/IndexingManager.spec.ts @@ -635,6 +635,32 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => { expect(selected.media.length).to.equal(subDir.media.length); }) as any).timeout(40000); + it('should save .md with date', async () => { + Config.Server.Threading.enabled = false; + Config.Album.enabled = true; + Config.Faces.enabled = true; + + Config.Media.folder = path.join(__dirname, '/../../../assets'); + ProjectPath.ImageFolder = path.join(__dirname, '/../../../assets'); + const im = new IndexingManagerTest(); + const gm = new GalleryManagerTest(); + + const d = await DiskManager.scanDirectory('/'); + + await im.saveToDB(d); + + const dir = await gm.listDirectory('/'); + expect(dir.metaFile).to.be.an('array'); + + expect(dir.metaFile).to.be.deep.equal([ + { + date: 1126455782000, + id: 1, + name: 'index.md' + } + ]); + }); + DBTestHelper.savedDescribe('Test listDirectory', () => { const statSync = fs.statSync; let dirTime = 0; @@ -691,7 +717,6 @@ describe('IndexingManager', (sqlHelper: DBTestHelper) => { }); }); - DBTestHelper.savedDescribe('should index .pg2conf', () => {