Add SLQ entities to extensions #753

src/backend/model/database/SQLConnection.ts

@@ -29,6 +29,38 @@ const LOG_TAG = '[SQLConnection]';
 type Writeable<T> = { -readonly [P in keyof T]: T[P] };
 
 export class SQLConnection {
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  public static getEntries(): Function[] {
+    return this.entries;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  public static async addEntries(tables: Function[]) {
+    if (!tables?.length) {
+      return;
+    }
+    await this.close();
+    this.entries = Utils.getUnique(this.entries.concat(tables));
+    await (await this.getConnection()).synchronize();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  private static entries: Function[] = [
+    UserEntity,
+    FileEntity,
+    MDFileEntity,
+    PersonJunctionTable,
+    PersonEntry,
+    MediaEntity,
+    PhotoEntity,
+    VideoEntity,
+    DirectoryEntity,
+    SharingEntity,
+    AlbumBaseEntity,
+    SavedSearchEntity,
+    VersionEntity,
+  ];
+
   private static connection: Connection = null;
 
 
@@ -37,10 +69,10 @@ export class SQLConnection {
       const options = this.getDriver(Config.Database);
 
       Logger.debug(
-          LOG_TAG,
-          'Creating connection: ' + DatabaseType[Config.Database.type],
-          ', with driver:',
-          options.type
+        LOG_TAG,
+        'Creating connection: ' + DatabaseType[Config.Database.type],
+        ', with driver:',
+        options.type
       );
       this.connection = await this.createConnection(options);
       await SQLConnection.schemeSync(this.connection);
@@ -49,7 +81,7 @@ export class SQLConnection {
   }
 
   public static async tryConnection(
-      config: ServerDataBaseConfig
+    config: ServerDataBaseConfig
   ): Promise<boolean> {
     try {
       await getConnection('test').close();
@@ -73,8 +105,8 @@ export class SQLConnection {
     // Adding enforced users to the db
     const userRepository = connection.getRepository(UserEntity);
     if (
-        Array.isArray(Config.Users.enforcedUsers) &&
-        Config.Users.enforcedUsers.length > 0
+      Array.isArray(Config.Users.enforcedUsers) &&
+      Config.Users.enforcedUsers.length > 0
     ) {
       for (let i = 0; i < Config.Users.enforcedUsers.length; ++i) {
         const uc = Config.Users.enforcedUsers[i];
@@ -106,12 +138,12 @@ export class SQLConnection {
       role: UserRoles.Admin,
     });
     if (
-        defAdmin &&
-        PasswordHelper.comparePassword('admin', defAdmin.password)
+      defAdmin &&
+      PasswordHelper.comparePassword('admin', defAdmin.password)
     ) {
       NotificationManager.error(
-          'Using default admin user!',
-          'You are using the default admin/admin user/password, please change or remove it.'
+        'Using default admin user!',
+        'You are using the default admin/admin user/password, please change or remove it.'
       );
     }
   }
@@ -128,12 +160,39 @@ export class SQLConnection {
     }
   }
 
+  private static FIXED_SQL_TABLE = [
+    'sqlite_sequence'
+  ];
+
+  /**
+   * Clears up the DB from unused tables. use it when the entities list are up-to-date (extensions won't add any new)
+   */
+  public static async removeUnusedTables() {
+    const conn = await this.getConnection();
+    const validTableNames = this.entries.map(e => conn.getRepository(e).metadata.tableName).concat(this.FIXED_SQL_TABLE);
+    let currentTables: string[];
+
+    if (Config.Database.type === DatabaseType.sqlite) {
+      currentTables = (await conn.query('SELECT name FROM sqlite_master  WHERE type=\'table\''))
+        .map((r: { name: string }) => r.name);
+    } else {
+      currentTables = (await conn.query(`SELECT table_name FROM information_schema.tables ` +
+        `WHERE table_schema = '${Config.Database.mysql.database}'`))
+        .map((r: { table_name: string }) => r.table_name);
+    }
+
+    const tableToDrop = currentTables.filter(ct => !validTableNames.includes(ct));
+    for (let i = 0; i < tableToDrop.length; ++i) {
+      await conn.query('DROP TABLE ' + tableToDrop[i]);
+    }
+  }
+
   public static getSQLiteDB(config: ServerDataBaseConfig): string {
     return path.join(ProjectPath.getAbsolutePath(config.dbFolder), 'sqlite.db');
   }
 
   private static async createConnection(
-      options: DataSourceOptions
+    options: DataSourceOptions
   ): Promise<Connection> {
     if (options.type === 'sqlite' || options.type === 'better-sqlite3') {
       return await createConnection(options);
@@ -149,7 +208,7 @@ export class SQLConnection {
         delete tmpOption.database;
         const tmpConn = await createConnection(tmpOption);
         await tmpConn.query(
-            'CREATE DATABASE IF NOT EXISTS ' + options.database
+          'CREATE DATABASE IF NOT EXISTS ' + options.database
         );
         await tmpConn.close();
         return await createConnection(options);
@@ -177,9 +236,9 @@ export class SQLConnection {
     let users: UserEntity[] = [];
     try {
       users = await connection
-          .getRepository(UserEntity)
-          .createQueryBuilder('user')
-          .getMany();
+        .getRepository(UserEntity)
+        .createQueryBuilder('user')
+        .getMany();
       // eslint-disable-next-line no-empty
     } catch (ex) {
     }
@@ -193,9 +252,9 @@ export class SQLConnection {
       await connection.synchronize();
       await connection.getRepository(VersionEntity).save(version);
       Logger.warn(
-          LOG_TAG,
-          'Could not move users to the new db scheme, deleting them. Details:' +
-          e.toString()
+        LOG_TAG,
+        'Could not move users to the new db scheme, deleting them. Details:' +
+        e.toString()
       );
     }
   }
@@ -217,26 +276,12 @@ export class SQLConnection {
       driver = {
         type: 'better-sqlite3',
         database: path.join(
-            ProjectPath.getAbsolutePath(config.dbFolder),
-            config.sqlite.DBFileName
+          ProjectPath.getAbsolutePath(config.dbFolder),
+          config.sqlite.DBFileName
         ),
       };
     }
-    driver.entities = [
-      UserEntity,
-      FileEntity,
-      MDFileEntity,
-      PersonJunctionTable,
-      PersonEntry,
-      MediaEntity,
-      PhotoEntity,
-      VideoEntity,
-      DirectoryEntity,
-      SharingEntity,
-      AlbumBaseEntity,
-      SavedSearchEntity,
-      VersionEntity,
-    ];
+    driver.entities = this.entries;
     driver.synchronize = false;
     if (Config.Server.Log.sqlLevel !== SQLLogLevel.none) {
       driver.logging = SQLLogLevel[Config.Server.Log.sqlLevel] as LoggerOptions;

src/backend/model/extension/ExpressRouterWrapper.ts

@@ -5,34 +5,36 @@ import {AuthenticationMWs} from '../../middlewares/user/AuthenticationMWs';
 import {RenderingMWs} from '../../middlewares/RenderingMWs';
 import {ParamsDictionary} from 'express-serve-static-core';
 import {IExtensionRESTApi, IExtensionRESTRoute} from './IExtension';
-import {Logger} from '../../Logger';
+import {ILogger} from '../../Logger';
 import {ExtensionManager} from './ExtensionManager';
 import {Utils} from '../../../common/Utils';
 
 
 export class ExpressRouterWrapper implements IExtensionRESTApi {
 
-  constructor(private readonly router: express.Router, private readonly name: string) {
+  constructor(private readonly router: express.Router,
+              private readonly name: string,
+              private readonly extLogger: ILogger) {
   }
 
   get use() {
-    return new ExpressRouteWrapper(this.router, this.name, 'use');
+    return new ExpressRouteWrapper(this.router, this.name, 'use', this.extLogger);
   }
 
   get get() {
-    return new ExpressRouteWrapper(this.router, this.name, 'get');
+    return new ExpressRouteWrapper(this.router, this.name, 'get', this.extLogger);
   }
 
   get put() {
-    return new ExpressRouteWrapper(this.router, this.name, 'put');
+    return new ExpressRouteWrapper(this.router, this.name, 'put', this.extLogger);
   }
 
   get post() {
-    return new ExpressRouteWrapper(this.router, this.name, 'post');
+    return new ExpressRouteWrapper(this.router, this.name, 'post', this.extLogger);
   }
 
   get delete() {
-    return new ExpressRouteWrapper(this.router, this.name, 'delete');
+    return new ExpressRouteWrapper(this.router, this.name, 'delete', this.extLogger);
   }
 
 }
@@ -41,7 +43,8 @@ export class ExpressRouteWrapper implements IExtensionRESTRoute {
 
   constructor(private readonly router: express.Router,
               private readonly name: string,
-              private readonly func: 'get' | 'use' | 'put' | 'post' | 'delete') {
+              private readonly func: 'get' | 'use' | 'put' | 'post' | 'delete',
+              private readonly extLogger: ILogger) {
   }
 
   private getAuthMWs(minRole: UserRoles) {
@@ -59,14 +62,14 @@ export class ExpressRouteWrapper implements IExtensionRESTRoute {
         },
         RenderingMWs.renderResult
       ])));
-    Logger.silly(`[ExtensionRest:${this.name}]`, `Listening on ${this.func} ${ExtensionManager.EXTENSION_API_PATH}${fullPaths}`);
+    this.extLogger.silly(`Listening on ${this.func} ${ExtensionManager.EXTENSION_API_PATH}${fullPaths}`);
   }
 
   public rawMiddleware(paths: string[], minRole: UserRoles, mw: (req: Request, res: Response, next: NextFunction) => void | Promise<void>) {
-    const fullPaths = paths.map(p =>  (Utils.concatUrls('/' + this.name + '/' + p)));
+    const fullPaths = paths.map(p => (Utils.concatUrls('/' + this.name + '/' + p)));
     this.router[this.func](fullPaths,
       ...this.getAuthMWs(minRole),
       mw);
-    Logger.silly(`[ExtensionRest:${this.name}]`, `Listening on ${this.func} ${ExtensionManager.EXTENSION_API_PATH}${fullPaths}`);
+    this.extLogger.silly(`Listening on ${this.func} ${ExtensionManager.EXTENSION_API_PATH}${fullPaths}`);
   }
 }

src/backend/model/extension/ExtensionApp.ts

@@ -0,0 +1,18 @@
+import {IExtensionApp} from './IExtension';
+import {ObjectManagers} from '../ObjectManagers';
+import {Config} from '../../../common/config/private/Config';
+import {Server} from '../../server';
+
+export class ExtensionApp implements IExtensionApp {
+  get config() {
+    return Config;
+  }
+
+  get expressApp() {
+    return Server.getInstance().app;
+  }
+
+  get objectManagers() {
+    return ObjectManagers.getInstance();
+  }
+}

src/backend/model/extension/ExtensionDB.ts

@@ -0,0 +1,26 @@
+import {IExtensionDB} from './IExtension';
+import {SQLConnection} from '../database/SQLConnection';
+import {Connection} from 'typeorm';
+import {ILogger} from '../../Logger';
+
+export class ExtensionDB implements IExtensionDB {
+
+  constructor(private readonly extLogger: ILogger) {
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  _getAllTables(): Function[] {
+    return SQLConnection.getEntries();
+  }
+
+  getSQLConnection(): Promise<Connection> {
+    return SQLConnection.getConnection();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  async setExtensionTables(tables: Function[]): Promise<void> {
+    this.extLogger.debug('Adding ' + tables?.length + ' extension tables to DB');
+    await SQLConnection.addEntries(tables);
+  }
+
+}

src/backend/model/extension/ExtensionManager.ts

@@ -5,11 +5,13 @@ import * as path from 'path';
 import {IObjectManager} from '../database/IObjectManager';
 import {createLoggerWrapper, Logger} from '../../Logger';
 import {IExtensionEvents, IExtensionObject, IServerExtension} from './IExtension';
-import {ObjectManagers} from '../ObjectManagers';
 import {Server} from '../../server';
 import {ExtensionEvent} from './ExtensionEvent';
 import {ExpressRouterWrapper} from './ExpressRouterWrapper';
 import * as express from 'express';
+import {ExtensionApp} from './ExtensionApp';
+import {ExtensionDB} from './ExtensionDB';
+import {SQLConnection} from '../database/SQLConnection';
 
 const LOG_TAG = '[ExtensionManager]';
 
@@ -17,36 +19,45 @@ export class ExtensionManager implements IObjectManager {
 
   public static EXTENSION_API_PATH = Config.Server.apiPath + '/extension';
 
-  events: IExtensionEvents = {
-    gallery: {
-      MetadataLoader: {
-        loadPhotoMetadata: new ExtensionEvent(),
-        loadVideoMetadata: new ExtensionEvent()
-      },
-      CoverManager: {
-        getCoverForDirectory: new ExtensionEvent(),
-        getCoverForAlbum: new ExtensionEvent(),
-        invalidateDirectoryCovers: new ExtensionEvent(),
-      },
-      DiskManager: {
-        scanDirectory: new ExtensionEvent()
-      },
-      ImageRenderer: {
-        render: new ExtensionEvent()
-      }
-    }
-  };
+  events: IExtensionEvents;
   extObjects: { [key: string]: IExtensionObject } = {};
   router: express.Router;
 
+  constructor() {
+    this.initEvents();
+  }
+
   public async init() {
     this.extObjects = {};
+    this.initEvents();
     this.router = express.Router();
     Server.getInstance().app.use(ExtensionManager.EXTENSION_API_PATH, this.router);
     this.loadExtensionsList();
     await this.initExtensions();
   }
 
+  private initEvents() {
+    this.events = {
+      gallery: {
+        MetadataLoader: {
+          loadPhotoMetadata: new ExtensionEvent(),
+          loadVideoMetadata: new ExtensionEvent()
+        },
+        CoverManager: {
+          getCoverForDirectory: new ExtensionEvent(),
+          getCoverForAlbum: new ExtensionEvent(),
+          invalidateDirectoryCovers: new ExtensionEvent(),
+        },
+        DiskManager: {
+          scanDirectory: new ExtensionEvent()
+        },
+        ImageRenderer: {
+          render: new ExtensionEvent()
+        }
+      }
+    };
+  }
+
   public loadExtensionsList() {
     Logger.debug(LOG_TAG, 'Loading extension list from ' + ProjectPath.ExtensionFolder);
     if (!fs.existsSync(ProjectPath.ExtensionFolder)) {
@@ -79,19 +90,14 @@ export class ExtensionManager implements IObjectManager {
 
   private createExtensionObject(name: string): IExtensionObject {
     if (!this.extObjects[name]) {
-      const rw = new ExpressRouterWrapper(this.router, name);
+      const logger = createLoggerWrapper(`[Extension][${name}]`);
       this.extObjects[name] = {
-        _app: {
-          get objectManagers() {
-            return ObjectManagers.getInstance();
-          },
-          expressApp: Server.getInstance().app,
-          config: Config
-        },
+        _app: new ExtensionApp(),
+        db: new ExtensionDB(logger),
         paths: ProjectPath,
-        Logger: createLoggerWrapper(`[Extension: ${name}]`),
+        Logger: logger,
         events: this.events,
-        RESTApi: rw
+        RESTApi: new ExpressRouterWrapper(this.router, name, logger)
       };
     }
     return this.extObjects[name];
@@ -104,6 +110,10 @@ export class ExtensionManager implements IObjectManager {
         await ext?.init(this.createExtensionObject(extName));
       }
     });
+    if (Config.Extensions.cleanUpUnusedTables) {
+      // Clean up tables after all Extension was initialized.
+      await SQLConnection.removeUnusedTables();
+    }
   }
 
   private async cleanUpExtensions() {
@@ -117,6 +127,7 @@ export class ExtensionManager implements IObjectManager {
 
 
   public async cleanUp() {
+    this.initEvents(); // reset events
     await this.cleanUpExtensions();
     Server.getInstance().app.use(ExtensionManager.EXTENSION_API_PATH, express.Router());
     this.extObjects = {};

src/backend/model/extension/IExtension.ts

@@ -6,6 +6,7 @@ import {ProjectPathClass} from '../../ProjectPath';
 import {ILogger} from '../../Logger';
 import {UserDTO, UserRoles} from '../../../common/entities/UserDTO';
 import {ParamsDictionary} from 'express-serve-static-core';
+import {Connection, EntitySchema} from 'typeorm';
 
 
 export type IExtensionBeforeEventHandler<I, O> = (input: { inputs: I }, event: { stopPropagation: boolean }) => Promise<{ inputs: I } | O>;
@@ -63,8 +64,20 @@ export interface IExtensionApp {
 }
 
 export interface IExtensionRESTRoute {
+  /**
+   * Sends a pigallery2 standard JSON object with payload or error message back to the client.
+   * @param paths
+   * @param minRole
+   * @param cb
+   */
   jsonResponse(paths: string[], minRole: UserRoles, cb: (params?: ParamsDictionary, body?: any, user?: UserDTO) => Promise<unknown> | unknown): void;
 
+  /**
+   * Exposes a standard expressjs middleware
+   * @param paths
+   * @param minRole
+   * @param mw
+   */
   rawMiddleware(paths: string[], minRole: UserRoles, mw: (req: Request, res: Response, next: NextFunction) => void | Promise<void>): void;
 }
 
@@ -76,11 +89,39 @@ export interface IExtensionRESTApi {
   delete: IExtensionRESTRoute;
 }
 
+export interface IExtensionDB {
+  /**
+   * Returns with a typeorm SQL connection
+   */
+  getSQLConnection(): Promise<Connection>;
+
+  /**
+   * Adds SQL tables to typeorm
+   * @param tables
+   */
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  setExtensionTables(tables: Function[]): Promise<void>;
+
+  /**
+   * Exposes all tables. You can use this if you van to have a foreign key to a built in table.
+   * Use with caution. This exposes the app's internal working.
+   */
+  // eslint-disable-next-line @typescript-eslint/ban-types
+  _getAllTables(): Function[];
+}
+
 export interface IExtensionObject {
   /**
-   * Inner functionality of the app. Use this wit caution
+   * Inner functionality of the app. Use this with caution.
+   * If you want to go deeper than the standard exposed APIs, you can try doing so here.
    */
   _app: IExtensionApp;
+
+  /**
+   * Create new SQL tables and access SQL connection
+   */
+  db: IExtensionDB;
+
   /**
    * Paths to the main components of the app.
    */
@@ -104,6 +145,10 @@ export interface IExtensionObject {
  * Extension interface. All extension is expected to implement and export these methods
  */
 export interface IServerExtension {
+  /**
+   * Extension init function. Extension should at minimum expose this function.
+   * @param extension
+   */
   init(extension: IExtensionObject): Promise<void>;
 
   cleanUp?: (extension: IExtensionObject) => Promise<void>;

src/common/config/private/PrivateConfig.ts

@@ -1019,6 +1019,15 @@ export class ServerServiceConfig extends ClientServiceConfig {
 export class ServerExtensionsConfig {
   @ConfigProperty({volatile: true})
   list: string[] = [];
+
+  @ConfigProperty({
+    tags: {
+      name: $localize`Clean up unused tables`,
+      priority: ConfigPriority.underTheHood,
+    },
+    description: $localize`Automatically removes all tables from the DB that are not used anymore.`,
+  })
+  cleanUpUnusedTables: boolean = true;
 }
 
 @SubConfigClass({softReadonly: true})