Code hygiene and cleanup

Signed-off-by: Damon Kaswell <damon.kaswell1@hp.com>

plugins/incremental-ingestion-backend/README.md

@@ -42,7 +42,7 @@ The Incremental Entity Provider backend is designed for data sources that provid
 
 ## Installation
 
-1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn add @backstage/plugin-incremental-ingestion-backend`
+1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn workspace backend add @backstage/plugin-incremental-ingestion-backend`
 2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`.
 
 ```ts

plugins/incremental-ingestion-backend/api-report.md

@@ -10,21 +10,19 @@ import type { Config } from '@backstage/config';
 import type { DeferredEntity } from '@backstage/plugin-catalog-backend';
 import { Duration } from 'luxon';
 import type { DurationObjectUnits } from 'luxon';
-import type { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { Knex } from 'knex';
 import type { Logger } from 'winston';
 import type { PermissionAuthorizer } from '@backstage/plugin-permission-common';
 import type { PluginDatabaseManager } from '@backstage/backend-common';
 import type { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { Router } from 'express';
-import type { TaskFunction } from '@backstage/backend-tasks';
 import type { UrlReader } from '@backstage/backend-common';
 
 // @public
 export interface EntityIteratorResult<T> {
-  cursor: T;
+  cursor?: T;
   done: boolean;
-  entities: DeferredEntity[];
+  entities?: DeferredEntity[];
 }
 
 // @public
@@ -198,30 +196,6 @@ export interface IngestionUpsertIFace {
     | 'backing off';
 }
 
-// @public (undocumented)
-export interface IterationEngine {
-  // (undocumented)
-  taskFn: TaskFunction;
-}
-
-// @public (undocumented)
-export interface IterationEngineOptions {
-  // (undocumented)
-  backoff?: IncrementalEntityProviderOptions['backoff'];
-  // (undocumented)
-  connection: EntityProviderConnection;
-  // (undocumented)
-  logger: Logger;
-  // (undocumented)
-  manager: IncrementalIngestionDatabaseManager;
-  // (undocumented)
-  provider: IncrementalEntityProvider<unknown, unknown>;
-  // (undocumented)
-  ready: Promise<void>;
-  // (undocumented)
-  restLength: DurationObjectUnits;
-}
-
 // @public
 export interface MarkRecord {
   // (undocumented)

plugins/incremental-ingestion-backend/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@backstage/plugin-incremental-ingestion-backend",
+  "description": "An entity provider for streaming large asset sources into the catalog",
   "version": "0.1.0",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "license": "Apache-2.0",
-  "private": true,
   "publishConfig": {
     "access": "public",
     "main": "dist/index.cjs.js",
@@ -30,7 +30,7 @@
     "@backstage/plugin-catalog-backend": "workspace:^",
     "@backstage/plugin-permission-common": "workspace:^",
     "@types/express": "^4.17.6",
-    "@types/luxon": "3.0.0",
+    "@types/luxon": "^3.0.0",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "knex": "^2.0.0",

plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts

@@ -27,6 +27,7 @@ import { performance } from 'perf_hooks';
 import { Duration, DurationObjectUnits } from 'luxon';
 import { v4 } from 'uuid';
 
+/** @public */
 export class IncrementalIngestionEngine implements IterationEngine {
   restLength: Duration;
   backoff: DurationObjectUnits[];
@@ -193,7 +194,7 @@ export class IncrementalIngestionEngine implements IterationEngine {
   async ingestOneBurst(id: string, signal: AbortSignal) {
     const lastMark = await this.manager.getLastMark(id);
 
-    const cursor = lastMark ? lastMark.cursor : void 0;
+    const cursor = lastMark ? lastMark.cursor : undefined;
     let sequence = lastMark ? lastMark.sequence + 1 : 0;
 
     const start = performance.now();
@@ -206,10 +207,15 @@ export class IncrementalIngestionEngine implements IterationEngine {
     await this.options.provider.around(async (context: unknown) => {
       let next = await this.options.provider.next(context, cursor);
       count++;
-      // eslint-disable-next-line no-constant-condition
-      while (true) {
+      for (;;) {
         done = next.done;
-        await this.mark(id, sequence, next.entities, next.done, next.cursor);
+        await this.mark({
+          id,
+          sequence,
+          entities: next?.entities,
+          done: next.done,
+          cursor: next?.cursor,
+        });
         if (signal.aborted || next.done) {
           break;
         } else {
@@ -228,17 +234,20 @@ export class IncrementalIngestionEngine implements IterationEngine {
     return done;
   }
 
-  async mark(
-    id: string,
-    sequence: number,
-    entities: DeferredEntity[],
-    done: boolean,
-    cursor?: unknown,
-  ) {
+  async mark(options: {
+    id: string;
+    sequence: number;
+    entities?: DeferredEntity[];
+    done: boolean;
+    cursor?: unknown;
+  }) {
+    const { id, sequence, entities, done, cursor } = options;
     this.options.logger.debug(
       `incremental-engine: Ingestion '${id}': MARK ${
-        entities.length
-      } entities, cursor: ${JSON.stringify(cursor)}, done: ${done}`,
+        entities ? entities.length : 0
+      } entities, cursor: ${
+        cursor ? JSON.stringify(cursor) : 'none'
+      }, done: ${done}`,
     );
     const markId = v4();
 
@@ -251,24 +260,25 @@ export class IncrementalIngestionEngine implements IterationEngine {
       },
     });
 
-    if (entities.length > 0) {
+    if (entities && entities.length > 0) {
       await this.manager.createMarkEntities(markId, entities);
     }
 
-    const added = entities.map(deferred => ({
-      ...deferred,
-      entity: {
-        ...deferred.entity,
-        metadata: {
-          ...deferred.entity.metadata,
-          annotations: {
-            ...deferred.entity.metadata.annotations,
-            [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]:
-              this.options.provider.getProviderName(),
+    const added =
+      entities?.map(deferred => ({
+        ...deferred,
+        entity: {
+          ...deferred.entity,
+          metadata: {
+            ...deferred.entity.metadata,
+            annotations: {
+              ...deferred.entity.metadata.annotations,
+              [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]:
+                this.options.provider.getProviderName(),
+            },
           },
         },
-      },
-    }));
+      })) ?? [];
 
     const removed: DeferredEntity[] = done
       ? []

plugins/incremental-ingestion-backend/src/index.ts

@@ -13,6 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from './service/IncrementalCatalogBuilder';
-export * from './types';
-export * from './database/IncrementalIngestionDatabaseManager';
+export { IncrementalCatalogBuilder } from './service/IncrementalCatalogBuilder';
+export type { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager';
+export type {
+  EntityIteratorResult,
+  IncrementalEntityProvider,
+  IncrementalEntityProviderOptions,
+  IngestionRecord,
+  IngestionRecordUpdate,
+  IngestionUpsertIFace,
+  MarkRecord,
+  MarkRecordInsert,
+  INCREMENTAL_ENTITY_PROVIDER_ANNOTATION,
+  PluginEnvironment,
+} from './types';

plugins/incremental-ingestion-backend/src/routes.ts

@@ -17,7 +17,7 @@ import { errorHandler } from '@backstage/backend-common';
 import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
-import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager';
+import { IncrementalIngestionDatabaseManager } from './';
 
 /** @public */
 export const createIncrementalProviderRouter = async (

plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts

@@ -27,12 +27,11 @@ import { IncrementalIngestionDatabaseManager } from '../database/IncrementalInge
 import { createIncrementalProviderRouter } from '../routes';
 
 class Deferred<T> implements Promise<T> {
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-ignore
-  resolve: (value: T) => void;
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-ignore
-  reject: (error: Error) => void;
+  #resolve?: (value: T) => void;
+  #reject?: (error: Error) => void;
+
+  get resolve() { return this.#resolve!; }
+  get reject() { return this.#reject!; }
 
   then: Promise<T>['then'];
   catch: Promise<T>['catch'];
@@ -40,8 +39,8 @@ class Deferred<T> implements Promise<T> {
 
   constructor() {
     const promise = new Promise<T>((resolve, reject) => {
-      this.resolve = resolve;
-      this.reject = reject;
+      this.#resolve = resolve;
+      this.#reject = reject;
     });
 
     this.then = promise.then.bind(promise);
@@ -98,9 +97,11 @@ export class IncrementalCatalogBuilder {
     options: IncrementalEntityProviderOptions,
   ) {
     const { burstInterval, burstLength, restLength } = options;
-    const { logger: catalogLogger, database, scheduler } = this.env;
+    const { logger: catalogLogger, scheduler } = this.env;
     const ready = this.ready;
 
+    const manager = this.manager;
+
     this.builder.addEntityProvider({
       getProviderName: provider.getProviderName.bind(provider),
       async connect(connection) {
@@ -110,10 +111,6 @@ export class IncrementalCatalogBuilder {
 
         logger.info(`Connecting`);
 
-        const client = await database.getClient();
-
-        const manager = new IncrementalIngestionDatabaseManager({ client });
-
         const engine = new IncrementalIngestionEngine({
           ...options,
           ready,

plugins/incremental-ingestion-backend/src/types.ts

@@ -29,7 +29,7 @@ import type {
 import type { PermissionAuthorizer } from '@backstage/plugin-permission-common';
 import type { DurationObjectUnits } from 'luxon';
 import type { Logger } from 'winston';
-import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager';
+import { IncrementalIngestionDatabaseManager } from './';
 /**
  * Entity annotation containing the incremental entity provider.
  *
@@ -100,12 +100,12 @@ export interface EntityIteratorResult<T> {
    * A value that marks the page of entities after this one. It will
    * be used to pass into the following invocation of `next()`
    */
-  cursor: T;
+  cursor?: T;
 
   /**
    * The entities to ingest.
    */
-  entities: DeferredEntity[];
+  entities?: DeferredEntity[];
 }
 
 /** @public */
@@ -225,7 +225,7 @@ export interface IngestionUpsertIFace {
 
 /**
  * This interface is for updating an existing ingestion record.
- *
+ * 
  * @public
  */
 export interface IngestionRecordUpdate {
@@ -235,7 +235,7 @@ export interface IngestionRecordUpdate {
 
 /**
  * The expected response from the `ingestion.ingestion_marks` table.
- *
+ * 
  * @public
  */
 export interface MarkRecord {
@@ -248,7 +248,7 @@ export interface MarkRecord {
 
 /**
  * The expected response from the `ingestion.ingestions` table.
- *
+ * 
  * @public
  */
 export interface IngestionRecord extends IngestionUpsertIFace {
@@ -262,7 +262,7 @@ export interface IngestionRecord extends IngestionUpsertIFace {
 
 /**
  * This interface supplies all the values for adding an ingestion mark.
- *
+ * 
  * @public
  */
 export interface MarkRecordInsert {