From 2133fd2926594bec0523bc90152947e56f1dffab Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 10:09:02 -0700 Subject: [PATCH] Apply mandatory ts-doc annotations for API report Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 63 ++++++++++--------- .../src/database/migrations.ts | 1 + .../src/engine/IncrementalIngestionEngine.ts | 1 + .../src/routes.ts | 1 + .../src/service/IncrementalCatalogBuilder.ts | 7 ++- .../src/types.ts | 31 ++++++++- 6 files changed, 68 insertions(+), 36 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index d593bf4292..d7dc7dc1de 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -28,6 +28,7 @@ import { MarkRecordInsert, } from '../types'; +/** @public */ export class IncrementalIngestionDatabaseManager { private client: Knex; @@ -37,7 +38,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching `id`. - * @param options IngestionRecordUpdate + * @param options - IngestionRecordUpdate */ async updateIngestionRecordById(options: IngestionRecordUpdate) { await this.client.transaction(async tx => { @@ -48,8 +49,8 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching provider name. Will only update active records. - * @param provider string - * @param update Partial + * @param provider - string + * @param update - Partial */ async updateIngestionRecordByProvider( provider: string, @@ -65,7 +66,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestions` table with the supplied values. - * @param options IngestionRecordInsert + * @param options - IngestionRecordInsert */ async insertIngestionRecord(options: IngestionRecordInsert) { await this.client.transaction(async tx => { @@ -101,7 +102,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finds the current ingestion record for the named provider. - * @param provider string + * @param provider - string * @returns IngestionRecord | undefined */ async getCurrentIngestionRecord(provider: string) { @@ -117,7 +118,7 @@ export class IncrementalIngestionDatabaseManager { /** * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` * for prior ingestions that completed (i.e., have a value for `rest_completed_at`). - * @param provider string + * @param provider - string * @returns A count of deletions for each record type. */ async clearFinishedIngestions(provider: string) { @@ -166,8 +167,8 @@ export class IncrementalIngestionDatabaseManager { * Automatically cleans up duplicate ingestion records if they were accidentally created. * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND * the ingestionId is incorrect is a duplicate ingestion record. - * @param ingestionId string - * @param provider string + * @param ingestionId - string + * @param provider - string */ async clearDuplicateIngestions(ingestionId: string, provider: string) { await this.client.transaction(async tx => { @@ -196,7 +197,7 @@ export class IncrementalIngestionDatabaseManager { /** * This method fully purges and resets all ingestion records for the named provider, and * leaves it in a paused state. - * @param provider string + * @param provider - string * @returns Counts of all deleted ingestion records */ async purgeAndResetProvider(provider: string) { @@ -264,7 +265,7 @@ export class IncrementalIngestionDatabaseManager { /** * Creates a new ingestion record. - * @param provider string + * @param provider - string * @returns A new ingestion record */ async createProviderIngestionRecord(provider: string) { @@ -283,8 +284,8 @@ export class IncrementalIngestionDatabaseManager { /** * Computes which entities to remove, if any, at the end of a burst. - * @param provider string - * @param ingestionId string + * @param provider - string + * @param ingestionId - string * @returns All entities to remove for this burst. */ async computeRemoved(provider: string, ingestionId: string) { @@ -339,7 +340,7 @@ export class IncrementalIngestionDatabaseManager { /** * Skips any wait time for the next action to run. - * @param provider string + * @param provider - string */ async triggerNextProviderAction(provider: string) { await this.updateIngestionRecordByProvider(provider, { @@ -389,7 +390,7 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion record to ingest a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderIngesting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -400,7 +401,7 @@ export class IncrementalIngestionDatabaseManager { /** * Indicates the provider is currently ingesting a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderBursting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -411,7 +412,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. - * @param ingestionId string + * @param ingestionId - string */ async setProviderComplete(ingestionId: string) { await this.updateIngestionRecordById({ @@ -426,8 +427,8 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as complete and starts the post-ingestion rest cycle. - * @param ingestionId string - * @param restLength Duration + * @param ingestionId - string + * @param restLength - Duration */ async setProviderResting(ingestionId: string, restLength: Duration) { await this.updateIngestionRecordById({ @@ -443,7 +444,7 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as paused after a burst completes. - * @param ingestionId string + * @param ingestionId - string */ async setProviderInterstitial(ingestionId: string) { await this.updateIngestionRecordById({ @@ -454,8 +455,8 @@ export class IncrementalIngestionDatabaseManager { /** * Starts the cancel process for the current ingestion. - * @param ingestionId string - * @param message string (optional) + * @param ingestionId - string + * @param message - string (optional) */ async setProviderCanceling(ingestionId: string, message?: string) { const update: Partial = { @@ -469,7 +470,7 @@ export class IncrementalIngestionDatabaseManager { /** * Completes the cancel process and triggers a new ingestion. - * @param ingestionId string + * @param ingestionId - string */ async setProviderCanceled(ingestionId: string) { await this.updateIngestionRecordById({ @@ -484,10 +485,10 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion to wait and retry, due to a data source error. - * @param ingestionId string - * @param attempts number - * @param error Error - * @param backoffLength number + * @param ingestionId - string + * @param attempts - number + * @param error - Error + * @param backoffLength - number */ async setProviderBackoff( ingestionId: string, @@ -509,7 +510,7 @@ export class IncrementalIngestionDatabaseManager { /** * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. - * @param ingestionId string + * @param ingestionId - string * @returns MarkRecord | undefined */ async getLastMark(ingestionId: string) { @@ -533,7 +534,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. - * @param options MarkRecordInsert + * @param options - MarkRecordInsert */ async createMark(options: MarkRecordInsert) { const { record } = options; @@ -543,8 +544,8 @@ export class IncrementalIngestionDatabaseManager { } /** * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. - * @param markId string - * @param entities DeferredEntity[] + * @param markId - string + * @param entities - DeferredEntity[] */ async createMarkEntities(markId: string, entities: DeferredEntity[]) { await this.client.transaction(async tx => { @@ -560,7 +561,7 @@ export class IncrementalIngestionDatabaseManager { /** * Deletes the entire content of a table, and returns the number of records deleted. - * @param table string + * @param table - string * @returns number */ async purgeTable(table: string) { diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 991a709732..270f07b0d7 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -16,6 +16,7 @@ import { resolvePackagePath } from '@backstage/backend-common'; import { Knex } from 'knex'; +/** @public */ export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( '@devex/backend-incremental-ingestion', diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 655380cec4..574d678e90 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -26,6 +26,7 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; +/** @public */ export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; backoff: DurationObjectUnits[]; diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts index a35f02e98e..81ebc1d7b6 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -19,6 +19,7 @@ import Router from 'express-promise-router'; import { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; +/** @public */ export const createIncrementalProviderRouter = async ( manager: IncrementalIngestionDatabaseManager, logger: Logger, diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index a9babf17e4..e6e1e05091 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -26,7 +26,7 @@ import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { createIncrementalProviderRouter } from '../routes'; -export class Deferred implements Promise { +class Deferred implements Promise { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore resolve: (value: T) => void; @@ -52,11 +52,12 @@ export class Deferred implements Promise { [Symbol.toStringTag]: 'Deferred' = 'Deferred'; } +/** @public */ export class IncrementalCatalogBuilder { /** * Creates the incremental catalog builder, which extends the regular catalog builder. - * @param env PluginEnvironment - * @param builder CatalogBuilder + * @param env - PluginEnvironment + * @param builder - CatalogBuilder * @returns IncrementalCatalogBuilder */ static async create(env: PluginEnvironment, builder: CoreCatalogBuilder) { diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 85f184b52e..12c7249aa2 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -51,6 +51,7 @@ export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = * batches of entities in sequence so that you never need to have more * than a few hundred in memory at a time. * + * @public */ export interface IncrementalEntityProvider { /** @@ -84,13 +85,17 @@ export interface IncrementalEntityProvider { } /** - * Value returned by an @{link IncrementalEntityProvider} to provide a + * Value returned by an {@link IncrementalEntityProvider} to provide a * single page of entities to ingest. + * + * @public */ export interface EntityIteratorResult { /** * Indicates whether there are any further pages of entities to * ingest after this one. + * + * @public */ done: boolean; @@ -106,6 +111,7 @@ export interface EntityIteratorResult { entities: DeferredEntity[]; } +/** @public */ export interface IncrementalEntityProviderOptions { /** * Entities are ingested in bursts. This interval determines how @@ -129,11 +135,16 @@ export interface IncrementalEntityProviderOptions { /** * In the event of an error during an ingestion burst, the backoff * determines how soon it will be retried. E.g. - * [{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }] + * `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]` */ backoff?: DurationObjectUnits[]; } +/** + * Provides the environment used by the incremental entity provider, + * + * @public + */ export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; @@ -145,6 +156,8 @@ export type PluginEnvironment = { /** * The core ingestion engine implements this interface + * + * @public */ export interface IterationEngine { taskFn: TaskFunction; @@ -152,6 +165,8 @@ export interface IterationEngine { /** * Options passed to the core ingestion engine during initialization. + * + * @public */ export interface IterationEngineOptions { logger: Logger; @@ -165,6 +180,8 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. + * + * @public */ export interface IngestionUpsertIFace { next_action: @@ -191,6 +208,8 @@ export interface IngestionUpsertIFace { /** * This interface supplies all potential values that can be inserted into the `ingestion.ingestions` table. + * + * @public */ export interface IngestionRecordInsert { record: IngestionUpsertIFace & { @@ -200,6 +219,8 @@ export interface IngestionRecordInsert { /** * This interface is for updating an existing ingestion record. + * + * @public */ export interface IngestionRecordUpdate { ingestionId: string; @@ -208,6 +229,8 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. + * + * @public */ export interface MarkRecord { id: string; @@ -219,6 +242,8 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. + * + * @public */ export interface IngestionRecord { id: string; @@ -235,6 +260,8 @@ export interface IngestionRecord { /** * This interface supplies all the values for adding an ingestion mark. + * + * @public */ export interface MarkRecordInsert { record: {