From 16af3f6dacf300b9c30195e9ed0af2aa08378b33 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 29 Nov 2022 15:18:20 -0800 Subject: [PATCH] Removed unneeded annotation Signed-off-by: Damon Kaswell --- .../README.md | 5 ++--- .../src/engine/IncrementalIngestionEngine.ts | 8 +------- .../src/types.ts | 8 -------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index d14e33ea53..0a646e4857 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -13,11 +13,10 @@ Unfortunately, these two kinds of mutations are insufficient for very large data 3. Addressing the above two use case with `full` mutation is not an option on very large datasets because a `full` mutation requires that all entities are in memory to create a diff. If your data source has 100k+ records, this can easily cause your processes to run out of memory. 4. In cases when you can use `full` mutation, committing many entities into the processing pipeline fills up the processing queue and delays the processing of entities from other entity providers. -We created the Incremental Entity Provider to address all of the above issues. The Incremental Entity Provider addresses these issues with a combination of `delta` mutations and a mark-and-sweep mechanism. Instead of doing a single `full` mutation, it performs a series of bursts. At the end of each burst, the Incremental Entity Provider performs the following three operations, +We created the Incremental Entity Provider to address all of the above issues. The Incremental Entity Provider addresses these issues with a combination of `delta` mutations and a mark-and-sweep mechanism. Instead of doing a single `full` mutation, it performs a series of bursts. At the end of each burst, the Incremental Entity Provider performs the following operations, 1. Marks each received entity in the database. -2. Annotates each entity with `backstage/incremental-entity-provider: ` annotation. -3. Commits all of the entities with a `delta` mutation. +2. Commits all of the entities with a `delta` mutation. Incremental Entity Providers will wait a configurable interval before proceeding to the next burst. diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index a47cd8f58c..d64c1318ac 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -15,11 +15,7 @@ */ import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { - INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, - IterationEngine, - IterationEngineOptions, -} from '../types'; +import { IterationEngine, IterationEngineOptions } from '../types'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import type { AbortSignal } from 'node-abort-controller'; import { performance } from 'perf_hooks'; @@ -271,8 +267,6 @@ export class IncrementalIngestionEngine implements IterationEngine { ...deferred.entity.metadata, annotations: { ...deferred.entity.metadata.annotations, - [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: - this.options.provider.getProviderName(), }, }, }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index c2603eea4e..ec46721a51 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -32,14 +32,6 @@ import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; -/** - * Entity annotation containing the incremental entity provider. - * - * @public - */ -export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = - 'backstage.io/incremental-provider-name'; - /** * Ingest entities into the catalog in bite-sized chunks. *