From 1d48ca073a9735dc5d5a591aaa6cc49da58fff71 Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Thu, 22 Aug 2024 07:44:50 +0530 Subject: [PATCH 1/8] add doc for new backend system for incremental ingestion provider Signed-off-by: zeshanziya --- .../README.md | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 0894d32031..20218c5f34 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -39,7 +39,7 @@ The Incremental Entity Provider backend is designed for data sources that provid 2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. 3. There must be sufficient storage in Postgres to handle the additional data. (Presumably, this is also true of sqlite, but it has only been tested with Postgres.) -## Installation +## Installation (Old Backend) 1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. 2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. @@ -95,6 +95,30 @@ export default async function createPlugin( } ``` +## Installation (New Backend) + +1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. + +2. Add the following code to the `packages/backend/src/index.ts` file: + +```diff +++import { catalogModuleCustomIncrementalIngestionProvider } from './extensions/catalogCustomIncrementalIngestion'; + + +const backend = createBackend(); + +++backend.add( +++ import( +++ '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha' +++ ), +++); + +// We have created this in section **Adding an Incremental Entity Provider to the catalog (New Backend)** +++ backend.add(catalogModuleCustomIncrementalIngestionProvider); + +backend.start(); +``` + ## Administrative Routes If you want to manage your incremental entity providers via REST endpoints, the following endpoints are available: @@ -307,7 +331,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProviderInstallation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. @@ -350,6 +374,77 @@ incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { }); ``` +## Adding an Incremental Entity Provider to the catalog (New Backend) + +We'll assume you followed the Installation instructions. Cretae a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. + +```ts +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha'; + +export const catalogModuleCustomIncrementalIngestionProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'custom-incremental-ingestion-provider', + register(env) { + env.registerInit({ + deps: { + incrementalBuilder: incrementalIngestionProvidersExtensionPoint, + config: coreServices.rootConfig, + }, + async init({ incrementalBuilder, config }) { + // Assuming the token for the API comes from config + const token = config.getString('myApiClient.token'); + const myEntityProvider = new MyIncrementalEntityProvider(token); + + const options = { + // How long should it attempt to read pages from the API in a + // single burst? Keep this short. The Incremental Entity Provider + // will attempt to read as many pages as it can in this time + burstLength: { seconds: 3 }, + + // How long should it wait between bursts? + burstInterval: { seconds: 3 }, + + // How long should it rest before re-ingesting again? + restLength: { day: 1 }, + + // Optional back-off configuration - how long should it wait to retry + // in the event of an error? + backoff: [ + { seconds: 5 }, + { seconds: 30 }, + { minutes: 10 }, + { hours: 3 }, + ], + + // Optional. Use this to prevent removal of entities above a given + // percentage. This can be helpful if a data source is flaky and + // sometimes returns a successful status, but fewer than expected + // assets to add or maintain in the catalog. + rejectRemovalsAbovePercentage: 5, + + // Optional. Similar to rejectRemovalsAbovePercentage, except it + // applies to complete, 100% failure of a data source. If true, + // a data source that returns a successful status but does not + // provide any assets to turn into entities will have its empty + // data set rejected. + rejectEmptySourceCollections: true, + }; + + incrementalBuilder.addProvider({ + provider: myEntityProvider, + options, + }); + }, + }); + }, + }); +``` + That's it!!! ## Error handling From 15e2d405b1a2be6fb80669f9981d75c893d1ef7c Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Thu, 22 Aug 2024 07:52:28 +0530 Subject: [PATCH 2/8] update installtion link ref Signed-off-by: zeshanziya --- .../catalog-backend-module-incremental-ingestion/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 20218c5f34..319ba19bdb 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -333,7 +333,7 @@ Now that you have your new Incremental Entity Provider, we can connect it to the ## Adding an Incremental Entity Provider to the catalog (Old Backend) -We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. +We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. ```ts const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); @@ -376,7 +376,7 @@ incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { ## Adding an Incremental Entity Provider to the catalog (New Backend) -We'll assume you followed the Installation instructions. Cretae a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. +We'll assume you followed the Installation instructions. Cretae a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. ```ts import { From 7fc3777b342c300e6ecbc06e41e7880ab0456d39 Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Thu, 22 Aug 2024 07:55:35 +0530 Subject: [PATCH 3/8] spell correction Signed-off-by: zeshanziya --- plugins/catalog-backend-module-incremental-ingestion/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 319ba19bdb..0a29a1d618 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -376,7 +376,7 @@ incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { ## Adding an Incremental Entity Provider to the catalog (New Backend) -We'll assume you followed the Installation instructions. Cretae a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. +We'll assume you followed the Installation instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. ```ts import { From 4b28e3917ec89c65a211cfd07b5a073eab9dbc16 Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Thu, 22 Aug 2024 07:59:36 +0530 Subject: [PATCH 4/8] add changeset Signed-off-by: zeshanziya --- .changeset/swift-seahorses-share.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/swift-seahorses-share.md diff --git a/.changeset/swift-seahorses-share.md b/.changeset/swift-seahorses-share.md new file mode 100644 index 0000000000..deaf2ab042 --- /dev/null +++ b/.changeset/swift-seahorses-share.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Updated the README to include documentation for the new backend support From 948dbc00c893f0862001c399e9037cc9b246124c Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Thu, 22 Aug 2024 16:04:03 +0530 Subject: [PATCH 5/8] move old backend doc at bottom Signed-off-by: zeshanziya --- .../README.md | 198 +++++++++--------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 0a29a1d618..3890f0247a 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -39,62 +39,6 @@ The Incremental Entity Provider backend is designed for data sources that provid 2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. 3. There must be sufficient storage in Postgres to handle the additional data. (Presumably, this is also true of sqlite, but it has only been tested with Postgres.) -## Installation (Old Backend) - -1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. -2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. - -```ts -const builder = CatalogBuilder.create(env); -// incremental builder receives builder because it'll register -// incremental entity providers with the builder -const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); -``` - -3. After building the regular `CatalogBuilder`, build the incremental builder: - -```ts -// Must be run first to ensure CatalogBuilder database migrations run before Incremental Entity Provider database migrations -const { processingEngine, router } = await builder.build(); - -// Returns an optional - but highly recommended - set of administrative routes -const { incrementalAdminRouter } = await incrementBuilder.build(); -``` - -The final result should look something like this, - -```ts -import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; -import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { Router } from 'express'; -import { Duration } from 'luxon'; -import { PluginEnvironment } from '../types'; - -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const builder = CatalogBuilder.create(env); - // incremental builder receives builder because it'll register - // incremental entity providers with the builder - const incrementalBuilder = await IncrementalCatalogBuilder.create( - env, - builder, - ); - - builder.addProcessor(new ScaffolderEntitiesProcessor()); - - const { processingEngine, router } = await builder.build(); - const { incrementalAdminRouter } = await incrementalBuilder.build(); - - router.use(incrementalAdminRouter); - - await processingEngine.start(); - - return router; -} -``` - ## Installation (New Backend) 1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. @@ -331,49 +275,6 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProviderInstallation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. - -```ts -const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); - -// Assuming the token for the API comes from config -const token = config.getString('myApiClient.token'); - -const myEntityProvider = new MyIncrementalEntityProvider(token); - -incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { - // How long should it attempt to read pages from the API in a - // single burst? Keep this short. The Incremental Entity Provider - // will attempt to read as many pages as it can in this time - burstLength: { seconds: 3 }, - - // How long should it wait between bursts? - burstInterval: { seconds: 3 }, - - // How long should it rest before re-ingesting again? - restLength: { day: 1 }, - - // Optional back-off configuration - how long should it wait to retry - // in the event of an error? - backoff: [{ seconds: 5 }, { seconds: 30 }, { minutes: 10 }, { hours: 3 }], - - // Optional. Use this to prevent removal of entities above a given - // percentage. This can be helpful if a data source is flaky and - // sometimes returns a successful status, but fewer than expected - // assets to add or maintain in the catalog. - rejectRemovalsAbovePercentage: 5, - - // Optional. Similar to rejectRemovalsAbovePercentage, except it - // applies to complete, 100% failure of a data source. If true, - // a data source that returns a successful status but does not - // provide any assets to turn into entities will have its empty - // data set rejected. - rejectEmptySourceCollections: true, -}); -``` - ## Adding an Incremental Entity Provider to the catalog (New Backend) We'll assume you followed the Installation instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. @@ -445,6 +346,105 @@ export const catalogModuleCustomIncrementalIngestionProvider = }); ``` +## Installation (Old Backend) + +1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. +2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. + +```ts +const builder = CatalogBuilder.create(env); +// incremental builder receives builder because it'll register +// incremental entity providers with the builder +const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); +``` + +3. After building the regular `CatalogBuilder`, build the incremental builder: + +```ts +// Must be run first to ensure CatalogBuilder database migrations run before Incremental Entity Provider database migrations +const { processingEngine, router } = await builder.build(); + +// Returns an optional - but highly recommended - set of administrative routes +const { incrementalAdminRouter } = await incrementBuilder.build(); +``` + +The final result should look something like this, + +```ts +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { Router } from 'express'; +import { Duration } from 'luxon'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + const builder = CatalogBuilder.create(env); + // incremental builder receives builder because it'll register + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create( + env, + builder, + ); + + builder.addProcessor(new ScaffolderEntitiesProcessor()); + + const { processingEngine, router } = await builder.build(); + const { incrementalAdminRouter } = await incrementalBuilder.build(); + + router.use(incrementalAdminRouter); + + await processingEngine.start(); + + return router; +} +``` + +## Adding an Incremental Entity Provider to the catalog (Old Backend) + +We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. + +```ts +const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); + +// Assuming the token for the API comes from config +const token = config.getString('myApiClient.token'); + +const myEntityProvider = new MyIncrementalEntityProvider(token); + +incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { + // How long should it attempt to read pages from the API in a + // single burst? Keep this short. The Incremental Entity Provider + // will attempt to read as many pages as it can in this time + burstLength: { seconds: 3 }, + + // How long should it wait between bursts? + burstInterval: { seconds: 3 }, + + // How long should it rest before re-ingesting again? + restLength: { day: 1 }, + + // Optional back-off configuration - how long should it wait to retry + // in the event of an error? + backoff: [{ seconds: 5 }, { seconds: 30 }, { minutes: 10 }, { hours: 3 }], + + // Optional. Use this to prevent removal of entities above a given + // percentage. This can be helpful if a data source is flaky and + // sometimes returns a successful status, but fewer than expected + // assets to add or maintain in the catalog. + rejectRemovalsAbovePercentage: 5, + + // Optional. Similar to rejectRemovalsAbovePercentage, except it + // applies to complete, 100% failure of a data source. If true, + // a data source that returns a successful status but does not + // provide any assets to turn into entities will have its empty + // data set rejected. + rejectEmptySourceCollections: true, +}); +``` + That's it!!! ## Error handling From cf6b898947960dc13a258851e4c02a986909ec61 Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Fri, 23 Aug 2024 11:06:31 +0530 Subject: [PATCH 6/8] add suggested changes Signed-off-by: zeshanziya --- .../README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 3890f0247a..37d1f98846 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -39,7 +39,7 @@ The Incremental Entity Provider backend is designed for data sources that provid 2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. 3. There must be sufficient storage in Postgres to handle the additional data. (Presumably, this is also true of sqlite, but it has only been tested with Postgres.) -## Installation (New Backend) +## Installation 1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. @@ -57,7 +57,7 @@ const backend = createBackend(); ++ ), ++); -// We have created this in section **Adding an Incremental Entity Provider to the catalog (New Backend)** +// We have created this in section **Adding an Incremental Entity Provider to the catalog** ++ backend.add(catalogModuleCustomIncrementalIngestionProvider); backend.start(); @@ -84,7 +84,7 @@ In all cases, `:provider` is the name of the incremental entity provider. ## Writing an Incremental Entity Provider -To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, check out the requirements section of this page. +To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, check out the [requirements](#requirements) section of this page. Here is the type definition for an Incremental Entity Provider. @@ -275,9 +275,9 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProviderInstallation instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. +We'll assume you followed the [Installation](#installation-new-backend) instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. ```ts import { @@ -348,6 +348,10 @@ export const catalogModuleCustomIncrementalIngestionProvider = ## Installation (Old Backend) +> **Note:** The old backend system is deprecated and will be removed soon. +> The [New Backend System](https://backstage.io/docs/backend-system/) has been the default since Backstage [version 1.24](https://backstage.io/docs/releases/v1.24.0). +> If you're still using the old backend, it's recommended that you [migrate](https://backstage.io/docs/backend-system/building-backends/migrating/). + 1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. 2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. @@ -404,7 +408,7 @@ export default async function createPlugin( ## Adding an Incremental Entity Provider to the catalog (Old Backend) -We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. +We'll assume you followed the [Installation](#installation-old-backend) instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. ```ts const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); From 9153833be442f2552563efb30d2f518cee211f0c Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Mon, 26 Aug 2024 07:57:45 +0530 Subject: [PATCH 7/8] remove old backend related docs Signed-off-by: zeshanziya --- .../README.md | 105 +----------------- 1 file changed, 1 insertion(+), 104 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 37d1f98846..15b84dd4a0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -277,7 +277,7 @@ Now that you have your new Incremental Entity Provider, we can connect it to the ## Adding an Incremental Entity Provider to the catalog -We'll assume you followed the [Installation](#installation-new-backend) instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. +We'll assume you followed the [Installation](#installation) instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`. ```ts import { @@ -346,109 +346,6 @@ export const catalogModuleCustomIncrementalIngestionProvider = }); ``` -## Installation (Old Backend) - -> **Note:** The old backend system is deprecated and will be removed soon. -> The [New Backend System](https://backstage.io/docs/backend-system/) has been the default since Backstage [version 1.24](https://backstage.io/docs/releases/v1.24.0). -> If you're still using the old backend, it's recommended that you [migrate](https://backstage.io/docs/backend-system/building-backends/migrating/). - -1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory. -2. In your catalog.ts, import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. - -```ts -const builder = CatalogBuilder.create(env); -// incremental builder receives builder because it'll register -// incremental entity providers with the builder -const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); -``` - -3. After building the regular `CatalogBuilder`, build the incremental builder: - -```ts -// Must be run first to ensure CatalogBuilder database migrations run before Incremental Entity Provider database migrations -const { processingEngine, router } = await builder.build(); - -// Returns an optional - but highly recommended - set of administrative routes -const { incrementalAdminRouter } = await incrementBuilder.build(); -``` - -The final result should look something like this, - -```ts -import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; -import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { Router } from 'express'; -import { Duration } from 'luxon'; -import { PluginEnvironment } from '../types'; - -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const builder = CatalogBuilder.create(env); - // incremental builder receives builder because it'll register - // incremental entity providers with the builder - const incrementalBuilder = await IncrementalCatalogBuilder.create( - env, - builder, - ); - - builder.addProcessor(new ScaffolderEntitiesProcessor()); - - const { processingEngine, router } = await builder.build(); - const { incrementalAdminRouter } = await incrementalBuilder.build(); - - router.use(incrementalAdminRouter); - - await processingEngine.start(); - - return router; -} -``` - -## Adding an Incremental Entity Provider to the catalog (Old Backend) - -We'll assume you followed the [Installation](#installation-old-backend) instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. - -```ts -const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); - -// Assuming the token for the API comes from config -const token = config.getString('myApiClient.token'); - -const myEntityProvider = new MyIncrementalEntityProvider(token); - -incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { - // How long should it attempt to read pages from the API in a - // single burst? Keep this short. The Incremental Entity Provider - // will attempt to read as many pages as it can in this time - burstLength: { seconds: 3 }, - - // How long should it wait between bursts? - burstInterval: { seconds: 3 }, - - // How long should it rest before re-ingesting again? - restLength: { day: 1 }, - - // Optional back-off configuration - how long should it wait to retry - // in the event of an error? - backoff: [{ seconds: 5 }, { seconds: 30 }, { minutes: 10 }, { hours: 3 }], - - // Optional. Use this to prevent removal of entities above a given - // percentage. This can be helpful if a data source is flaky and - // sometimes returns a successful status, but fewer than expected - // assets to add or maintain in the catalog. - rejectRemovalsAbovePercentage: 5, - - // Optional. Similar to rejectRemovalsAbovePercentage, except it - // applies to complete, 100% failure of a data source. If true, - // a data source that returns a successful status but does not - // provide any assets to turn into entities will have its empty - // data set rejected. - rejectEmptySourceCollections: true, -}); -``` - That's it!!! ## Error handling From 33701d9bc00cb73361ed6fc2f8a57ba1be8b390d Mon Sep 17 00:00:00 2001 From: zeshanziya Date: Mon, 26 Aug 2024 14:48:04 +0530 Subject: [PATCH 8/8] doc: remove double plus signs Signed-off-by: zeshanziya --- .../README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 15b84dd4a0..3b847d4ad7 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -46,19 +46,19 @@ The Incremental Entity Provider backend is designed for data sources that provid 2. Add the following code to the `packages/backend/src/index.ts` file: ```diff -++import { catalogModuleCustomIncrementalIngestionProvider } from './extensions/catalogCustomIncrementalIngestion'; ++ import { catalogModuleCustomIncrementalIngestionProvider } from './extensions/catalogCustomIncrementalIngestion'; const backend = createBackend(); -++backend.add( -++ import( -++ '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha' -++ ), -++); ++ backend.add( ++ import( ++ '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha' ++ ), ++ ); // We have created this in section **Adding an Incremental Entity Provider to the catalog** -++ backend.add(catalogModuleCustomIncrementalIngestionProvider); ++ backend.add(catalogModuleCustomIncrementalIngestionProvider); backend.start(); ```