Merge pull request #26228 from zeshanziya/doc-increment-entity-provider
doc: add Incremental Entity Provider to main docs for better visibility
This commit is contained in:
Fredrik Adelöw
@@ -1031,3 +1031,292 @@ backend.add(catalogModuleCustomDataParser);
|
||||
|
||||
backend.start();
|
||||
```
|
||||
|
||||
## Incremental Entity Provider
|
||||
|
||||
For large data sources that may not fit into memory but support pagination, the Incremental Entity Provider offers an efficient way to ingest data incrementally, handling deletions and updates seamlessly while minimizing memory usage.
|
||||
|
||||
You can find more details about [why it was created](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-incremental-ingestion#why-did-we-create-it) and its [requirements](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-incremental-ingestion#requirements).
|
||||
|
||||
### Installation
|
||||
|
||||
1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` from the Backstage root directory.
|
||||
|
||||
2. Add the following code to the `packages/backend/src/index.ts` file:
|
||||
|
||||
```ts title="packages/backend/src/index.ts"
|
||||
const backend = createBackend();
|
||||
|
||||
/* highlight-add-start */
|
||||
backend.add(
|
||||
import(
|
||||
'@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha'
|
||||
),
|
||||
);
|
||||
/* highlight-add-end */
|
||||
|
||||
backend.start();
|
||||
```
|
||||
|
||||
### Writing an Incremental Entity Provider
|
||||
|
||||
To create an Incremental Entity Provider, you need to know how to retrieve a single page of data from an API with pagination. The `IncrementalEntityProvider` facilitates this by requiring:
|
||||
|
||||
- **getProviderName:** A unique name to avoid conflicts with other providers.
|
||||
- **next:** Fetches a specific page of entities, moving the cursor forward.
|
||||
- **around:** Handles setup and tear-down, wrapping the process that iterates through multiple pages.
|
||||
|
||||
For more information on compatibility, refer to the [requirements](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-incremental-ingestion#requirements).
|
||||
|
||||
In this tutorial, we'll implement an Incremental Entity Provider that interacts with an imaginary API to fetch a list of imaginary services.
|
||||
|
||||
```ts
|
||||
interface MyApiClient {
|
||||
getServices(page: number): MyPaginatedResults<Service>;
|
||||
}
|
||||
|
||||
interface MyPaginatedResults<T> {
|
||||
items: T[];
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
interface Service {
|
||||
name: string;
|
||||
}
|
||||
```
|
||||
|
||||
These are the only 3 methods that you need to implement. `getProviderName()` is pretty self explanatory and it's identical to the `getProviderName()` method on a regular Entity Provider.
|
||||
|
||||
```ts
|
||||
import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
|
||||
|
||||
// This will include your pagination information, let's say our API accepts a `page` parameter.
|
||||
// In this case, the cursor will include `page`
|
||||
interface Cursor {
|
||||
page: number;
|
||||
}
|
||||
|
||||
// This interface describes the type of data that will be passed to your burst function.
|
||||
interface Context {
|
||||
apiClient: MyApiClient;
|
||||
}
|
||||
|
||||
export class MyIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<Cursor, Context>
|
||||
{
|
||||
getProviderName() {
|
||||
return `MyIncrementalEntityProvider`;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`around` method is used for setup and tear-down. For example, if you need to create a client that will connect to the API, you would do that here.
|
||||
|
||||
```ts
|
||||
export class MyIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<Cursor, Context>
|
||||
{
|
||||
getProviderName() {
|
||||
return `MyIncrementalEntityProvider`;
|
||||
}
|
||||
|
||||
async around(burst: (context: Context) => Promise<void>): Promise<void> {
|
||||
const apiClient = new MyApiClient();
|
||||
|
||||
await burst({ apiClient });
|
||||
|
||||
// If you need to do any teardown, you can do it here.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If you need to pass a token to your API, then you can create a constructor that will receive a token and use the token to setup the client.
|
||||
|
||||
```ts
|
||||
export class MyIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<Cursor, Context>
|
||||
{
|
||||
private readonly token: string;
|
||||
|
||||
constructor(token: string) {
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
getProviderName() {
|
||||
return `MyIncrementalEntityProvider`;
|
||||
}
|
||||
|
||||
async around(burst: (context: Context) => Promise<void>): Promise<void> {
|
||||
const apiClient = new MyApiClient(this.token);
|
||||
|
||||
await burst({ apiClient });
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The last step is to implement the actual `next` method that will accept the cursor, call the API, process the result and return the result.
|
||||
|
||||
```ts
|
||||
export class MyIncrementalEntityProvider
|
||||
implements IncrementalEntityProvider<Cursor, Context>
|
||||
{
|
||||
private readonly token: string;
|
||||
|
||||
constructor(token: string) {
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
getProviderName() {
|
||||
return `MyIncrementalEntityProvider`;
|
||||
}
|
||||
|
||||
async around(burst: (context: Context) => Promise<void>): Promise<void> {
|
||||
const apiClient = new MyApiClient(this.token);
|
||||
|
||||
await burst({ apiClient });
|
||||
}
|
||||
|
||||
async next(
|
||||
context: Context,
|
||||
cursor?: Cursor = { page: 1 },
|
||||
): Promise<EntityIteratorResult<Cursor>> {
|
||||
const { apiClient } = context;
|
||||
|
||||
// call your API with the current cursor
|
||||
const data = await apiClient.getServices(cursor);
|
||||
|
||||
// calculate the next page
|
||||
const nextPage = page + 1;
|
||||
|
||||
// figure out if there are any more pages to fetch
|
||||
const done = nextPage > data.totalPages;
|
||||
|
||||
// convert returned items into entities
|
||||
const entities = data.items.map(item => ({
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Component',
|
||||
metadata: {
|
||||
name: item.name,
|
||||
annotations: {
|
||||
// You need to define these, otherwise they'll fail validation
|
||||
[ANNOTATION_LOCATION]: this.getProviderName(),
|
||||
[ANNOTATION_ORIGIN_LOCATION]: this.getProviderName(),
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'service',
|
||||
lifecycle: 'production', // Ideally your source has this information
|
||||
owner: 'unknown', // Ideally your source has this information
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
// create the next cursor
|
||||
const nextCursor = {
|
||||
page: nextPage,
|
||||
};
|
||||
|
||||
return {
|
||||
done,
|
||||
entities,
|
||||
cursor: nextCursor,
|
||||
};
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Now that you have your new Incremental Entity Provider, we can connect it to the catalog.
|
||||
|
||||
### Installing the Incremental Entity Provider
|
||||
|
||||
We'll assume you followed the [Installation](#installation) instructions. Now create a module inside `packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts`.
|
||||
|
||||
```ts title="packages/backend/src/extensions/catalogCustomIncrementalIngestion.ts"
|
||||
import {
|
||||
coreServices,
|
||||
createBackendModule,
|
||||
} from '@backstage/backend-plugin-api';
|
||||
import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha';
|
||||
|
||||
export const catalogModuleCustomIncrementalIngestionProvider =
|
||||
createBackendModule({
|
||||
pluginId: 'catalog',
|
||||
moduleId: 'custom-incremental-ingestion-provider',
|
||||
register(env) {
|
||||
env.registerInit({
|
||||
deps: {
|
||||
incrementalBuilder: incrementalIngestionProvidersExtensionPoint,
|
||||
config: coreServices.rootConfig,
|
||||
},
|
||||
async init({ incrementalBuilder, config }) {
|
||||
// Assuming the token for the API comes from config
|
||||
const token = config.getString('myApiClient.token');
|
||||
const myEntityProvider = new MyIncrementalEntityProvider(token);
|
||||
|
||||
const options = {
|
||||
// How long should it attempt to read pages from the API in a
|
||||
// single burst? Keep this short. The Incremental Entity Provider
|
||||
// will attempt to read as many pages as it can in this time
|
||||
burstLength: { seconds: 3 },
|
||||
|
||||
// How long should it wait between bursts?
|
||||
burstInterval: { seconds: 3 },
|
||||
|
||||
// How long should it rest before re-ingesting again?
|
||||
restLength: { day: 1 },
|
||||
|
||||
// Optional back-off configuration - how long should it wait to retry
|
||||
// in the event of an error?
|
||||
backoff: [
|
||||
{ seconds: 5 },
|
||||
{ seconds: 30 },
|
||||
{ minutes: 10 },
|
||||
{ hours: 3 },
|
||||
],
|
||||
|
||||
// Optional. Use this to prevent removal of entities above a given
|
||||
// percentage. This can be helpful if a data source is flaky and
|
||||
// sometimes returns a successful status, but fewer than expected
|
||||
// assets to add or maintain in the catalog.
|
||||
rejectRemovalsAbovePercentage: 5,
|
||||
|
||||
// Optional. Similar to rejectRemovalsAbovePercentage, except it
|
||||
// applies to complete, 100% failure of a data source. If true,
|
||||
// a data source that returns a successful status but does not
|
||||
// provide any assets to turn into entities will have its empty
|
||||
// data set rejected.
|
||||
rejectEmptySourceCollections: true,
|
||||
};
|
||||
|
||||
incrementalBuilder.addProvider({
|
||||
provider: myEntityProvider,
|
||||
options,
|
||||
});
|
||||
},
|
||||
});
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
Add the module to `packages/backend/src/index.ts`
|
||||
|
||||
```ts title="packages/backend/src/index.ts"
|
||||
/* highlight-add-next-line */
|
||||
import { catalogModuleCustomIncrementalIngestionProvider } from './extensions/catalogCustomIncrementalIngestion';
|
||||
|
||||
const backend = createBackend();
|
||||
|
||||
backend.add(
|
||||
import(
|
||||
'@backstage/plugin-catalog-backend-module-incremental-ingestion/alpha'
|
||||
),
|
||||
);
|
||||
|
||||
/* highlight-add-next-line */
|
||||
backend.add(catalogModuleCustomIncrementalIngestionProvider);
|
||||
|
||||
backend.start();
|
||||
```
|
||||
|
||||
For a deep dive into the technical details of the Incremental Entity Provider, see [the README](https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-incremental-ingestion).
|
||||
|
||||
Reference in New Issue
Block a user