From c1f9764004456234c984dfc4a5f9fe73ecf949ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jerna=C5=9B?= Date: Wed, 9 Oct 2024 14:36:24 +0200 Subject: [PATCH 1/2] feat(catalog-backend): Add configuration parameters for deferred stitcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows for settings the stitch timeout and polling interval. Signed-off-by: Łukasz Jernaś --- .changeset/quiet-islands-learn.md | 5 +++++ plugins/catalog-backend/config.d.ts | 4 ++++ plugins/catalog-backend/src/stitching/types.ts | 10 +++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 .changeset/quiet-islands-learn.md diff --git a/.changeset/quiet-islands-learn.md b/.changeset/quiet-islands-learn.md new file mode 100644 index 0000000000..7e2ad6bdf1 --- /dev/null +++ b/.changeset/quiet-islands-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add configuration parameters for deferred stitcher diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 265d78b496..183607341c 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -156,6 +156,10 @@ export interface Config { | { /** Defer stitching to be performed asynchronously */ mode: 'deferred'; + /** Polling interval for tasks in seconds */ + pollingInterval?: number; + /** How long to wait for a stitch to complete before giving up in seconds */ + stitchTimeout?: number; }; /** diff --git a/plugins/catalog-backend/src/stitching/types.ts b/plugins/catalog-backend/src/stitching/types.ts index 9f7a5652dd..afeeff7335 100644 --- a/plugins/catalog-backend/src/stitching/types.ts +++ b/plugins/catalog-backend/src/stitching/types.ts @@ -66,11 +66,15 @@ export function stitchingStrategyFromConfig(config: Config): StitchingStrategy { mode: 'immediate', }; } else if (strategyMode === 'deferred') { - // TODO(freben): Make parameters configurable + const pollingInterval = + config.getOptionalNumber('catalog.stitchingStrategy.pollingInterval') ?? + 1; + const stitchTimeout = + config.getOptionalNumber('catalog.stitchingStrategy.stitchTimeout') ?? 60; return { mode: 'deferred', - pollingInterval: { seconds: 1 }, - stitchTimeout: { seconds: 60 }, + pollingInterval: { seconds: pollingInterval }, + stitchTimeout: { seconds: stitchTimeout }, }; } From 718ce10f57360db609a6456f7762920caa8d7ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Jerna=C5=9B?= Date: Wed, 9 Oct 2024 17:52:36 +0200 Subject: [PATCH 2/2] feat(catalog-backend): Added doc and review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Jernaś --- .../software-catalog/configuration.md | 36 +++++++++++++++++-- plugins/catalog-backend/config.d.ts | 4 +-- .../catalog-backend/src/stitching/types.ts | 24 ++++++++----- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/docs/features/software-catalog/configuration.md b/docs/features/software-catalog/configuration.md index dff09a9397..82cf80596f 100644 --- a/docs/features/software-catalog/configuration.md +++ b/docs/features/software-catalog/configuration.md @@ -165,7 +165,7 @@ catalog: ## Processing Interval -The [processing loop](https://backstage.io/docs/features/software-catalog/life-of-an-entity) is +The [processing loop](./life-of-an-entity.md#processing) is responsible for running your registered processors on all entities, on a certain interval. That interval can be configured with the `processingInterval` app-config parameter. @@ -192,13 +192,43 @@ Setting this value too low risks exhausting rate limits on external systems that are queried by processors, such as version control systems housing catalog-info files. +## Stitching strategy + +[Stitching](./life-of-an-entity.md#stitching) finalizes the entity. It can be run in +two modes: + +- `immediate` - performs stitching in-band immediately when needed +- `deferred` - performs the stitching asynchronously + +It can be configured with the `stitchingStrategy` app-config parameter. + +```yaml title="app-config.yaml" +catalog: + stitchingStrategy: immediate +``` + +For the `deferred` mode you can set up additional parameters to further tune the process, +by setting the following parameters: + +- `pollingInterval` - the interval between polling for entities that need stitching +- `stitchTimeout` - the maximum time to wait for an entity to be stitched + +These parameters accept a duration object, similar to the `processingInterval` parameter. + +```yaml title="app-config.yaml" +catalog: + stitchingStrategy: deferred + pollingInterval: { seconds: 1 } + stitchTimeout: { minutes: 1 }; +``` + ## Subscribing to Catalog Errors Catalog errors are published to the [events plugin](https://github.com/backstage/backstage/tree/master/plugins/events-node): `@backstage/plugin-events-node`. You can subscribe to events and respond to errors, for example you may wish to log them. The first step is to add the events backend plugin to your Backstage application. Navigate to your Backstage application directory and add the plugin package. -```ts title="From your Backstage root directory" +```bash title="From your Backstage root directory" yarn --cwd packages/backend add @backstage/plugin-events-backend ``` @@ -214,7 +244,7 @@ If you want to log catalog errors you can install the `@backstage/plugin-catalog Install the catalog logs module. -```ts title="From your Backstage root directory" +```bash title="From your Backstage root directory" yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-logs ``` diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 183607341c..8bab83d4b4 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -157,9 +157,9 @@ export interface Config { /** Defer stitching to be performed asynchronously */ mode: 'deferred'; /** Polling interval for tasks in seconds */ - pollingInterval?: number; + pollingInterval?: HumanDuration; /** How long to wait for a stitch to complete before giving up in seconds */ - stitchTimeout?: number; + stitchTimeout?: HumanDuration; }; /** diff --git a/plugins/catalog-backend/src/stitching/types.ts b/plugins/catalog-backend/src/stitching/types.ts index afeeff7335..7064dd1a1c 100644 --- a/plugins/catalog-backend/src/stitching/types.ts +++ b/plugins/catalog-backend/src/stitching/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; +import { Config, readDurationFromConfig } from '@backstage/config'; import { HumanDuration } from '@backstage/types'; /** @@ -66,15 +66,23 @@ export function stitchingStrategyFromConfig(config: Config): StitchingStrategy { mode: 'immediate', }; } else if (strategyMode === 'deferred') { - const pollingInterval = - config.getOptionalNumber('catalog.stitchingStrategy.pollingInterval') ?? - 1; - const stitchTimeout = - config.getOptionalNumber('catalog.stitchingStrategy.stitchTimeout') ?? 60; + const pollingIntervalKey = 'catalog.stitchingStrategy.pollingInterval'; + const stitchTimeoutKey = 'catalog.stitchingStrategy.stitchTimeout'; + + const pollingInterval = config.has(pollingIntervalKey) + ? readDurationFromConfig(config, { + key: pollingIntervalKey, + }) + : { seconds: 1 }; + const stitchTimeout = config.has(stitchTimeoutKey) + ? readDurationFromConfig(config, { + key: stitchTimeoutKey, + }) + : { seconds: 60 }; return { mode: 'deferred', - pollingInterval: { seconds: pollingInterval }, - stitchTimeout: { seconds: stitchTimeout }, + pollingInterval: pollingInterval, + stitchTimeout: stitchTimeout, }; }