From 1acd16ecbea406bded68cbc194d1e99ff0306ee5 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 10 Oct 2024 09:41:07 +0200 Subject: [PATCH 1/5] feat: added the ability to disable catalog processing Signed-off-by: blam --- plugins/catalog-backend/config.d.ts | 16 +++++++++++++++- .../catalog-backend/src/service/CatalogPlugin.ts | 12 ++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 8bab83d4b4..57b31305c4 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -164,7 +164,6 @@ export interface Config { /** * The interval at which the catalog should process its entities. - * * @remarks * * Example: @@ -186,5 +185,20 @@ export interface Config { * housing catalog-info files. */ processingInterval?: HumanDuration; + /** + * If the processing engine should be started or not, defaults to true. + * @remarks + * + * Example: + * + * ```yaml + * catalog: + * processingInterval: false + * ``` + * + * This will enable or disable the processing of entities in the Catalog. This can be useful + * to use with Read Replica deployments of the catalog. + */ + processingEnabled?: boolean; }; } diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index e36e4e1333..b39898afb2 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -42,6 +42,7 @@ import { import { merge } from 'lodash'; import { Permission } from '@backstage/plugin-permission-common'; import { ForwardedError } from '@backstage/errors'; +import { constrainedMemory } from 'node:process'; class CatalogLocationsExtensionPointImpl implements CatalogLocationsExtensionPoint @@ -301,10 +302,13 @@ export const catalogPlugin = createBackendPlugin({ const { processingEngine, router } = await builder.build(); - lifecycle.addStartupHook(async () => { - await processingEngine.start(); - }); - lifecycle.addShutdownHook(() => processingEngine.stop()); + if (config.getOptionalBoolean('catalog.processingEnabled') ?? true) { + lifecycle.addStartupHook(async () => { + await processingEngine.start(); + }); + lifecycle.addShutdownHook(() => processingEngine.stop()); + } + httpRouter.use(router); }, }); From d1cf90ac31f22383e2a2d08dfd512a95e0700582 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 10 Oct 2024 09:44:11 +0200 Subject: [PATCH 2/5] chore: add changeset Signed-off-by: blam Signed-off-by: blam --- .changeset/silly-ligers-tan.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silly-ligers-tan.md diff --git a/.changeset/silly-ligers-tan.md b/.changeset/silly-ligers-tan.md new file mode 100644 index 0000000000..d5b666a07b --- /dev/null +++ b/.changeset/silly-ligers-tan.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Adds the ability to disable catalog processing with `catalog.processingEnabled` config flag From e79dcef6e1673197f8c2c3527eb767713b7206a6 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 14 Oct 2024 11:59:04 +0200 Subject: [PATCH 3/5] chore: support processingInterval as false Signed-off-by: blam --- .changeset/silly-ligers-tan.md | 2 +- plugins/catalog-backend/config.d.ts | 10 ++++++++-- plugins/catalog-backend/src/service/CatalogBuilder.ts | 9 +++++++++ plugins/catalog-backend/src/service/CatalogPlugin.ts | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.changeset/silly-ligers-tan.md b/.changeset/silly-ligers-tan.md index d5b666a07b..6b9097497d 100644 --- a/.changeset/silly-ligers-tan.md +++ b/.changeset/silly-ligers-tan.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Adds the ability to disable catalog processing with `catalog.processingEnabled` config flag +Adds the ability to disable catalog processing `catalog.processingInterval: false` in `app-config` diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 57b31305c4..2f8ec3f230 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -173,6 +173,13 @@ export interface Config { * processingInterval: { minutes: 30 } * ``` * + * or to disabled processing: + * + * ```yaml + * catalog: + * processingInterval: false + * ``` + * * Note that this is only a suggested minimum, and the actual interval may * be longer. Internally, the catalog will scale up this number by a small * factor and choose random numbers in that range to spread out the load. If @@ -184,7 +191,7 @@ export interface Config { * systems that are queried by processors, such as version control systems * housing catalog-info files. */ - processingInterval?: HumanDuration; + processingInterval?: HumanDuration | false; /** * If the processing engine should be started or not, defaults to true. * @remarks @@ -199,6 +206,5 @@ export interface Config { * This will enable or disable the processing of entities in the Catalog. This can be useful * to use with Read Replica deployments of the catalog. */ - processingEnabled?: boolean; }; } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index c404bfaa0d..31ec40b887 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -843,9 +843,18 @@ export class CatalogBuilder { }); } + if (!Boolean(config.get('catalog.processingInterval'))) { + return () => { + throw new Error( + 'catalog.processingInterval is set to false, processing is disabled.', + ); + }; + } + const duration = readDurationFromConfig(config, { key: processingIntervalKey, }); + const seconds = Math.max( 1, Math.round(durationToMilliseconds(duration) / 1000), diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index b39898afb2..98c99e38dc 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -302,7 +302,7 @@ export const catalogPlugin = createBackendPlugin({ const { processingEngine, router } = await builder.build(); - if (config.getOptionalBoolean('catalog.processingEnabled') ?? true) { + if (config.get('catalog.processingInterval') ?? true) { lifecycle.addStartupHook(async () => { await processingEngine.start(); }); From 09aa2b14495774b78ad475fd87bcce9ac0f9121a Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 14 Oct 2024 12:02:29 +0200 Subject: [PATCH 4/5] chore: fix extra config docs Signed-off-by: blam Signed-off-by: blam --- plugins/catalog-backend/config.d.ts | 14 -------------- .../catalog-backend/src/service/CatalogPlugin.ts | 1 - 2 files changed, 15 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 2f8ec3f230..1ab107ef10 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -192,19 +192,5 @@ export interface Config { * housing catalog-info files. */ processingInterval?: HumanDuration | false; - /** - * If the processing engine should be started or not, defaults to true. - * @remarks - * - * Example: - * - * ```yaml - * catalog: - * processingInterval: false - * ``` - * - * This will enable or disable the processing of entities in the Catalog. This can be useful - * to use with Read Replica deployments of the catalog. - */ }; } diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index 98c99e38dc..dfe415afa6 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -42,7 +42,6 @@ import { import { merge } from 'lodash'; import { Permission } from '@backstage/plugin-permission-common'; import { ForwardedError } from '@backstage/errors'; -import { constrainedMemory } from 'node:process'; class CatalogLocationsExtensionPointImpl implements CatalogLocationsExtensionPoint From 4aad556a5ebd1b77474e436adc0f2c6a50040af8 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 14 Oct 2024 15:20:25 +0200 Subject: [PATCH 5/5] chore: woops - this should be optional Signed-off-by: blam --- plugins/catalog-backend/src/service/CatalogPlugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index dfe415afa6..4da0b612cd 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -301,7 +301,7 @@ export const catalogPlugin = createBackendPlugin({ const { processingEngine, router } = await builder.build(); - if (config.get('catalog.processingInterval') ?? true) { + if (config.getOptional('catalog.processingInterval') ?? true) { lifecycle.addStartupHook(async () => { await processingEngine.start(); });