From 965d84af25746ace20f6894e9dc97f4fd3bd665b Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 13:28:00 +0100 Subject: [PATCH 01/20] Pass in event broker to notify on conflicts Signed-off-by: sblausten --- packages/backend/src/plugins/catalog.ts | 14 ++++++++++++++ plugins/catalog-backend/package.json | 1 + .../src/database/DefaultProcessingDatabase.ts | 14 ++++++++++++++ .../catalog-backend/src/service/CatalogBuilder.ts | 8 ++++++++ plugins/events-backend/src/index.ts | 1 + yarn.lock | 4 +++- 6 files changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index f6fe25f86a..53204d6957 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,14 +19,28 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; +import { InMemoryEventBroker } from '@backstage/plugin-events-backend'; +import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; export default async function createPlugin( env: PluginEnvironment, providers?: Array, ): Promise { const builder = await CatalogBuilder.create(env); + const githubEntityProviders = GithubEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 5 }, + timeout: { minutes: 3 }, + }), + scheduler: env.scheduler, + }); + builder.addEntityProvider(githubEntityProviders); builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); + const conflictBroker = new InMemoryEventBroker(env.logger); + builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); return router; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 1e837fd9ba..6805a0d21e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -54,6 +54,7 @@ "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-permission-node": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 761f5bfd72..4fe24eab97 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -44,6 +44,7 @@ import { checkLocationKeyConflict } from './operations/refreshState/checkLocatio import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; import { generateStableHash } from './util'; +import { EventBroker } from '@backstage/plugin-events-node'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -57,6 +58,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { database: Knex; logger: Logger; refreshInterval: ProcessingIntervalFunction; + conflictEventBroker?: EventBroker; }, ) { initDatabaseMetrics(options.database); @@ -356,6 +358,18 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); + if (this.options.conflictEventBroker) { + this.options.conflictEventBroker?.publish({ + topic: 'conflicting-entities', + eventPayload: { + ...entity, + entityRef, + newLocationKey: locationKey, + existingLocationKey: conflictingKey, + }, + metadata: {}, + }); + } } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 7da5d69767..d0584477ab 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -98,6 +98,7 @@ import { import { AuthorizedLocationService } from './AuthorizedLocationService'; import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; +import { EventBroker } from '@backstage/plugin-events-node'; /** * This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API. @@ -166,6 +167,7 @@ export class CatalogBuilder { private readonly permissionRules: CatalogPermissionRuleInput[]; private allowedLocationType: string[]; private legacySingleProcessorValidation = false; + private conflictEventBroker?: EventBroker; /** * Creates a catalog builder. @@ -417,6 +419,11 @@ export class CatalogBuilder { return this; } + setConflictEventBroker(broker: EventBroker): CatalogBuilder { + this.conflictEventBroker = broker; + return this; + } + /** * Wires up and returns all of the component parts of the catalog */ @@ -440,6 +447,7 @@ export class CatalogBuilder { database: dbClient, logger, refreshInterval: this.processingInterval, + conflictEventBroker: this.conflictEventBroker, }); const providerDatabase = new DefaultProviderDatabase({ database: dbClient, diff --git a/plugins/events-backend/src/index.ts b/plugins/events-backend/src/index.ts index 4645e9a479..215ea2c203 100644 --- a/plugins/events-backend/src/index.ts +++ b/plugins/events-backend/src/index.ts @@ -21,4 +21,5 @@ */ export { EventsBackend } from './service/EventsBackend'; +export { InMemoryEventBroker } from './service/InMemoryEventBroker'; export { HttpPostIngressEventPublisher } from './service/http'; diff --git a/yarn.lock b/yarn.lock index adc0ba595e..f078ec6138 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5006,7 +5006,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": +"@backstage/plugin-catalog-backend-module-github@workspace:^, @backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: @@ -5204,6 +5204,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-permission-node": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" @@ -22508,6 +22509,7 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-entity-feedback-backend": "workspace:^" From 9d3bd37defa152e1c423d070571f1a7be61d1d31 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 16:05:44 +0100 Subject: [PATCH 02/20] Working example Signed-off-by: sblausten --- packages/backend/package.json | 1 + packages/backend/src/index.ts | 6 +-- packages/backend/src/plugins/catalog.ts | 4 ++ .../src/plugins/conflictEventSubscriber.ts | 50 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 4 +- 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 packages/backend/src/plugins/conflictEventSubscriber.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 54dbe640df..d91428ac9e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,6 +40,7 @@ "@backstage/plugin-azure-sites-backend": "workspace:^", "@backstage/plugin-badges-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-backend-module-github": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-code-coverage-backend": "workspace:^", "@backstage/plugin-entity-feedback-backend": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 011be941cf..cfd18f243a 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -38,11 +38,10 @@ import { import { TaskScheduler } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; -import { metricsInit, metricsHandler } from './metrics'; +import { metricsHandler, metricsInit } from './metrics'; import auth from './plugins/auth'; import azureDevOps from './plugins/azure-devops'; import catalog from './plugins/catalog'; -import catalogEventBasedProviders from './plugins/catalogEventBasedProviders'; import codeCoverage from './plugins/codecoverage'; import entityFeedback from './plugins/entityFeedback'; import events from './plugins/events'; @@ -68,6 +67,7 @@ import linguist from './plugins/linguist'; import { PluginEnvironment } from './types'; import { ServerPermissionClient } from '@backstage/plugin-permission-node'; import { DefaultIdentityClient } from '@backstage/plugin-auth-node'; +import createCatalogEventBasedProviders from './plugins/catalogEventBasedProviders'; function makeCreateEnv(config: Config) { const root = getRootLogger(); @@ -156,7 +156,7 @@ async function main() { const exploreEnv = useHotMemoize(module, () => createEnv('explore')); const lighthouseEnv = useHotMemoize(module, () => createEnv('lighthouse')); - const eventBasedEntityProviders = await catalogEventBasedProviders( + const eventBasedEntityProviders = await createCatalogEventBasedProviders( catalogEnv, ); const linguistEnv = useHotMemoize(module, () => createEnv('linguist')); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 53204d6957..a70e56d301 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -21,6 +21,7 @@ import { Router } from 'express'; import { PluginEnvironment } from '../types'; import { InMemoryEventBroker } from '@backstage/plugin-events-backend'; import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; +import { ConflictEventSubscriber } from './conflictEventSubscriber'; export default async function createPlugin( env: PluginEnvironment, @@ -40,6 +41,9 @@ export default async function createPlugin( builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); const conflictBroker = new InMemoryEventBroker(env.logger); + conflictBroker.subscribe( + new ConflictEventSubscriber(env.logger, ['conflicts']), + ); builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); diff --git a/packages/backend/src/plugins/conflictEventSubscriber.ts b/packages/backend/src/plugins/conflictEventSubscriber.ts new file mode 100644 index 0000000000..746d01cca6 --- /dev/null +++ b/packages/backend/src/plugins/conflictEventSubscriber.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; +import { Logger } from 'winston'; + +export class ConflictEventSubscriber + implements EntityProvider, EventSubscriber +{ + constructor( + private readonly logger: Logger, + private readonly topics: string[], + ) {} + + async onEvent(params: EventParams): Promise { + this.logger.info( + `onEvent: topic=${params.topic}, metadata=${JSON.stringify( + params.metadata, + )}, payload=${JSON.stringify(params.eventPayload)}`, + ); + } + + supportsEventTopics(): string[] { + return this.topics; + } + + async connect(_: EntityProviderConnection): Promise { + // not doing anything here + } + + getProviderName(): string { + return ConflictEventSubscriber.name; + } +} diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 4fe24eab97..f7e9a0c81c 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -359,8 +359,8 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); if (this.options.conflictEventBroker) { - this.options.conflictEventBroker?.publish({ - topic: 'conflicting-entities', + await this.options.conflictEventBroker?.publish({ + topic: 'conflicts', eventPayload: { ...entity, entityRef, From d38a97ed74bb2b3caf91b9239c70192299c20963 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 16:11:26 +0100 Subject: [PATCH 03/20] Remove example code Signed-off-by: sblausten --- packages/backend/package.json | 1 - packages/backend/src/index.ts | 6 +-- packages/backend/src/plugins/catalog.ts | 18 ------- .../src/plugins/conflictEventSubscriber.ts | 50 ------------------- 4 files changed, 3 insertions(+), 72 deletions(-) delete mode 100644 packages/backend/src/plugins/conflictEventSubscriber.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index d91428ac9e..54dbe640df 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,7 +40,6 @@ "@backstage/plugin-azure-sites-backend": "workspace:^", "@backstage/plugin-badges-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", - "@backstage/plugin-catalog-backend-module-github": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-code-coverage-backend": "workspace:^", "@backstage/plugin-entity-feedback-backend": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index cfd18f243a..011be941cf 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -38,10 +38,11 @@ import { import { TaskScheduler } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; import healthcheck from './plugins/healthcheck'; -import { metricsHandler, metricsInit } from './metrics'; +import { metricsInit, metricsHandler } from './metrics'; import auth from './plugins/auth'; import azureDevOps from './plugins/azure-devops'; import catalog from './plugins/catalog'; +import catalogEventBasedProviders from './plugins/catalogEventBasedProviders'; import codeCoverage from './plugins/codecoverage'; import entityFeedback from './plugins/entityFeedback'; import events from './plugins/events'; @@ -67,7 +68,6 @@ import linguist from './plugins/linguist'; import { PluginEnvironment } from './types'; import { ServerPermissionClient } from '@backstage/plugin-permission-node'; import { DefaultIdentityClient } from '@backstage/plugin-auth-node'; -import createCatalogEventBasedProviders from './plugins/catalogEventBasedProviders'; function makeCreateEnv(config: Config) { const root = getRootLogger(); @@ -156,7 +156,7 @@ async function main() { const exploreEnv = useHotMemoize(module, () => createEnv('explore')); const lighthouseEnv = useHotMemoize(module, () => createEnv('lighthouse')); - const eventBasedEntityProviders = await createCatalogEventBasedProviders( + const eventBasedEntityProviders = await catalogEventBasedProviders( catalogEnv, ); const linguistEnv = useHotMemoize(module, () => createEnv('linguist')); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index a70e56d301..f6fe25f86a 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,32 +19,14 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; -import { InMemoryEventBroker } from '@backstage/plugin-events-backend'; -import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; -import { ConflictEventSubscriber } from './conflictEventSubscriber'; export default async function createPlugin( env: PluginEnvironment, providers?: Array, ): Promise { const builder = await CatalogBuilder.create(env); - const githubEntityProviders = GithubEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 5 }, - timeout: { minutes: 3 }, - }), - scheduler: env.scheduler, - }); - builder.addEntityProvider(githubEntityProviders); builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); - const conflictBroker = new InMemoryEventBroker(env.logger); - conflictBroker.subscribe( - new ConflictEventSubscriber(env.logger, ['conflicts']), - ); - builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); return router; diff --git a/packages/backend/src/plugins/conflictEventSubscriber.ts b/packages/backend/src/plugins/conflictEventSubscriber.ts deleted file mode 100644 index 746d01cca6..0000000000 --- a/packages/backend/src/plugins/conflictEventSubscriber.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - EntityProvider, - EntityProviderConnection, -} from '@backstage/plugin-catalog-node'; -import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -import { Logger } from 'winston'; - -export class ConflictEventSubscriber - implements EntityProvider, EventSubscriber -{ - constructor( - private readonly logger: Logger, - private readonly topics: string[], - ) {} - - async onEvent(params: EventParams): Promise { - this.logger.info( - `onEvent: topic=${params.topic}, metadata=${JSON.stringify( - params.metadata, - )}, payload=${JSON.stringify(params.eventPayload)}`, - ); - } - - supportsEventTopics(): string[] { - return this.topics; - } - - async connect(_: EntityProviderConnection): Promise { - // not doing anything here - } - - getProviderName(): string { - return ConflictEventSubscriber.name; - } -} From 44c7ad6b8e117cf5a689ca3191694acf22f9006f Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 16:18:05 +0100 Subject: [PATCH 04/20] Api reports and changeset Signed-off-by: sblausten --- .changeset/sweet-fans-juggle.md | 6 ++++++ plugins/catalog-backend/api-report.md | 3 +++ plugins/events-backend/api-report.md | 12 ++++++++++++ .../src/service/InMemoryEventBroker.ts | 2 ++ yarn.lock | 3 +-- 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 .changeset/sweet-fans-juggle.md diff --git a/.changeset/sweet-fans-juggle.md b/.changeset/sweet-fans-juggle.md new file mode 100644 index 0000000000..e90e4643f8 --- /dev/null +++ b/.changeset/sweet-fans-juggle.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-events-backend': patch +--- + +Adds an optional eventBroker for sending an event when there are conflicts, and exports InMemoryBroker for use in other plugins. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 5e7b2dee02..e22aa622fb 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -31,6 +31,7 @@ import { EntityProvider as EntityProvider_2 } from '@backstage/plugin-catalog-no import { EntityProviderConnection as EntityProviderConnection_2 } from '@backstage/plugin-catalog-node'; import { EntityProviderMutation as EntityProviderMutation_2 } from '@backstage/plugin-catalog-node'; import { EntityRelationSpec as EntityRelationSpec_2 } from '@backstage/plugin-catalog-node'; +import { EventBroker } from '@backstage/plugin-events-node'; import { GetEntitiesRequest } from '@backstage/catalog-client'; import { JsonValue } from '@backstage/types'; import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common'; @@ -139,6 +140,8 @@ export class CatalogBuilder { replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder; replaceProcessors(processors: CatalogProcessor_2[]): CatalogBuilder; setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder; + // (undocumented) + setConflictEventBroker(broker: EventBroker): CatalogBuilder; setEntityDataParser(parser: CatalogProcessorParser_2): CatalogBuilder; setFieldFormatValidators(validators: Partial): CatalogBuilder; setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder; diff --git a/plugins/events-backend/api-report.md b/plugins/events-backend/api-report.md index 7d8ede1979..5d520f10c2 100644 --- a/plugins/events-backend/api-report.md +++ b/plugins/events-backend/api-report.md @@ -5,6 +5,7 @@ ```ts import { Config } from '@backstage/config'; import { EventBroker } from '@backstage/plugin-events-node'; +import { EventParams } from '@backstage/plugin-events-node'; import { EventPublisher } from '@backstage/plugin-events-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; import express from 'express'; @@ -42,4 +43,15 @@ export class HttpPostIngressEventPublisher implements EventPublisher { // (undocumented) setEventBroker(eventBroker: EventBroker): Promise; } + +// @public +export class InMemoryEventBroker implements EventBroker { + constructor(logger: Logger); + // (undocumented) + publish(params: EventParams): Promise; + // (undocumented) + subscribe( + ...subscribers: Array> + ): void; +} ``` diff --git a/plugins/events-backend/src/service/InMemoryEventBroker.ts b/plugins/events-backend/src/service/InMemoryEventBroker.ts index 04be01f658..d800d88dc5 100644 --- a/plugins/events-backend/src/service/InMemoryEventBroker.ts +++ b/plugins/events-backend/src/service/InMemoryEventBroker.ts @@ -25,6 +25,8 @@ import { Logger } from 'winston'; * In-memory event broker which will pass the event to all registered subscribers * interested in it. * Events will not be persisted in any form. + * + * @public */ // TODO(pjungermann): add prom metrics? (see plugins/catalog-backend/src/util/metrics.ts, etc.) export class InMemoryEventBroker implements EventBroker { diff --git a/yarn.lock b/yarn.lock index f078ec6138..5ad56a501c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5006,7 +5006,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-github@workspace:^, @backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": +"@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: @@ -22509,7 +22509,6 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-entity-feedback-backend": "workspace:^" From 3844a954bd925869fbcde2d4f3a486769bff2028 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 16:24:58 +0100 Subject: [PATCH 05/20] Add detail to api report Signed-off-by: sblausten --- plugins/catalog-backend/api-report.md | 1 - plugins/catalog-backend/src/service/CatalogBuilder.ts | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index e22aa622fb..7babcc5bf6 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -140,7 +140,6 @@ export class CatalogBuilder { replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder; replaceProcessors(processors: CatalogProcessor_2[]): CatalogBuilder; setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder; - // (undocumented) setConflictEventBroker(broker: EventBroker): CatalogBuilder; setEntityDataParser(parser: CatalogProcessorParser_2): CatalogBuilder; setFieldFormatValidators(validators: Partial): CatalogBuilder; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index d0584477ab..dc6a5ab533 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -419,6 +419,10 @@ export class CatalogBuilder { return this; } + /** + * Enables the publishing of events for conflicting entities so that a subscriber + * can listen to them and implement a conflict resolution strategy like retries. + */ setConflictEventBroker(broker: EventBroker): CatalogBuilder { this.conflictEventBroker = broker; return this; From a6ef0169d3f430db3c5d77674e95240eabf33d64 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 20 Mar 2023 16:30:55 +0100 Subject: [PATCH 06/20] Fix changeset Signed-off-by: sblausten --- .changeset/sweet-fans-juggle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/sweet-fans-juggle.md b/.changeset/sweet-fans-juggle.md index e90e4643f8..db8abd8ca6 100644 --- a/.changeset/sweet-fans-juggle.md +++ b/.changeset/sweet-fans-juggle.md @@ -3,4 +3,4 @@ '@backstage/plugin-events-backend': patch --- -Adds an optional eventBroker for sending an event when there are conflicts, and exports InMemoryBroker for use in other plugins. +Adds an optional `EventBroker` for sending an event when there are conflicts, and exports `InMemoryBroker` for use in other plugins. From 8b9073e2c71e1eef81a9a90a081ecbf423dd7cdf Mon Sep 17 00:00:00 2001 From: sblausten Date: Tue, 21 Mar 2023 15:50:59 +0100 Subject: [PATCH 07/20] PR comments Signed-off-by: sblausten --- plugins/catalog-backend/api-report.md | 2 +- .../src/database/DefaultProcessingDatabase.ts | 10 +++++----- plugins/catalog-backend/src/service/CatalogBuilder.ts | 11 +++++------ 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7babcc5bf6..f93f2d354f 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -140,8 +140,8 @@ export class CatalogBuilder { replaceEntityPolicies(policies: EntityPolicy[]): CatalogBuilder; replaceProcessors(processors: CatalogProcessor_2[]): CatalogBuilder; setAllowedLocationTypes(allowedLocationTypes: string[]): CatalogBuilder; - setConflictEventBroker(broker: EventBroker): CatalogBuilder; setEntityDataParser(parser: CatalogProcessorParser_2): CatalogBuilder; + setEventBroker(broker: EventBroker): CatalogBuilder; setFieldFormatValidators(validators: Partial): CatalogBuilder; setLocationAnalyzer(locationAnalyzer: LocationAnalyzer): CatalogBuilder; setPlaceholderResolver( diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index f7e9a0c81c..f5159f4010 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -58,7 +58,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { database: Knex; logger: Logger; refreshInterval: ProcessingIntervalFunction; - conflictEventBroker?: EventBroker; + eventBroker?: EventBroker; }, ) { initDatabaseMetrics(options.database); @@ -358,11 +358,11 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); - if (this.options.conflictEventBroker) { - await this.options.conflictEventBroker?.publish({ - topic: 'conflicts', + if (this.options.eventBroker) { + await this.options.eventBroker?.publish({ + topic: 'catalog.conflicts', eventPayload: { - ...entity, + entity, entityRef, newLocationKey: locationKey, existingLocationKey: conflictingKey, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index dc6a5ab533..59a5afce38 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -167,7 +167,7 @@ export class CatalogBuilder { private readonly permissionRules: CatalogPermissionRuleInput[]; private allowedLocationType: string[]; private legacySingleProcessorValidation = false; - private conflictEventBroker?: EventBroker; + private eventBroker?: EventBroker; /** * Creates a catalog builder. @@ -420,11 +420,10 @@ export class CatalogBuilder { } /** - * Enables the publishing of events for conflicting entities so that a subscriber - * can listen to them and implement a conflict resolution strategy like retries. + * Enables the publishing of events for cloflicts in the DefaultProcessingDatabase */ - setConflictEventBroker(broker: EventBroker): CatalogBuilder { - this.conflictEventBroker = broker; + setEventBroker(broker: EventBroker): CatalogBuilder { + this.eventBroker = broker; return this; } @@ -451,7 +450,7 @@ export class CatalogBuilder { database: dbClient, logger, refreshInterval: this.processingInterval, - conflictEventBroker: this.conflictEventBroker, + eventBroker: this.eventBroker, }); const providerDatabase = new DefaultProviderDatabase({ database: dbClient, From 2632cda41aa74e0658af9117a0c9c00250be6d81 Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 22 Mar 2023 09:59:51 +0100 Subject: [PATCH 08/20] Add timestamp to event Signed-off-by: sblausten --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index f5159f4010..64bc16db6f 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -45,6 +45,7 @@ import { insertUnprocessedEntity } from './operations/refreshState/insertUnproce import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; import { generateStableHash } from './util'; import { EventBroker } from '@backstage/plugin-events-node'; +import { DateTime } from 'luxon'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -366,6 +367,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { entityRef, newLocationKey: locationKey, existingLocationKey: conflictingKey, + lastConflictAt: DateTime.now().toISO(), }, metadata: {}, }); From e70e36d72c9085ba8c5d2a306a378f734b2f1c54 Mon Sep 17 00:00:00 2001 From: sblausten Date: Wed, 22 Mar 2023 10:00:28 +0100 Subject: [PATCH 09/20] Remove unused metadata field Signed-off-by: sblausten --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 64bc16db6f..2a51227983 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -369,7 +369,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { existingLocationKey: conflictingKey, lastConflictAt: DateTime.now().toISO(), }, - metadata: {}, }); } } From e544eb27137282c3892a0499cca8dc185ca2d76c Mon Sep 17 00:00:00 2001 From: sblausten Date: Thu, 30 Mar 2023 13:00:31 +0200 Subject: [PATCH 10/20] Export conflict topic name Signed-off-by: sblausten --- packages/backend/src/plugins/catalog.ts | 16 ++++++++++++++++ plugins/catalog-backend/api-report.md | 3 +++ .../src/database/DefaultProcessingDatabase.ts | 3 ++- .../src/service/CatalogBuilder.ts | 3 +++ plugins/catalog-backend/src/service/index.ts | 2 +- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index f6fe25f86a..8cf44f9722 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,14 +19,30 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; +import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; export default async function createPlugin( env: PluginEnvironment, providers?: Array, ): Promise { const builder = await CatalogBuilder.create(env); + const githubEntityProviders = GithubEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 5 }, + timeout: { minutes: 3 }, + }), + scheduler: env.scheduler, + }); + builder.addEntityProvider(githubEntityProviders); builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); + // const conflictBroker = new InMemoryEventBroker(env.logger); + // conflictBroker.subscribe( + // new ConflictEventSubscriber(env.logger, ['conflicts']), + // ); + // builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); return router; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index eae8c0b9e4..1317481046 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -167,6 +167,9 @@ export class CatalogBuilder { export type CatalogCollatorEntityTransformer = CatalogCollatorEntityTransformer_2; +// @public (undocumented) +export const catalogConflictsTopic = 'catalog.conflict'; + // @public (undocumented) export type CatalogEnvironment = { logger: Logger; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 2a51227983..4b701758ec 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -46,6 +46,7 @@ import { updateUnprocessedEntity } from './operations/refreshState/updateUnproce import { generateStableHash } from './util'; import { EventBroker } from '@backstage/plugin-events-node'; import { DateTime } from 'luxon'; +import { catalogConflictsTopic } from '../service'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -361,7 +362,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); if (this.options.eventBroker) { await this.options.eventBroker?.publish({ - topic: 'catalog.conflicts', + topic: catalogConflictsTopic, eventPayload: { entity, entityRef, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index c0d5c7acc7..39a40c95de 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -118,6 +118,9 @@ export type CatalogEnvironment = { permissions: PermissionEvaluator | PermissionAuthorizer; }; +/** @public */ +export const catalogConflictsTopic = 'catalog.conflict'; + /** * A builder that helps wire up all of the component parts of the catalog. * diff --git a/plugins/catalog-backend/src/service/index.ts b/plugins/catalog-backend/src/service/index.ts index 44645d39fe..70c65cf08f 100644 --- a/plugins/catalog-backend/src/service/index.ts +++ b/plugins/catalog-backend/src/service/index.ts @@ -18,4 +18,4 @@ export type { CatalogEnvironment, CatalogPermissionRuleInput, } from './CatalogBuilder'; -export { CatalogBuilder } from './CatalogBuilder'; +export { CatalogBuilder, catalogConflictsTopic } from './CatalogBuilder'; From 7565b498320769df467ce13c2e6698d21283bce3 Mon Sep 17 00:00:00 2001 From: sblausten Date: Thu, 30 Mar 2023 13:04:42 +0200 Subject: [PATCH 11/20] Remove test code Signed-off-by: sblausten --- packages/backend/src/plugins/catalog.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 8cf44f9722..f6fe25f86a 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,30 +19,14 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; -import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; export default async function createPlugin( env: PluginEnvironment, providers?: Array, ): Promise { const builder = await CatalogBuilder.create(env); - const githubEntityProviders = GithubEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 5 }, - timeout: { minutes: 3 }, - }), - scheduler: env.scheduler, - }); - builder.addEntityProvider(githubEntityProviders); builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); - // const conflictBroker = new InMemoryEventBroker(env.logger); - // conflictBroker.subscribe( - // new ConflictEventSubscriber(env.logger, ['conflicts']), - // ); - // builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); return router; From fb8c2971ae83e3b5e391520858bc98f3be1e222f Mon Sep 17 00:00:00 2001 From: sblausten Date: Thu, 30 Mar 2023 15:35:53 +0200 Subject: [PATCH 12/20] Change constant export Signed-off-by: sblausten --- plugins/catalog-backend/api-report.md | 6 +++--- plugins/catalog-backend/src/constants.ts | 18 ++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 4 ++-- plugins/catalog-backend/src/index.ts | 1 + .../src/service/CatalogBuilder.ts | 3 --- plugins/catalog-backend/src/service/index.ts | 2 +- 6 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 plugins/catalog-backend/src/constants.ts diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 1317481046..59d198a18a 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -113,6 +113,9 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor_2 { validateEntityKind(entity: Entity): Promise; } +// @public (undocumented) +export const CATALOG_CONFLICTS_TOPIC = 'catalog.conflict'; + // @public export class CatalogBuilder { addEntityPolicy( @@ -167,9 +170,6 @@ export class CatalogBuilder { export type CatalogCollatorEntityTransformer = CatalogCollatorEntityTransformer_2; -// @public (undocumented) -export const catalogConflictsTopic = 'catalog.conflict'; - // @public (undocumented) export type CatalogEnvironment = { logger: Logger; diff --git a/plugins/catalog-backend/src/constants.ts b/plugins/catalog-backend/src/constants.ts new file mode 100644 index 0000000000..2ba21da629 --- /dev/null +++ b/plugins/catalog-backend/src/constants.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** @public */ +export const CATALOG_CONFLICTS_TOPIC = 'catalog.conflict'; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 4b701758ec..e634d18c4a 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -46,7 +46,7 @@ import { updateUnprocessedEntity } from './operations/refreshState/updateUnproce import { generateStableHash } from './util'; import { EventBroker } from '@backstage/plugin-events-node'; import { DateTime } from 'luxon'; -import { catalogConflictsTopic } from '../service'; +import { CATALOG_CONFLICTS_TOPIC } from '../constants'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -362,7 +362,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { ); if (this.options.eventBroker) { await this.options.eventBroker?.publish({ - topic: catalogConflictsTopic, + topic: CATALOG_CONFLICTS_TOPIC, eventPayload: { entity, entityRef, diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index eca155ce46..345102adcd 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -27,6 +27,7 @@ export * from './processing'; export * from './search'; export * from './service'; export * from './deprecated'; +export * from './constants'; import { DefaultCatalogCollatorFactory as _DefaultCatalogCollatorFactory, diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 39a40c95de..c0d5c7acc7 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -118,9 +118,6 @@ export type CatalogEnvironment = { permissions: PermissionEvaluator | PermissionAuthorizer; }; -/** @public */ -export const catalogConflictsTopic = 'catalog.conflict'; - /** * A builder that helps wire up all of the component parts of the catalog. * diff --git a/plugins/catalog-backend/src/service/index.ts b/plugins/catalog-backend/src/service/index.ts index 70c65cf08f..44645d39fe 100644 --- a/plugins/catalog-backend/src/service/index.ts +++ b/plugins/catalog-backend/src/service/index.ts @@ -18,4 +18,4 @@ export type { CatalogEnvironment, CatalogPermissionRuleInput, } from './CatalogBuilder'; -export { CatalogBuilder, catalogConflictsTopic } from './CatalogBuilder'; +export { CatalogBuilder } from './CatalogBuilder'; From 1f6ef3146767d5389c494102b6849816e949be7c Mon Sep 17 00:00:00 2001 From: sblausten Date: Tue, 4 Apr 2023 17:42:14 +0200 Subject: [PATCH 13/20] Refactor event to a type Signed-off-by: sblausten --- packages/backend/package.json | 1 + plugins/catalog-backend/src/catalog/types.ts | 26 +++++++++++++++++++ .../src/database/DefaultProcessingDatabase.ts | 10 ++++--- yarn.lock | 3 ++- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index fd8ec82d91..a83929a15d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,6 +40,7 @@ "@backstage/plugin-azure-sites-backend": "workspace:^", "@backstage/plugin-badges-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-backend-module-github": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-code-coverage-backend": "workspace:^", "@backstage/plugin-entity-feedback-backend": "workspace:^", diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 404b816d4f..76841af260 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -311,3 +311,29 @@ export type Cursor = { */ totalItems?: number; }; + +/** + * The details of the conflicting entity. + */ +export type CatalogConflictEventPayload = { + /** + * The new conflicting entity + */ + entity: Entity; + /** + * The new conflicting entity's ref + */ + entityRef: string; + /** + * The location key of this new conflicting entity + */ + newLocationKey: string; + /** + * The location key of the entity already in the catalog that shares the same name and namespace as this entity + */ + existingLocationKey: string; + /** + * The datetime that the conflict was processed at + */ + lastConflictAt: string; +}; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index e634d18c4a..fe61cc694e 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -44,9 +44,10 @@ import { checkLocationKeyConflict } from './operations/refreshState/checkLocatio import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; import { generateStableHash } from './util'; -import { EventBroker } from '@backstage/plugin-events-node'; +import { EventBroker, EventParams } from '@backstage/plugin-events-node'; import { DateTime } from 'luxon'; import { CATALOG_CONFLICTS_TOPIC } from '../constants'; +import { CatalogConflictEventPayload } from '../catalog/types'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -360,8 +361,8 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); - if (this.options.eventBroker) { - await this.options.eventBroker?.publish({ + if (this.options.eventBroker && locationKey) { + const eventParams: EventParams = { topic: CATALOG_CONFLICTS_TOPIC, eventPayload: { entity, @@ -370,7 +371,8 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { existingLocationKey: conflictingKey, lastConflictAt: DateTime.now().toISO(), }, - }); + }; + await this.options.eventBroker?.publish(eventParams); } } } diff --git a/yarn.lock b/yarn.lock index db64dfc04e..d9b937d457 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5142,7 +5142,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": +"@backstage/plugin-catalog-backend-module-github@workspace:^, @backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: @@ -23482,6 +23482,7 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-entity-feedback-backend": "workspace:^" From c5c988e3681cd91dd11aad3fe2fc28f5c07e5d09 Mon Sep 17 00:00:00 2001 From: sblausten Date: Tue, 4 Apr 2023 18:07:37 +0200 Subject: [PATCH 14/20] Remove redundant export Signed-off-by: sblausten --- plugins/events-backend/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/events-backend/src/index.ts b/plugins/events-backend/src/index.ts index b7089fb07d..be173b677c 100644 --- a/plugins/events-backend/src/index.ts +++ b/plugins/events-backend/src/index.ts @@ -21,6 +21,5 @@ */ export { EventsBackend } from './service/EventsBackend'; -export { InMemoryEventBroker } from './service/InMemoryEventBroker'; export { HttpPostIngressEventPublisher } from './service/http'; export { DefaultEventBroker } from './service/DefaultEventBroker'; From 7755bb1828a56393e39def3191c4966c4e1590b7 Mon Sep 17 00:00:00 2001 From: sblausten Date: Tue, 4 Apr 2023 18:20:26 +0200 Subject: [PATCH 15/20] Fix out of date api-report Signed-off-by: sblausten --- plugins/events-backend/api-report.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/plugins/events-backend/api-report.md b/plugins/events-backend/api-report.md index adbe5e3592..aeb6f9363d 100644 --- a/plugins/events-backend/api-report.md +++ b/plugins/events-backend/api-report.md @@ -54,15 +54,4 @@ export class HttpPostIngressEventPublisher implements EventPublisher { // (undocumented) setEventBroker(eventBroker: EventBroker): Promise; } - -// @public -export class InMemoryEventBroker implements EventBroker { - constructor(logger: Logger); - // (undocumented) - publish(params: EventParams): Promise; - // (undocumented) - subscribe( - ...subscribers: Array> - ): void; -} ``` From a2cdf3796bb90fe7254e8b477a8c2fc0419cabcb Mon Sep 17 00:00:00 2001 From: sblausten Date: Fri, 12 May 2023 13:48:12 +0100 Subject: [PATCH 16/20] Address PR Comments Signed-off-by: sblausten --- .changeset/sweet-fans-juggle.md | 2 +- plugins/catalog-backend/api-report.md | 2 +- plugins/catalog-backend/src/constants.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/sweet-fans-juggle.md b/.changeset/sweet-fans-juggle.md index db8abd8ca6..ffb57a3e4c 100644 --- a/.changeset/sweet-fans-juggle.md +++ b/.changeset/sweet-fans-juggle.md @@ -3,4 +3,4 @@ '@backstage/plugin-events-backend': patch --- -Adds an optional `EventBroker` for sending an event when there are conflicts, and exports `InMemoryBroker` for use in other plugins. +Adds an optional `EventBroker` for sending an event when there are conflicts. diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 0c8a688fd1..3e97442dd4 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -115,7 +115,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor_2 { } // @public (undocumented) -export const CATALOG_CONFLICTS_TOPIC = 'catalog.conflict'; +export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict'; // @public export class CatalogBuilder { diff --git a/plugins/catalog-backend/src/constants.ts b/plugins/catalog-backend/src/constants.ts index 2ba21da629..00ae2673e2 100644 --- a/plugins/catalog-backend/src/constants.ts +++ b/plugins/catalog-backend/src/constants.ts @@ -15,4 +15,4 @@ */ /** @public */ -export const CATALOG_CONFLICTS_TOPIC = 'catalog.conflict'; +export const CATALOG_CONFLICTS_TOPIC = 'experimental.catalog.conflict'; From 0bc5ab7058405e806a6db7297b321f498045ab0a Mon Sep 17 00:00:00 2001 From: sblausten Date: Fri, 12 May 2023 13:50:44 +0100 Subject: [PATCH 17/20] Update changeset message Signed-off-by: sblausten --- .changeset/sweet-fans-juggle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/sweet-fans-juggle.md b/.changeset/sweet-fans-juggle.md index ffb57a3e4c..af299e9cac 100644 --- a/.changeset/sweet-fans-juggle.md +++ b/.changeset/sweet-fans-juggle.md @@ -3,4 +3,4 @@ '@backstage/plugin-events-backend': patch --- -Adds an optional `EventBroker` for sending an event when there are conflicts. +Adds an optional `EventBroker` that is used for sending an event when there are conflicts, with details of the conflict so that it can be handled elsewhere. From 2ad15d5f8244f2f861a02297b73d5f508ba426cd Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 29 May 2023 12:30:00 +0200 Subject: [PATCH 18/20] Changes Signed-off-by: sblausten --- .changeset/sweet-fans-juggle.md | 3 +-- packages/backend/package.json | 1 - plugins/catalog-backend/src/catalog/types.ts | 2 +- yarn.lock | 3 +-- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.changeset/sweet-fans-juggle.md b/.changeset/sweet-fans-juggle.md index af299e9cac..72df182886 100644 --- a/.changeset/sweet-fans-juggle.md +++ b/.changeset/sweet-fans-juggle.md @@ -1,6 +1,5 @@ --- -'@backstage/plugin-catalog-backend': patch -'@backstage/plugin-events-backend': patch +'@backstage/plugin-catalog-backend': minor --- Adds an optional `EventBroker` that is used for sending an event when there are conflicts, with details of the conflict so that it can be handled elsewhere. diff --git a/packages/backend/package.json b/packages/backend/package.json index 61b01a57a0..b5a82fd611 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,7 +40,6 @@ "@backstage/plugin-azure-sites-backend": "workspace:^", "@backstage/plugin-badges-backend": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", - "@backstage/plugin-catalog-backend-module-github": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-code-coverage-backend": "workspace:^", "@backstage/plugin-devtools-backend": "workspace:^", diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 76841af260..45a3ff6144 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -317,7 +317,7 @@ export type Cursor = { */ export type CatalogConflictEventPayload = { /** - * The new conflicting entity + * The new conflicting entity in raw, unvalidated form */ entity: Entity; /** diff --git a/yarn.lock b/yarn.lock index f9f870ce77..e2b0dabbde 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5385,7 +5385,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-github@workspace:^, @backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": +"@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: @@ -23951,7 +23951,6 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-devtools-backend": "workspace:^" From 599ad683de8da7b26a1f17d364b3342f9a85cb62 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 12 Jun 2023 11:07:25 +0100 Subject: [PATCH 19/20] Change field name Signed-off-by: sblausten --- plugins/catalog-backend/src/catalog/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts index 45a3ff6144..156b8aa64d 100644 --- a/plugins/catalog-backend/src/catalog/types.ts +++ b/plugins/catalog-backend/src/catalog/types.ts @@ -319,7 +319,7 @@ export type CatalogConflictEventPayload = { /** * The new conflicting entity in raw, unvalidated form */ - entity: Entity; + unprocessedEntity: Entity; /** * The new conflicting entity's ref */ From 1684c673d225d6e920708a6fadb7662d69015164 Mon Sep 17 00:00:00 2001 From: sblausten Date: Mon, 12 Jun 2023 11:10:20 +0100 Subject: [PATCH 20/20] Api report Signed-off-by: sblausten --- .../catalog-backend/src/database/DefaultProcessingDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 04005101da..4959a33eef 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -370,7 +370,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { const eventParams: EventParams = { topic: CATALOG_CONFLICTS_TOPIC, eventPayload: { - entity, + unprocessedEntity: entity, entityRef, newLocationKey: locationKey, existingLocationKey: conflictingKey,