diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index f6fe25f86a..53204d6957 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,14 +19,28 @@ import { EntityProvider } from '@backstage/plugin-catalog-node'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; +import { InMemoryEventBroker } from '@backstage/plugin-events-backend'; +import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; export default async function createPlugin( env: PluginEnvironment, providers?: Array, ): Promise { const builder = await CatalogBuilder.create(env); + const githubEntityProviders = GithubEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 5 }, + timeout: { minutes: 3 }, + }), + scheduler: env.scheduler, + }); + builder.addEntityProvider(githubEntityProviders); builder.addProcessor(new ScaffolderEntitiesProcessor()); builder.addEntityProvider(providers ?? []); + const conflictBroker = new InMemoryEventBroker(env.logger); + builder.setConflictEventBroker(conflictBroker); const { processingEngine, router } = await builder.build(); await processingEngine.start(); return router; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 1e837fd9ba..6805a0d21e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -54,6 +54,7 @@ "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@backstage/plugin-permission-node": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 761f5bfd72..4fe24eab97 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -44,6 +44,7 @@ import { checkLocationKeyConflict } from './operations/refreshState/checkLocatio import { insertUnprocessedEntity } from './operations/refreshState/insertUnprocessedEntity'; import { updateUnprocessedEntity } from './operations/refreshState/updateUnprocessedEntity'; import { generateStableHash } from './util'; +import { EventBroker } from '@backstage/plugin-events-node'; // The number of items that are sent per batch to the database layer, when // doing .batchInsert calls to knex. This needs to be low enough to not cause @@ -57,6 +58,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { database: Knex; logger: Logger; refreshInterval: ProcessingIntervalFunction; + conflictEventBroker?: EventBroker; }, ) { initDatabaseMetrics(options.database); @@ -356,6 +358,18 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { this.options.logger.warn( `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, ); + if (this.options.conflictEventBroker) { + this.options.conflictEventBroker?.publish({ + topic: 'conflicting-entities', + eventPayload: { + ...entity, + entityRef, + newLocationKey: locationKey, + existingLocationKey: conflictingKey, + }, + metadata: {}, + }); + } } } diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 7da5d69767..d0584477ab 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -98,6 +98,7 @@ import { import { AuthorizedLocationService } from './AuthorizedLocationService'; import { DefaultProviderDatabase } from '../database/DefaultProviderDatabase'; import { DefaultCatalogDatabase } from '../database/DefaultCatalogDatabase'; +import { EventBroker } from '@backstage/plugin-events-node'; /** * This is a duplicate of the alpha `CatalogPermissionRule` type, for use in the stable API. @@ -166,6 +167,7 @@ export class CatalogBuilder { private readonly permissionRules: CatalogPermissionRuleInput[]; private allowedLocationType: string[]; private legacySingleProcessorValidation = false; + private conflictEventBroker?: EventBroker; /** * Creates a catalog builder. @@ -417,6 +419,11 @@ export class CatalogBuilder { return this; } + setConflictEventBroker(broker: EventBroker): CatalogBuilder { + this.conflictEventBroker = broker; + return this; + } + /** * Wires up and returns all of the component parts of the catalog */ @@ -440,6 +447,7 @@ export class CatalogBuilder { database: dbClient, logger, refreshInterval: this.processingInterval, + conflictEventBroker: this.conflictEventBroker, }); const providerDatabase = new DefaultProviderDatabase({ database: dbClient, diff --git a/plugins/events-backend/src/index.ts b/plugins/events-backend/src/index.ts index 4645e9a479..215ea2c203 100644 --- a/plugins/events-backend/src/index.ts +++ b/plugins/events-backend/src/index.ts @@ -21,4 +21,5 @@ */ export { EventsBackend } from './service/EventsBackend'; +export { InMemoryEventBroker } from './service/InMemoryEventBroker'; export { HttpPostIngressEventPublisher } from './service/http'; diff --git a/yarn.lock b/yarn.lock index adc0ba595e..f078ec6138 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5006,7 +5006,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": +"@backstage/plugin-catalog-backend-module-github@workspace:^, @backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-github@workspace:plugins/catalog-backend-module-github" dependencies: @@ -5204,6 +5204,7 @@ __metadata: "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@backstage/plugin-permission-node": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" @@ -22508,6 +22509,7 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-entity-feedback-backend": "workspace:^"