From 2eb0da3931de862ce9757d045cf987bf790af9f9 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Tue, 19 Mar 2024 01:40:56 +0100 Subject: [PATCH] feat(events,github-org): add support for EventsService, events at new backend Adds optional support for the EventsService passed as argument to the factory method. For the new backend, the EventsService is used as dependency and passed on. Events support will be active always. The support for the deprecated EventBroker and its interfaces was kept for easier migration and will be removed as a separate follow-up change that could go into a following release. Signed-off-by: Patrick Jungermann --- .changeset/happy-walls-think.md | 33 +++++++++++++++++ docs/integrations/github/org.md | 31 ++++++++++++++-- .../package.json | 3 +- .../src/module.ts | 5 ++- .../api-report.md | 5 +++ .../providers/GithubMultiOrgEntityProvider.ts | 37 ++++++++++++++----- .../src/providers/GithubOrgEntityProvider.ts | 25 ++++++++++++- yarn.lock | 1 + 8 files changed, 124 insertions(+), 16 deletions(-) create mode 100644 .changeset/happy-walls-think.md diff --git a/.changeset/happy-walls-think.md b/.changeset/happy-walls-think.md new file mode 100644 index 0000000000..53e246cd7f --- /dev/null +++ b/.changeset/happy-walls-think.md @@ -0,0 +1,33 @@ +--- +'@backstage/plugin-catalog-backend-module-github-org': patch +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Support EventsService and events with the new backend system (through EventsService) for `GithubOrgEntityProvider` and `GithubMultiOrgEntityProvider`. + +_New/Current Backend System:_ + +The events support for the provider will be enabled always, making it ready to consume events from its subscribed topics. +In order to receive events and make use of this feature, you still need to set up receiving events from the event source as before. + +_Legacy Backend System:_ + +You can pass the `EventsService` instance to the factory method as one of its options: + +```diff + // packages/backend/src/plugins/catalog.ts + const githubOrgProvider = GithubOrgEntityProvider.fromConfig(env.config, { + events: env.events, + // ... + }); +- env.eventBroker.subscribe(githubOrgProvider); +``` + +```diff + // packages/backend/src/plugins/catalog.ts + const githubMultiOrgProvider = GithubMultiOrgEntityProvider.fromConfig(env.config, { + events: env.events, + // ... + }); +- env.eventBroker.subscribe(githubMultiOrgProvider); +``` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 263ceda6c6..974362c125 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -103,6 +103,32 @@ export default async function createPlugin( ## Installation with Events Support +_For the legacy backend system, please read the subsection below._ + +The catalog module `github-org` comes with events support enabled for the `GithubMultiOrgEntityProvider`. +This will make it subscribe to its relevant topics and expects these events to be published via the `EventsService`. + +Topics: + +- `github.installation` +- `github.membership` +- `github.organization` +- `github.team` + +Additionally, you should install the +[event router by `events-backend-module-github`](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md) +which will route received events from the generic topic `github` to more specific ones +based on the event type (e.g., `github.membership`). + +In order to receive Webhook events by GitHub, you have to decide how you want them +to be ingested into Backstage and published to its `EventsService`. +You can decide between the following options (extensible): + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +### Legacy Backend System + Please follow the installation instructions at - @@ -133,12 +159,12 @@ export default async function createPlugin( id: 'production', orgUrl: 'https://github.com/backstage', logger: env.logger, + events: env.events, schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 60 }, timeout: { minutes: 15 }, }), }); - env.eventBroker.subscribe(githubOrgProvider); builder.addEntityProvider(githubOrgProvider); /* highlight-add-end */ const { processingEngine, router } = await builder.build(); @@ -169,12 +195,11 @@ export default async function createPlugin( // also omit this option to ingest all orgs accessible by your GitHub integration orgs: ['org-a', 'org-b'], logger: env.logger, + events: env.events, schedule: env.scheduler.createScheduledTaskRunner({ frequency: { minutes: 60 }, timeout: { minutes: 15 }, }), - // Pass in the eventBroker to allow this provider to subscribe to GitHub events - eventBroker: env.eventBroker, }), ); /* highlight-add-end */ diff --git a/plugins/catalog-backend-module-github-org/package.json b/plugins/catalog-backend-module-github-org/package.json index b5734adac4..59d88c6f15 100644 --- a/plugins/catalog-backend-module-github-org/package.json +++ b/plugins/catalog-backend-module-github-org/package.json @@ -36,7 +36,8 @@ "@backstage/backend-tasks": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-catalog-backend-module-github": "workspace:^", - "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^" }, "devDependencies": { "@backstage/backend-test-utils": "workspace:^", diff --git a/plugins/catalog-backend-module-github-org/src/module.ts b/plugins/catalog-backend-module-github-org/src/module.ts index d666f78bde..5aeec60572 100644 --- a/plugins/catalog-backend-module-github-org/src/module.ts +++ b/plugins/catalog-backend-module-github-org/src/module.ts @@ -31,6 +31,7 @@ import { UserTransformer, } from '@backstage/plugin-catalog-backend-module-github'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; /** * Interface for {@link githubOrgEntityProviderTransformsExtensionPoint}. @@ -95,16 +96,18 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({ deps: { catalog: catalogProcessingExtensionPoint, config: coreServices.rootConfig, + events: eventsServiceRef, logger: coreServices.logger, scheduler: coreServices.scheduler, }, - async init({ catalog, config, logger, scheduler }) { + async init({ catalog, config, events, logger, scheduler }) { for (const definition of readDefinitionsFromConfig(config)) { catalog.addEntityProvider( GithubMultiOrgEntityProvider.fromConfig(config, { id: definition.id, githubUrl: definition.githubUrl, orgs: definition.orgs, + events, schedule: scheduler.createScheduledTaskRunner( definition.schedule, ), diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 7006f4357b..8367393e7f 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -142,6 +142,7 @@ export type GithubMultiOrgConfig = Array<{ // @public export class GithubMultiOrgEntityProvider implements EntityProvider { constructor(options: { + events?: EventsService; id: string; gitHubConfig: GithubIntegrationConfig; githubCredentialsProvider: GithubCredentialsProvider; @@ -165,7 +166,9 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { // @public export interface GithubMultiOrgEntityProviderOptions { + // @deprecated eventBroker?: EventBroker; + events?: EventsService; githubCredentialsProvider?: GithubCredentialsProvider; githubUrl: string; id: string; @@ -220,6 +223,7 @@ export class GithubOrgEntityProvider implements EntityProvider, EventSubscriber { constructor(options: { + events?: EventsService; id: string; orgUrl: string; gitHubConfig: GithubIntegrationConfig; @@ -249,6 +253,7 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions; // @public export interface GithubOrgEntityProviderOptions { + events?: EventsService; githubCredentialsProvider?: GithubCredentialsProvider; id: string; logger: Logger; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index 8387a91e3a..714edbd62a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -38,7 +38,11 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; -import { EventBroker, EventParams } from '@backstage/plugin-events-node'; +import { + EventBroker, + EventParams, + EventsService, +} from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; import { InstallationCreatedEvent, @@ -80,6 +84,13 @@ import { import { splitTeamSlug } from '../lib/util'; import { areGroupEntities, areUserEntities } from '../lib/guards'; +const EVENT_TOPICS = [ + 'github.installation', + 'github.membership', + 'github.organization', + 'github.team', +]; + /** * Options for {@link GithubMultiOrgEntityProvider}. * @@ -101,11 +112,16 @@ export interface GithubMultiOrgEntityProviderOptions { githubUrl: string; /** - * The list of the GitHub orgs to consume. By default will consume all accessible + * The list of the GitHub orgs to consume. By default, it will consume all accessible * orgs on the given GitHub instance (support for GitHub App integration only). */ orgs?: string[]; + /** + * Passing the optional EventsService enables event-based delta updates. + */ + events?: EventsService; + /** * The refresh schedule to use. * @@ -138,12 +154,14 @@ export interface GithubMultiOrgEntityProviderOptions { /** * Optionally include a team transformer for transforming from GitHub teams to Group Entities. - * By default groups will be namespaced according to their GitHub org. + * By default, groups will be namespaced according to their GitHub org. */ teamTransformer?: TeamTransformer; /** * An EventBroker to subscribe this provider to GitHub events to trigger delta mutations + * + * @deprecated Use `events` instead. */ eventBroker?: EventBroker; } @@ -206,6 +224,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { constructor( private readonly options: { + events?: EventsService; id: string; gitHubConfig: GithubIntegrationConfig; githubCredentialsProvider: GithubCredentialsProvider; @@ -225,6 +244,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; + await this.options.events?.subscribe({ + id: this.getProviderName(), + topics: EVENT_TOPICS, + onEvent: params => this.onEvent(params), + }); await this.scheduleFn?.(); } @@ -312,12 +336,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { } private supportsEventTopics(): string[] { - return [ - 'github.installation', - 'github.organization', - 'github.team', - 'github.membership', - ]; + return EVENT_TOPICS; } private async onEvent(params: EventParams): Promise { diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 0fe61aa229..ba9b4bc0b6 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -28,7 +28,11 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; -import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; +import { + EventParams, + EventsService, + EventSubscriber, +} from '@backstage/plugin-events-node'; import { graphql } from '@octokit/graphql'; import { MembershipEvent, @@ -62,6 +66,12 @@ import { parseGithubOrgUrl } from '../lib/util'; import { withLocations } from '../lib/withLocations'; import { areGroupEntities, areUserEntities } from '../lib/guards'; +const EVENT_TOPICS = [ + 'github.membership', + 'github.organization', + 'github.team', +]; + /** * Options for {@link GithubOrgEntityProvider}. * @@ -82,6 +92,11 @@ export interface GithubOrgEntityProviderOptions { */ orgUrl: string; + /** + * Passing the optional EventsService enables event-based delta updates. + */ + events?: EventsService; + /** * The refresh schedule to use. * @@ -163,6 +178,7 @@ export class GithubOrgEntityProvider constructor( private options: { + events?: EventsService; id: string; orgUrl: string; gitHubConfig: GithubIntegrationConfig; @@ -185,6 +201,11 @@ export class GithubOrgEntityProvider /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; + await this.options.events?.subscribe({ + id: this.getProviderName(), + topics: EVENT_TOPICS, + onEvent: params => this.onEvent(params), + }); await this.scheduleFn?.(); } @@ -315,7 +336,7 @@ export class GithubOrgEntityProvider /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ supportsEventTopics(): string[] { - return ['github.organization', 'github.team', 'github.membership']; + return EVENT_TOPICS; } private async onTeamEditedInOrganization( diff --git a/yarn.lock b/yarn.lock index f0f995970b..08b7f87405 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5593,6 +5593,7 @@ __metadata: "@backstage/config": "workspace:^" "@backstage/plugin-catalog-backend-module-github": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" luxon: ^3.0.0 languageName: unknown linkType: soft