diff --git a/.changeset/old-bulldogs-fry.md b/.changeset/old-bulldogs-fry.md new file mode 100644 index 0000000000..9c8eef67ad --- /dev/null +++ b/.changeset/old-bulldogs-fry.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Handle GitHub `push` events at the `GithubEntityProvider` by subscribing to the topic `github.push.` + +Implements `EventSubscriber` to receive events for the topic `github.push`. + +On `github.push`, the affected repository will be refreshed. +This includes adding new Location entities, refreshing existing ones, +and removing obsolete ones. + +Please find more information at +https://backstage.io/docs/integrations/github/discovery#installation-with-events-support diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 4f032901a9..bbbe86a834 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -14,7 +14,7 @@ organization and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. This is the preferred method for ingesting entities into the catalog. -## Installation +## Installation without Events Support You will have to add the provider in the catalog initialization code of your backend. They are not installed by default, therefore you have to add a @@ -53,6 +53,52 @@ And then add the entity provider to your catalog builder: } ``` +## Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ GithubEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, ++ }), ++ ); + return providers.flat(); + } +``` + +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `push` events. + ## Configuration To use the discovery provider, you'll need a GitHub integration diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index e749549646..7a85d685a3 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -11,6 +11,8 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EventParams } from '@backstage/plugin-events-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GithubIntegrationConfig } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; @@ -76,7 +78,7 @@ export class GitHubEntityProvider implements EntityProvider { } // @public -export class GithubEntityProvider implements EntityProvider { +export class GithubEntityProvider implements EntityProvider, EventSubscriber { // (undocumented) connect(connection: EntityProviderConnection): Promise; // (undocumented) @@ -91,7 +93,11 @@ export class GithubEntityProvider implements EntityProvider { // (undocumented) getProviderName(): string; // (undocumented) + onEvent(params: EventParams): Promise; + // (undocumented) refresh(logger: Logger): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @alpha diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index d6d996068e..2e6892ab2b 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -42,7 +42,9 @@ "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/types": "workspace:^", "@octokit/graphql": "^5.0.0", "@octokit/rest": "^19.0.3", diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index cc9a9bfee3..a41404ad9a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -24,6 +24,7 @@ import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GithubEntityProvider } from './GithubEntityProvider'; import * as helpers from '../lib/github'; +import { EventParams } from '@backstage/plugin-events-node'; jest.mock('../lib/github', () => { return { @@ -716,4 +717,392 @@ describe('GithubEntityProvider', () => { expect(providers).toHaveLength(1); expect(providers[0].getProviderName()).toEqual('github-provider:default'); }); + + it('apply delta update on added files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: ['catalog-info.yaml'], + removed: [], + modified: [], + }, + ], + }, + }; + const url = + 'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml'; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:default', + }, + ]; + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: expectedEntities, + removed: [], + }); + }); + + it('apply delta update on removed files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: ['catalog-info.yaml'], + modified: [], + }, + ], + }, + }; + const url = + 'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml'; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:default', + }, + ]; + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntities, + }); + }); + + it('apply refresh call on modified files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + 'url:https://github.com/test-org/test-repo/tree/main/catalog-info.yaml', + 'url:https://github.com/test-org/test-repo/blob/main/catalog-info.yaml', + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should process repository when match filters from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + filters: { + branch: 'my-special-branch', + repository: 'test-repo', + }, + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/my-special-branch', + repository: { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + 'url:https://github.com/test-org/test-repo/tree/my-special-branch/catalog-info.yaml', + 'url:https://github.com/test-org/test-repo/blob/my-special-branch/catalog-info.yaml', + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it("should skip process when didn't match filters from push event", async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + filters: { + repository: 'only-special-repository', + }, + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 9a75f52afa..8816d8ee80 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -24,12 +24,14 @@ import { SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; + import { graphql } from '@octokit/graphql'; import * as uuid from 'uuid'; import { Logger } from 'winston'; @@ -37,9 +39,23 @@ import { readProviderConfigs, GithubEntityProviderConfig, } from './GithubEntityProviderConfig'; -import { getOrganizationRepositories, RepositoryResponse } from '../lib/github'; +import { getOrganizationRepositories } from '../lib/github'; import { satisfiesTopicFilter } from '../lib/util'; +import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; +import { PushEvent, Commit } from '@octokit/webhooks-types'; + +const TOPIC_REPO_PUSH = 'github.push'; + +type Repository = { + name: string; + url: string; + isArchived: boolean; + repositoryTopics: string[]; + defaultBranchRef?: string; + isCatalogInfoFilePresent: boolean; +}; + /** * Discovers catalog files located in [GitHub](https://github.com). * The provider will search your GitHub account and register catalog files matching the configured path @@ -48,7 +64,7 @@ import { satisfiesTopicFilter } from '../lib/util'; * * @public */ -export class GithubEntityProvider implements EntityProvider { +export class GithubEntityProvider implements EntityProvider, EventSubscriber { private readonly config: GithubEntityProviderConfig; private readonly logger: Logger; private readonly integration: GithubIntegrationConfig; @@ -175,7 +191,7 @@ export class GithubEntityProvider implements EntityProvider { } // go to the server and get all of the repositories - private async findCatalogFiles(): Promise { + private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; const catalogPath = this.config.catalogPath; @@ -190,45 +206,49 @@ export class GithubEntityProvider implements EntityProvider { headers, }); - const { repositories } = await getOrganizationRepositories( - client, - organization, - catalogPath, - ); + const { repositories: repositoriesFromGithub } = + await getOrganizationRepositories(client, organization, catalogPath); + const repositories = repositoriesFromGithub.map(r => { + return { + url: r.url, + name: r.name, + defaultBranchRef: r.defaultBranchRef?.name, + repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name), + isArchived: r.isArchived, + isCatalogInfoFilePresent: + r.catalogInfoFile?.__typename === 'Blob' && + r.catalogInfoFile.text !== '', + }; + }); if (this.config.validateLocationsExist) { - return repositories.filter(repository => { - return ( - repository.catalogInfoFile?.__typename === 'Blob' && - repository.catalogInfoFile.text !== '' - ); - }); + return repositories.filter( + repository => repository.isCatalogInfoFilePresent, + ); } return repositories; } - private matchesFilters(repositories: RepositoryResponse[]) { + private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; const topicFilters = this.config.filters?.topic; const matchingRepositories = repositories.filter(r => { - const repoTopics: string[] = r.repositoryTopics.nodes.map( - node => node.topic.name, - ); + const repoTopics: string[] = r.repositoryTopics; return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && - r.defaultBranchRef?.name + r.defaultBranchRef ); }); return matchingRepositories; } - private createLocationUrl(repository: RepositoryResponse): string { + private createLocationUrl(repository: Repository): string { const branch = - this.config.filters?.branch || repository.defaultBranchRef?.name || '-'; + this.config.filters?.branch || repository.defaultBranchRef || '-'; const catalogFile = this.config.catalogPath.startsWith('/') ? this.config.catalogPath.substring(1) : this.config.catalogPath; @@ -242,6 +262,151 @@ export class GithubEntityProvider implements EntityProvider { presence: 'optional', }; } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + this.logger.debug(`Received event from ${params.topic}`); + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + await this.onRepoPush(params.eventPayload as PushEvent); + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return [TOPIC_REPO_PUSH]; + } + + private async onRepoPush(event: PushEvent) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const repoName = event.repository.name; + const repoUrl = event.repository.url; + this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`); + + const branch = + this.config.filters?.branch || event.repository.default_branch; + + if (!event.ref.includes(branch)) { + this.logger.debug(`skipping push event from ref ${event.ref}`); + return; + } + + const repository: Repository = { + url: event.repository.url, + name: event.repository.name, + defaultBranchRef: event.repository.default_branch, + repositoryTopics: event.repository.topics, + isArchived: event.repository.archived, + // we can consider this file present because + // only the catalog file will be recovered from the commits + isCatalogInfoFilePresent: true, + }; + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping push event from repository ${repoName} because didn't match provider filters`, + ); + return; + } + + // the commit has information about the files (added,removed,modified) + // so we will process the change based in this data + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push + const added = this.collectDeferredEntitiesFromCommit( + event.repository.url, + branch, + event.commits, + (commit: Commit) => [...commit.added], + ); + const removed = this.collectDeferredEntitiesFromCommit( + event.repository.url, + branch, + event.commits, + (commit: Commit) => [...commit.removed], + ); + const modified = this.collectFilesFromCommit( + event.commits, + (commit: Commit) => [...commit.modified], + ); + + if (modified.length > 0) { + await this.connection.refresh({ + keys: [ + ...modified.map( + filePath => + `url:${event.repository.url}/tree/${branch}/${filePath}`, + ), + ...modified.map( + filePath => + `url:${event.repository.url}/blob/${branch}/${filePath}`, + ), + ], + }); + } + + if (added.length > 0 || removed.length > 0) { + await this.connection.applyMutation({ + type: 'delta', + added: added, + removed: removed, + }); + } + + this.logger.info( + `Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`, + ); + } + + private collectDeferredEntitiesFromCommit( + repositoryUrl: string, + branch: string, + commits: Commit[], + transformOperation: (commit: Commit) => string[], + ): DeferredEntity[] { + const catalogFiles = this.collectFilesFromCommit( + commits, + transformOperation, + ); + return this.toDeferredEntities( + catalogFiles.map( + filePath => `${repositoryUrl}/blob/${branch}/${filePath}`, + ), + ); + } + + private collectFilesFromCommit( + commits: Commit[], + transformOperation: (commit: Commit) => string[], + ): string[] { + const catalogFile = this.config.catalogPath.startsWith('/') + ? this.config.catalogPath.substring(1) + : this.config.catalogPath; + + return commits + .map(transformOperation) + .flat() + .filter(file => catalogFile.includes(file)); + } + + private toDeferredEntities(targets: string[]): DeferredEntity[] { + return targets + .map(target => { + const location = GithubEntityProvider.toLocationSpec(target); + + return locationSpecToLocationEntity({ location }); + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } } /* diff --git a/yarn.lock b/yarn.lock index 8f61f17229..59815efb77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4577,7 +4577,9 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" "@backstage/types": "workspace:^" "@octokit/graphql": ^5.0.0 "@octokit/rest": ^19.0.3