From 9112efc364d24f134881e9c09db1323fdfcc4165 Mon Sep 17 00:00:00 2001 From: Patrick Jungermann Date: Sat, 22 Jun 2024 03:02:42 +0200 Subject: [PATCH] feat(catalog,github): support `repository` events The provider adds a subscription to the topic `github.repository`. Hereby, it supports events of type `repository` with actions - `archived` - `deleted` - `edited` - `renamed` - `transferred` - `unarchived` Actions skipped as they don't require entity changes: - `created` - `privatized` - `publicized` If the config option `validateLocationsExist` is enabled, an API request is necessary and will be executed. This affects the actions `renamed`, `transferred`, and `unarchive` of event type `repository`. Catalog entities related to the `GithubEntityProvider` instance will be adjusted according to action and its meaning for them. Closes: #21906 Signed-off-by: Patrick Jungermann --- .changeset/poor-cheetahs-raise.md | 30 + docs/integrations/github/discovery.md | 15 +- .../src/lib/github.ts | 56 ++ .../providers/GithubEntityProvider.test.ts | 547 +++++++++++++++++- .../src/providers/GithubEntityProvider.ts | 386 ++++++++++-- 5 files changed, 971 insertions(+), 63 deletions(-) create mode 100644 .changeset/poor-cheetahs-raise.md diff --git a/.changeset/poor-cheetahs-raise.md b/.changeset/poor-cheetahs-raise.md new file mode 100644 index 0000000000..0ee2162a22 --- /dev/null +++ b/.changeset/poor-cheetahs-raise.md @@ -0,0 +1,30 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Adds support for `repository` events. + +The provider adds a subscription to the topic `github.repository`. + +Hereby, it supports events of type `repository` with actions + +- `archived` +- `deleted` +- `edited` +- `renamed` +- `transferred` +- `unarchived` + +Actions skipped as they don't require entity changes: + +- `created` +- `privatized` +- `publicized` + +If the config option `validateLocationsExist` is enabled, an API request +is necessary and will be executed. +This affects the actions `renamed`, `transferred`, and `unarchive` +of event type `repository`. + +Catalog entities related to the `GithubEntityProvider` instance will be adjusted +according to action and its meaning for them. diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 5c3a33416b..d8374932b8 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -40,8 +40,9 @@ backend.add(import('@backstage/plugin-catalog-backend-module-github/alpha')); ## Events Support The catalog module for GitHub comes with events support enabled. -This will make it subscribe to its relevant topics (`github.push`) -and expects these events to be published via the `EventsService`. +This will make it subscribe to its relevant topics (`github.push`, +`github.repository`) and expects these events to be published +via the `EventsService`. Additionally, you should install the [event router by `events-backend-module-github`](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md) @@ -55,7 +56,15 @@ You can decide between the following options (extensible): - [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) - [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) -You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `push` events. +You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). + +The webhook(s) will need to be configured to react to `push` and +`repository` events. + +Certain actions like `transferred` by the `repository` event type +will not be supported when you use repository webhooks. +Please check the GitHubs documentation for these event types and +its actions. ## Configuration diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index eff52ea98d..cba65e522e 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -38,6 +38,7 @@ export type QueryResponse = { type RepositoryOwnerResponse = { repositories?: Connection; + repository?: RepositoryResponse; }; export type OrganizationResponse = { @@ -509,6 +510,61 @@ export async function getOrganizationRepositories( return { repositories }; } +export async function getOrganizationRepository( + client: typeof graphql, + org: string, + repoName: string, + catalogPath: string, +): Promise { + let relativeCatalogPathRef: string; + // We must strip the leading slash or the query for objects does not work + if (catalogPath.startsWith('/')) { + relativeCatalogPathRef = catalogPath.substring(1); + } else { + relativeCatalogPathRef = catalogPath; + } + const catalogPathRef = `HEAD:${relativeCatalogPathRef}`; + const query = ` + query repository($org: String!, $repoName: String!, $catalogPathRef: String!) { + repositoryOwner(login: $org) { + repository(name: $repoName) { + name + catalogInfoFile: object(expression: $catalogPathRef) { + __typename + ... on Blob { + id + text + } + } + url + isArchived + isFork + visibility + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } + defaultBranchRef { + name + } + } + } + }`; + + const response: QueryResponse = await client(query, { + org, + repoName, + catalogPathRef, + }); + + return response.repositoryOwner?.repository || null; +} + /** * Gets all the users out of a Github organization. * diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index a28d216053..002b2bbfd1 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -29,7 +29,12 @@ import { GithubEntityProvider } from './GithubEntityProvider'; import * as helpers from '../lib/github'; import { EventParams } from '@backstage/plugin-events-node'; import { mockServices } from '@backstage/backend-test-utils'; -import { Commit, PushEvent } from '@octokit/webhooks-types'; +import { + Commit, + PushEvent, + RepositoryEvent, + RepositoryRenamedEvent, +} from '@octokit/webhooks-types'; jest.mock('../lib/github', () => { return { @@ -657,7 +662,7 @@ describe('GithubEntityProvider', () => { describe('on event', () => { const createExpectedEntitiesForEvent = ( - event: EventParams, + event: EventParams, options?: { branch?: string; catalogFilePath?: string }, ): DeferredEntity[] => { const url = `${event.eventPayload.repository.url}/blob/${ @@ -948,5 +953,543 @@ describe('GithubEntityProvider', () => { expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); }); + + describe('on repository event', () => { + const createRepoEvent = ( + action: RepositoryEvent['action'], + ): EventParams => { + const repo = { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + master_branch: 'main', + organization: 'test-org', + topics: [], + archived: action === 'archived', + private: action !== 'publicized', + } as Partial; + + const event = { + action, + repository: repo as RepositoryEvent['repository'], + } as RepositoryEvent; + + if (action === 'renamed') { + (event as RepositoryRenamedEvent).changes = { + repository: { + name: { + from: `old-${event.repository.name}`, + }, + }, + }; + } + + return { + topic: 'github.repository', + metadata: { + 'x-github-event': 'repository', + }, + eventPayload: event, + }; + }; + + describe('on repository archived event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('archived'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update removing entities', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('archived'); + const expectedEntities = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntities, + }); + }); + }); + + describe('on repository created event', () => { + it('skip', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('created'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + }); + + describe('on repository deleted event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('deleted'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update removing entities', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('deleted'); + const expectedEntities = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntities, + }); + }); + }); + + describe('on repository edited event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('edited'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update removing entities with non-matching filters', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + filters: { + topic: { + include: ['backstage-include'], + }, + }, + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('edited'); + const expectedEntities = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntities, + }); + }); + + it('apply no delta update with matching filters', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + filters: { + topic: { + include: ['backstage-include'], + }, + }, + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('edited'); + event.eventPayload.repository.topics = ['backstage-include']; + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + }); + + describe('on repository privatized event', () => { + it('skip', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('privatized'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + }); + + describe('on repository publicized event', () => { + it('skip', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('publicized'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + }); + + describe('on repository renamed event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('renamed'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update removing entities with non-matching filters', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + filters: { + repository: 'other-.*', + }, + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent( + 'renamed', + ) as EventParams; + const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`; + const expectedEntitiesRemoved = + createExpectedEntitiesForUrl(urlOldRepo); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntitiesRemoved, + }); + }); + + it('apply delta update removing and adding entities with matching filters', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent( + 'renamed', + ) as EventParams; + const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`; + const expectedEntitiesRemoved = + createExpectedEntitiesForUrl(urlOldRepo); + const expectedEntitiesAdded = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 2, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntitiesRemoved, + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: expectedEntitiesAdded, + removed: [], + }); + }); + }); + + describe('on repository transferred event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('transferred'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply no delta update with non-matching filters', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + filters: { + topic: { + include: ['backstage-include'], + }, + }, + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('transferred'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update adding entities with matching filters', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('transferred'); + const expectedEntitiesAdded = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: expectedEntitiesAdded, + removed: [], + }); + }); + }); + + describe('on repository unarchived event', () => { + it('skip on non-matching org', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + organization: 'other-org', + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('unarchived'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply no delta update with non-matching filters', async () => { + const config = createSingleProviderConfig({ + providerConfig: { + filters: { + topic: { + include: ['backstage-include'], + }, + }, + }, + }); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('unarchived'); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 0, + ); + }); + + it('apply delta update adding entities with matching filters', async () => { + const config = createSingleProviderConfig(); + const provider = createProviders(config)[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event = createRepoEvent('unarchived'); + const expectedEntitiesAdded = createExpectedEntitiesForEvent(event); + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes( + 1, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: expectedEntitiesAdded, + removed: [], + }); + }); + }); + }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 978d9593c1..daade9660a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -38,7 +38,11 @@ import { GithubEntityProviderConfig, readProviderConfigs, } from './GithubEntityProviderConfig'; -import { getOrganizationRepositories } from '../lib/github'; +import { + getOrganizationRepositories, + getOrganizationRepository, + RepositoryResponse, +} from '../lib/github'; import { satisfiesForkFilter, satisfiesTopicFilter, @@ -50,11 +54,21 @@ import { EventsService, EventSubscriber, } from '@backstage/plugin-events-node'; -import { Commit, PushEvent } from '@octokit/webhooks-types'; +import { + Commit, + PushEvent, + RepositoryArchivedEvent, + RepositoryDeletedEvent, + RepositoryEditedEvent, + RepositoryEvent, + RepositoryRenamedEvent, + RepositoryTransferredEvent, + RepositoryUnarchivedEvent, +} from '@octokit/webhooks-types'; import { Minimatch } from 'minimatch'; import { LoggerService } from '@backstage/backend-plugin-api'; -const TOPIC_REPO_PUSH = 'github.push'; +const EVENT_TOPICS = ['github.push', 'github.repository']; type Repository = { name: string; @@ -157,7 +171,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { this.connection = connection; await this.events?.subscribe({ id: this.getProviderName(), - topics: [TOPIC_REPO_PUSH], + topics: EVENT_TOPICS, onEvent: params => this.onEvent(params), }); return await this.scheduleFn(); @@ -194,15 +208,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { const targets = await this.findCatalogFiles(); const matchingTargets = this.matchesFilters(targets); - const entities = matchingTargets - .map(repository => this.createLocationUrl(repository)) - .map(GithubEntityProvider.toLocationSpec) - .map(location => { - return { - locationKey: this.getProviderName(), - entity: locationSpecToLocationEntity({ location }), - }; - }); + const entities = this.toDeferredEntitiesFromRepos(matchingTargets); await this.connection.applyMutation({ type: 'full', @@ -214,38 +220,32 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { ); } - // go to the server and get all of the repositories - private async findCatalogFiles(): Promise { + private async createGraphqlClient() { const organization = this.config.organization; const host = this.integration.host; - const catalogPath = this.config.catalogPath; const orgUrl = `https://${host}/${organization}`; const { headers } = await this.githubCredentialsProvider.getCredentials({ url: orgUrl, }); - const client = graphql.defaults({ + return graphql.defaults({ baseUrl: this.integration.apiBaseUrl, headers, }); + } + + // go to the server and get all repositories + private async findCatalogFiles(): Promise { + const organization = this.config.organization; + const catalogPath = this.config.catalogPath; + const client = await this.createGraphqlClient(); const { repositories: repositoriesFromGithub } = await getOrganizationRepositories(client, organization, catalogPath); - const repositories = repositoriesFromGithub.map(r => { - return { - url: r.url, - name: r.name, - defaultBranchRef: r.defaultBranchRef?.name, - repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name), - isArchived: r.isArchived, - isFork: r.isFork, - isCatalogInfoFilePresent: - r.catalogInfoFile?.__typename === 'Blob' && - r.catalogInfoFile.text !== '', - visibility: r.visibility, - }; - }); + const repositories = repositoriesFromGithub.map( + this.createRepoFromGithubResponse, + ); if (this.config.validateLocationsExist) { return repositories.filter( @@ -256,13 +256,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { return repositories; } - private matchesFilters(repositories: Repository[]) { + private matchesFilters(repositories: Repository[]): Repository[] { const repositoryFilter = this.config.filters?.repository; const topicFilters = this.config.filters?.topic; const allowForks = this.config.filters?.allowForks ?? true; const visibilities = this.config.filters?.visibility ?? []; - const matchingRepositories = repositories.filter(r => { + return repositories.filter(r => { const repoTopics: string[] = r.repositoryTopics; return ( !r.isArchived && @@ -273,7 +273,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { r.defaultBranchRef ); }); - return matchingRepositories; } private createLocationUrl(repository: Repository): string { @@ -296,23 +295,34 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ async onEvent(params: EventParams): Promise { this.logger.debug(`Received event from ${params.topic}`); - if (params.topic !== TOPIC_REPO_PUSH) { - return; - } + if (EVENT_TOPICS.some(topic => topic === params.topic)) { + if (!this.connection) { + throw new Error('Not initialized'); + } - await this.onRepoPush(params.eventPayload as PushEvent); + switch (params.topic) { + case 'github.push': + await this.onPush(params.eventPayload as PushEvent); + return; + + case 'github.repository': + await this.onRepoChange(params.eventPayload as RepositoryEvent); + return; + + default: // should never be reached + this.logger.warn( + `Missing implementation for event of topic ${params.topic}`, + ); + } + } } /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ supportsEventTopics(): string[] { - return [TOPIC_REPO_PUSH]; + return EVENT_TOPICS; } - private async onRepoPush(event: PushEvent) { - if (!this.connection) { - throw new Error('Not initialized'); - } - + private async onPush(event: PushEvent) { if (this.config.organization !== event.repository.organization) { this.logger.debug( `skipping push event from organization ${event.repository.organization}`, @@ -332,18 +342,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { return; } - const repository: Repository = { - url: event.repository.url, - name: event.repository.name, - defaultBranchRef: event.repository.default_branch, - repositoryTopics: event.repository.topics, - isArchived: event.repository.archived, - isFork: event.repository.fork, - // we can consider this file present because - // only the catalog file will be recovered from the commits - isCatalogInfoFilePresent: true, - visibility: event.repository.visibility, - }; + const repository = this.createRepoFromEvent(event); const matchingTargets = this.matchesFilters([repository]); if (matchingTargets.length === 0) { @@ -378,7 +377,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { ? this.config.catalogPath.substring(1) : this.config.catalogPath; - await this.connection.refresh({ + await this.connection!.refresh({ keys: [ ...new Set([ ...modified.map( @@ -396,7 +395,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } if (added.length > 0 || removed.length > 0) { - await this.connection.applyMutation({ + await this.connection!.applyMutation({ type: 'delta', added: added, removed: removed, @@ -408,6 +407,263 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { ); } + private async onRepoChange(event: RepositoryEvent) { + if (this.config.organization !== event.repository.organization) { + this.logger.debug( + `skipping repository event from organization ${event.repository.organization}`, + ); + return; + } + + const action = event.action; + switch (action) { + case 'archived': + await this.onRepoArchived(event as RepositoryArchivedEvent); + return; + + // A repository was created. + case 'created': + // skip these events + return; + + case 'deleted': + await this.onRepoDeleted(event as RepositoryDeletedEvent); + return; + + case 'edited': + await this.onRepoEdited(event as RepositoryEditedEvent); + return; + + // The visibility of a repository was changed to `private`. + case 'privatized': + // skip these events + return; + + // The visibility of a repository was changed to `public`. + case 'publicized': + // skip these events + return; + + case 'renamed': + await this.onRepoRenamed(event as RepositoryRenamedEvent); + return; + + case 'transferred': + await this.onRepoTransferred(event as RepositoryTransferredEvent); + return; + + case 'unarchived': + await this.onRepoUnarchived(event as RepositoryUnarchivedEvent); + return; + + default: // should never be reached + this.logger.warn( + `Missing implementation for event of topic repository with action ${action}`, + ); + } + } + + /** + * A repository was archived. + * + * Removes all entities associated with the repository. + * + * @param event - The repository archived event. + * @private + */ + private async onRepoArchived(event: RepositoryArchivedEvent) { + const repository = this.createRepoFromEvent(event); + await this.removeEntitiesForRepo(repository); + this.logger.debug( + `Removed entities for archived repository ${repository.name}`, + ); + } + + /** + * A repository was deleted. + * + * Removes all entities associated with the repository. + * + * @param event - The repository deleted event. + * @private + */ + private async onRepoDeleted(event: RepositoryDeletedEvent) { + const repository = this.createRepoFromEvent(event); + await this.removeEntitiesForRepo(repository); + this.logger.debug( + `Removed entities for deleted repository ${repository.name}`, + ); + } + + /** + * The topics, default branch, description, or homepage of a repository was changed. + * + * We are interested in potential topic changes as these can be used as part of the filters. + * + * Removes all entities associated with the repository if the repository no longer matches the filters. + * + * @param event - The repository edited event. + * @private + */ + private async onRepoEdited(event: RepositoryEditedEvent) { + const repository = this.createRepoFromEvent(event); + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + await this.removeEntitiesForRepo(repository); + } + // else: repository is (still) matching the filters, so we don't need to do anything + } + + /** + * The name of a repository was changed. + * + * Removes all entities associated with the repository's old name. + * Creates new entities for the repository's new name if it still matches the filters. + * + * @param event - The repository renamed event. + * @private + */ + private async onRepoRenamed(event: RepositoryRenamedEvent) { + const repository = this.createRepoFromEvent(event); + const oldRepoName = event.changes.repository.name.from; + const urlParts = event.repository.url.split('/'); + urlParts[urlParts.length - 1] = oldRepoName; + const oldRepoUrl = urlParts.join('/'); + const oldRepository: Repository = { + ...repository, + name: oldRepoName, + url: oldRepoUrl, + }; + await this.removeEntitiesForRepo(oldRepository); + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`, + ); + return; + } + + await this.addEntitiesForRepo(repository); + } + + /** + * Ownership of the repository was transferred to a user or organization account. + * This event is only sent to the account where the ownership is transferred. + * To receive the `repository.transferred` event, the new owner account must have the GitHub App installed, + * and the App must be subscribed to "Repository" events. + * + * Creates new entities for the repository if it matches the filters. + * + * @param event - The repository unarchived event. + * @private + */ + private async onRepoTransferred(event: RepositoryTransferredEvent) { + const repository = this.createRepoFromEvent(event); + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`, + ); + return; + } + + await this.addEntitiesForRepo(repository); + } + + /** + * A previously archived repository was unarchived. + * + * Creates new entities for the repository if it matches the filters. + * + * @param event - The repository unarchived event. + * @private + */ + private async onRepoUnarchived(event: RepositoryUnarchivedEvent) { + const repository = this.createRepoFromEvent(event); + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`, + ); + return; + } + + await this.addEntitiesForRepo(repository); + } + + private async removeEntitiesForRepo(repository: Repository) { + const removed = this.toDeferredEntitiesFromRepos([repository]); + await this.connection!.applyMutation({ + type: 'delta', + added: [], + removed: removed, + }); + } + + private async addEntitiesForRepo(repository: Repository) { + if (this.config.validateLocationsExist) { + const organization = this.config.organization; + const catalogPath = this.config.catalogPath; + const client = await this.createGraphqlClient(); + + const repositoryFromGithub = await getOrganizationRepository( + client, + organization, + repository.name, + catalogPath, + ).then(r => (r ? this.createRepoFromGithubResponse(r) : null)); + + if (!repositoryFromGithub?.isCatalogInfoFilePresent) { + return; + } + } + + const added = this.toDeferredEntitiesFromRepos([repository]); + await this.connection!.applyMutation({ + type: 'delta', + added: added, + removed: [], + }); + } + + private createRepoFromEvent(event: RepositoryEvent | PushEvent): Repository { + return { + url: event.repository.url, + name: event.repository.name, + defaultBranchRef: event.repository.default_branch, + repositoryTopics: event.repository.topics, + isArchived: event.repository.archived, + isFork: event.repository.fork, + // we can consider this file present because + // only the catalog file will be recovered from the commits + isCatalogInfoFilePresent: true, + visibility: event.repository.visibility, + }; + } + + private createRepoFromGithubResponse( + repositoryResponse: RepositoryResponse, + ): Repository { + return { + url: repositoryResponse.url, + name: repositoryResponse.name, + defaultBranchRef: repositoryResponse.defaultBranchRef?.name, + repositoryTopics: repositoryResponse.repositoryTopics.nodes.map( + t => t.topic.name, + ), + isArchived: repositoryResponse.isArchived, + isFork: repositoryResponse.isFork, + isCatalogInfoFilePresent: + repositoryResponse.catalogInfoFile?.__typename === 'Blob' && + repositoryResponse.catalogInfoFile.text !== '', + visibility: repositoryResponse.visibility, + }; + } + private collectDeferredEntitiesFromCommit( repositoryUrl: string, branch: string, @@ -454,6 +710,20 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { }; }); } + + private toDeferredEntitiesFromRepos( + repositories: Repository[], + ): DeferredEntity[] { + return repositories + .map(repository => this.createLocationUrl(repository)) + .map(GithubEntityProvider.toLocationSpec) + .map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }); + } } /*