diff --git a/.changeset/old-bulldogs-fry.md b/.changeset/old-bulldogs-fry.md new file mode 100644 index 0000000000..67a57e8fa8 --- /dev/null +++ b/.changeset/old-bulldogs-fry.md @@ -0,0 +1,16 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Handle Github `github.push` events at the `GithubEntityProvider` by subscribing to the topic `github.push.` + +Implements `EventSubscriber` to receive events for the topic `github.push`. + +On `github.push`, the affected repository will be refreshed. +This includes adding new Location entities, refreshing existing ones, +and removing obsolete ones. + +**Installation and Migration** + +Please find more information at +https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 4f032901a9..c6688d2b65 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -111,6 +111,13 @@ catalog: branch: 'main' # string repository: '.*' # Regex validateLocationsExist: true # optional boolean + checkRepositoryFiltersForWebhook: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + checkRepositoryFiltersForWebhook: true # optional boolean enterpriseProviderId: host: ghe.example.net organization: 'backstage' # string @@ -153,6 +160,11 @@ This provider supports multiple organizations via unique provider IDs. Defaults to `false`. Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in conjunction with wildcards in the `catalogPath`. +- **`checkRepositoryFiltersForWebhook`** _(optional)_: + Whether to validate push events received from webhook. + This option enforce check the repository from push event to be matched against the filters configured in the entity provider. + Defaults to `false`. + It is disabled for default because you can configure the webhook only in expected repositories - **`schedule`** _(optional)_: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index e749549646..9f0d060196 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -5,12 +5,15 @@ ```ts import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CatalogApi } from '@backstage/catalog-client'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import { EntityProvider } from '@backstage/plugin-catalog-backend'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { EventParams } from '@backstage/plugin-events-node'; +import { EventSubscriber } from '@backstage/plugin-events-node'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GithubIntegrationConfig } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; @@ -76,22 +79,28 @@ export class GitHubEntityProvider implements EntityProvider { } // @public -export class GithubEntityProvider implements EntityProvider { +export class GithubEntityProvider implements EntityProvider, EventSubscriber { // (undocumented) connect(connection: EntityProviderConnection): Promise; // (undocumented) static fromConfig( config: Config, options: { + catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + tokenManager?: TokenManager; }, ): GithubEntityProvider[]; // (undocumented) getProviderName(): string; // (undocumented) + onEvent(params: EventParams): Promise; + // (undocumented) refresh(logger: Logger): Promise; + // (undocumented) + supportsEventTopics(): string[]; } // @alpha diff --git a/plugins/catalog-backend-module-github/package.json b/plugins/catalog-backend-module-github/package.json index d6d996068e..4676e847d7 100644 --- a/plugins/catalog-backend-module-github/package.json +++ b/plugins/catalog-backend-module-github/package.json @@ -42,13 +42,16 @@ "@backstage/errors": "workspace:^", "@backstage/integration": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-common": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/types": "workspace:^", "@octokit/graphql": "^5.0.0", "@octokit/rest": "^19.0.3", "git-url-parse": "^13.0.0", "lodash": "^4.17.21", "node-fetch": "^2.6.7", + "p-limit": "^3.0.2", "uuid": "^8.0.0", "winston": "^3.2.1" }, diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 4d583d0730..403b06681f 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -34,6 +34,7 @@ export type QueryResponse = { type RepositoryOwnerResponse = { repositories?: Connection; + repository?: RepositoryResponse; }; export type OrganizationResponse = { @@ -300,6 +301,60 @@ export async function getOrganizationRepositories( return { repositories }; } +export async function getOrganizationRepository( + client: typeof graphql, + org: string, + catalogPath: string, + repoName: string, +): Promise<{ repository: RepositoryResponse | undefined }> { + let relativeCatalogPathRef: string; + // We must strip the leading slash or the query for objects does not work + if (catalogPath.startsWith('/')) { + relativeCatalogPathRef = catalogPath.substring(1); + } else { + relativeCatalogPathRef = catalogPath; + } + const catalogPathRef = `HEAD:${relativeCatalogPathRef}`; + const query = ` + query repositories($org: String!, $catalogPathRef: String!, $repoName: String!) { + repositoryOwner(login: $org) { + login + repository(name: $repoName) { + name + catalogInfoFile: object(expression: $catalogPathRef) { + __typename + ... on Blob { + id + text + } + } + url + isArchived + repositoryTopics(first: 100) { + nodes { + ... on RepositoryTopic { + topic { + name + } + } + } + } + defaultBranchRef { + name + } + } + } + }`; + + const response: QueryResponse = await client(query, { + org, + catalogPathRef, + repoName, + }); + + return { repository: response.repositoryOwner?.repository }; +} + /** * Gets all the users out of a Github organization. * diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index cc9a9bfee3..1d7a71dd0c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { getVoidLogger } from '@backstage/backend-common'; +import { getVoidLogger, TokenManager } from '@backstage/backend-common'; import { PluginTaskScheduler, TaskInvocationDefinition, @@ -24,10 +24,13 @@ import { ConfigReader } from '@backstage/config'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GithubEntityProvider } from './GithubEntityProvider'; import * as helpers from '../lib/github'; +import { EventParams } from '@backstage/plugin-events-node'; +import { CatalogApi } from '@backstage/catalog-client'; jest.mock('../lib/github', () => { return { getOrganizationRepositories: jest.fn(), + getOrganizationRepository: jest.fn(), }; }); class PersistingTaskRunner implements TaskRunner { @@ -45,6 +48,15 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); +const mockCatalogApi: Partial = { + refreshEntity: jest.fn(), +}; + +const mockTokenManager: jest.Mocked = { + getToken: jest.fn(), + authenticate: jest.fn(), +}; + describe('GithubEntityProvider', () => { afterEach(() => jest.resetAllMocks()); @@ -716,4 +728,420 @@ describe('GithubEntityProvider', () => { expect(providers).toHaveLength(1); expect(providers[0].getProviderName()).toEqual('github-provider:default'); }); + + it('apply delta update on added files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const mockGetOrganizationRepository = jest.spyOn( + helpers, + 'getOrganizationRepository', + ); + + mockGetOrganizationRepository.mockReturnValue( + Promise.resolve({ + repository: { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + }), + ); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: ['catalog-info.yaml'], + removed: [], + modified: [], + }, + ], + }, + }; + const url = + 'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml'; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:default', + }, + ]; + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: expectedEntities, + removed: [], + }); + }); + + it('apply delta update on removed files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const mockGetOrganizationRepository = jest.spyOn( + helpers, + 'getOrganizationRepository', + ); + + mockGetOrganizationRepository.mockReturnValue( + Promise.resolve({ + repository: { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + }), + ); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: ['catalog-info.yaml'], + modified: [], + }, + ], + }, + }; + const url = + 'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml'; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:default', + }, + ]; + + await provider.onEvent(event); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expectedEntities, + }); + }); + + it('apply refresh call on modified files from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + + mockTokenManager.getToken.mockResolvedValue({ token: '' }); + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + tokenManager: mockTokenManager, + catalogApi: mockCatalogApi as unknown as CatalogApi, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const mockGetOrganizationRepository = jest.spyOn( + helpers, + 'getOrganizationRepository', + ); + + mockGetOrganizationRepository.mockReturnValue( + Promise.resolve({ + repository: { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + }), + ); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1); + expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith( + 'location:default/generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', + { token: '' }, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it('should recover repository information when match filters from push event', async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + checkRepositoryFiltersForWebhook: true, + filters: { + branch: 'my-special-branch', + }, + }, + }, + }, + }); + + mockTokenManager.getToken.mockResolvedValue({ token: '' }); + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + tokenManager: mockTokenManager, + catalogApi: mockCatalogApi as unknown as CatalogApi, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const mockGetOrganizationRepository = jest.spyOn( + helpers, + 'getOrganizationRepository', + ); + + mockGetOrganizationRepository.mockReturnValue( + Promise.resolve({ + repository: { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { nodes: [] }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + }), + ); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/my-special-branch', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1); + expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith( + 'location:default/generated-6ffd478ea33caf9af61fa75cc09b5aa7770470f0de', + { token: '' }, + ); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 9a75f52afa..4fefb81dbb 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -24,22 +24,36 @@ import { SingleInstanceGithubCredentialsProvider, } from '@backstage/integration'; import { + DeferredEntity, EntityProvider, EntityProviderConnection, - LocationSpec, locationSpecToLocationEntity, } from '@backstage/plugin-catalog-backend'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; + import { graphql } from '@octokit/graphql'; import * as uuid from 'uuid'; +import limiterFactory from 'p-limit'; import { Logger } from 'winston'; import { readProviderConfigs, GithubEntityProviderConfig, } from './GithubEntityProviderConfig'; -import { getOrganizationRepositories, RepositoryResponse } from '../lib/github'; +import { + getOrganizationRepositories, + getOrganizationRepository, + RepositoryResponse, +} from '../lib/github'; import { satisfiesTopicFilter } from '../lib/util'; +import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; +import { PushEvent, Commit } from '@octokit/webhooks-types'; +import { CatalogApi } from '@backstage/catalog-client'; +import { TokenManager } from '@backstage/backend-common'; +import { stringifyEntityRef } from '@backstage/catalog-model'; + +const TOPIC_REPO_PUSH = 'github.push'; /** * Discovers catalog files located in [GitHub](https://github.com). * The provider will search your GitHub account and register catalog files matching the configured path @@ -48,20 +62,24 @@ import { satisfiesTopicFilter } from '../lib/util'; * * @public */ -export class GithubEntityProvider implements EntityProvider { +export class GithubEntityProvider implements EntityProvider, EventSubscriber { private readonly config: GithubEntityProviderConfig; private readonly logger: Logger; private readonly integration: GithubIntegrationConfig; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; + private readonly catalogApi?: CatalogApi; + private readonly tokenManager?: TokenManager; private readonly githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, options: { + catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; + tokenManager?: TokenManager; }, ): GithubEntityProvider[] { if (!options.schedule && !options.scheduler) { @@ -95,6 +113,8 @@ export class GithubEntityProvider implements EntityProvider { integration, options.logger, taskRunner, + options.catalogApi, + options.tokenManager, ); }); } @@ -104,6 +124,8 @@ export class GithubEntityProvider implements EntityProvider { integration: GithubIntegration, logger: Logger, taskRunner: TaskRunner, + catalogApi?: CatalogApi, + tokenManager?: TokenManager, ) { this.config = config; this.integration = integration.config; @@ -113,6 +135,9 @@ export class GithubEntityProvider implements EntityProvider { this.scheduleFn = this.createScheduleFn(taskRunner); this.githubCredentialsProvider = SingleInstanceGithubCredentialsProvider.create(integration.config); + + this.catalogApi = catalogApi; + this.tokenManager = tokenManager; } /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ @@ -242,6 +267,200 @@ export class GithubEntityProvider implements EntityProvider { presence: 'optional', }; } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */ + async onEvent(params: EventParams): Promise { + this.logger.debug(`Received event from ${params.topic}`); + if (params.topic !== TOPIC_REPO_PUSH) { + return; + } + + if (params.metadata?.['x-github-event'] === 'push') { + await this.onRepoPush(params.eventPayload as PushEvent); + } + + return; + } + + /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ + supportsEventTopics(): string[] { + return [TOPIC_REPO_PUSH]; + } + + private async onRepoPush(event: PushEvent) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const repoName = event.repository.name; + const repoUrl = event.repository.url; + this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`); + + const branch = + this.config.filters?.branch || event.repository.default_branch; + + if (!event.ref.includes(branch)) { + this.logger.debug(`skipping push event from ref ${event.ref}`); + return; + } + + if (this.config.checkRepositoryFiltersForWebhook) { + const repository = await this.getRepositoryInfo(repoName); + + if (!repository) { + this.logger.debug( + `skipping push event from repository ${repoName} because didn't find information in Github`, + ); + return; + } + + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping push event from repository ${repoName} because didn't match provider filters`, + ); + return; + } + } + + // the commit has information about the files (added,removed,modified) + // so we will process the change based in this data + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push + const added = this.collectDeferredEntitiesFromCommit( + event.repository.url, + branch, + event.commits, + (commit: Commit) => [...commit.added], + ); + const removed = this.collectDeferredEntitiesFromCommit( + event.repository.url, + branch, + event.commits, + (commit: Commit) => [...commit.removed], + ); + const modifiedCatalogFiles = this.collectFilesFromCommit( + event.commits, + (commit: Commit) => [...commit.modified], + ); + + const limiter = limiterFactory(10); + + let modified = []; + let promises: Promise[] = []; + + if (this.catalogApi && this.tokenManager) { + modified = modifiedCatalogFiles + .map(filePath => `${event.repository.url}/blob/${branch}/${filePath}`) + .map(url => { + const location = GithubEntityProvider.toLocationSpec(url); + + return locationSpecToLocationEntity({ location }); + }); + + const { token } = await this.tokenManager.getToken(); + + promises = modified.map(entity => + limiter(async () => + this.catalogApi!.refreshEntity(stringifyEntityRef(entity), { token }), + ), + ); + } else { + this.logger.debug( + `skipping modified operation because is missing CatalogApi and/or TokenManager.`, + ); + } + + if (added.length > 0 || removed.length > 0) { + const connection = this.connection; + promises.push( + limiter(async () => + connection.applyMutation({ + type: 'delta', + added: added, + removed: removed, + }), + ), + ); + } + + await Promise.all(promises); + this.logger.info( + `Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`, + ); + } + + private collectDeferredEntitiesFromCommit( + repositoryUrl: string, + branch: string, + commits: Commit[], + transformOperation: (commit: Commit) => string[], + ): DeferredEntity[] { + const catalogFiles = this.collectFilesFromCommit( + commits, + transformOperation, + ); + return this.toDeferredEntities( + catalogFiles.map( + filePath => `${repositoryUrl}/blob/${branch}/${filePath}`, + ), + ); + } + + private collectFilesFromCommit( + commits: Commit[], + transformOperation: (commit: Commit) => string[], + ): string[] { + const catalogFile = this.config.catalogPath.startsWith('/') + ? this.config.catalogPath.substring(1) + : this.config.catalogPath; + + return commits + .map(transformOperation) + .flat() + .filter(file => catalogFile.includes(file)); + } + + private async getRepositoryInfo( + repoName: string, + ): Promise { + const organization = this.config.organization; + const host = this.integration.host; + const catalogPath = this.config.catalogPath; + const orgUrl = `https://${host}/${organization}`; + + const { headers } = await this.githubCredentialsProvider.getCredentials({ + url: orgUrl, + }); + + const client = graphql.defaults({ + baseUrl: this.integration.apiBaseUrl, + headers, + }); + + const { repository } = await getOrganizationRepository( + client, + organization, + catalogPath, + repoName, + ); + + return repository; + } + + private toDeferredEntities(targets: string[]): DeferredEntity[] { + return targets + .map(target => { + const location = GithubEntityProvider.toLocationSpec(target); + + return locationSpecToLocationEntity({ location }); + }) + .map(entity => { + return { + locationKey: this.getProviderName(), + entity: entity, + }; + }); + } } /* diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 0d9b5ce65b..725771bb86 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -102,6 +102,7 @@ describe('readProviderConfigs', () => { id: 'providerOrganizationOnly', organization: 'test-org1', catalogPath: '/catalog-info.yaml', + checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -119,6 +120,7 @@ describe('readProviderConfigs', () => { organization: 'test-org2', catalogPath: 'custom/path/catalog-info.yaml', host: 'github.com', + checkRepositoryFiltersForWebhook: false, filters: { repository: undefined, branch: undefined, @@ -134,6 +136,7 @@ describe('readProviderConfigs', () => { id: 'providerWithRepositoryFilter', organization: 'test-org3', // organization catalogPath: '/catalog-info.yaml', // file + checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: /^repository.*filter$/, // repo @@ -150,6 +153,7 @@ describe('readProviderConfigs', () => { id: 'providerWithBranchFilter', organization: 'test-org4', catalogPath: '/catalog-info.yaml', + checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -166,6 +170,7 @@ describe('readProviderConfigs', () => { id: 'providerWithTopicFilter', organization: 'test-org5', catalogPath: '/catalog-info.yaml', + checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -182,6 +187,7 @@ describe('readProviderConfigs', () => { id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', + checkRepositoryFiltersForWebhook: false, host: 'ghe.internal.com', filters: { repository: undefined, @@ -198,6 +204,7 @@ describe('readProviderConfigs', () => { id: 'providerWithSchedule', organization: 'test-org1', catalogPath: '/catalog-info.yaml', + checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index e888743258..ef5df5733c 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -34,6 +34,7 @@ export type GithubEntityProviderConfig = { topic?: GithubTopicFilters; }; validateLocationsExist: boolean; + checkRepositoryFiltersForWebhook: boolean; schedule?: TaskScheduleDefinition; }; @@ -81,6 +82,9 @@ function readProviderConfig( const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; + const checkRepositoryFiltersForWebhook = + config?.getOptionalBoolean('checkRepositoryFiltersForWebhook') ?? false; + const catalogPathContainsWildcard = catalogPath.includes('*'); if (validateLocationsExist && catalogPathContainsWildcard) { @@ -110,6 +114,7 @@ function readProviderConfig( }, schedule, validateLocationsExist, + checkRepositoryFiltersForWebhook, }; } diff --git a/yarn.lock b/yarn.lock index 8e9d159818..1bb601d415 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4577,7 +4577,9 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/integration": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-common": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" "@backstage/types": "workspace:^" "@octokit/graphql": ^5.0.0 "@octokit/rest": ^19.0.3 @@ -4587,6 +4589,7 @@ __metadata: luxon: ^3.0.0 msw: ^0.49.0 node-fetch: ^2.6.7 + p-limit: ^3.0.2 uuid: ^8.0.0 winston: ^3.2.1 languageName: unknown