From e90b092d2a902c561fd31ebcd45e52fa8aea4dc8 Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Wed, 23 Nov 2022 15:34:39 +0000 Subject: [PATCH] chore(events,catalog/github): update with review comments Signed-off-by: Rogerio Angeliski --- .changeset/old-bulldogs-fry.md | 4 +- docs/integrations/github/discovery.md | 59 +++-- .../api-report.md | 3 - .../src/lib/github.ts | 55 ----- .../providers/GithubEntityProvider.test.ts | 213 +++++++----------- .../src/providers/GithubEntityProvider.ts | 169 +++++--------- .../GithubEntityProviderConfig.test.ts | 7 - .../providers/GithubEntityProviderConfig.ts | 5 - 8 files changed, 196 insertions(+), 319 deletions(-) diff --git a/.changeset/old-bulldogs-fry.md b/.changeset/old-bulldogs-fry.md index 67a57e8fa8..c6bbce6c53 100644 --- a/.changeset/old-bulldogs-fry.md +++ b/.changeset/old-bulldogs-fry.md @@ -2,7 +2,7 @@ '@backstage/plugin-catalog-backend-module-github': patch --- -Handle Github `github.push` events at the `GithubEntityProvider` by subscribing to the topic `github.push.` +Handle Github `push` events at the `GithubEntityProvider` by subscribing to the topic `github.push.` Implements `EventSubscriber` to receive events for the topic `github.push`. @@ -13,4 +13,4 @@ and removing obsolete ones. **Installation and Migration** Please find more information at -https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks +https://backstage.io/docs/integrations/github/discovery#installation-with-events-support diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index c6688d2b65..02310052a1 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -14,7 +14,7 @@ organization and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. This is the preferred method for ingesting entities into the catalog. -## Installation +## Installation without Events Support You will have to add the provider in the catalog initialization code of your backend. They are not installed by default, therefore you have to add a @@ -53,6 +53,51 @@ And then add the entity provider to your catalog builder: } ``` +## Installation with Events Support + +Please follow the installation instructions at + +- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md +- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md + +Additionally, you need to decide how you want to receive events from external sources like + +- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md) +- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md) + +Set up your provider + +```diff +// packages/backend/src/plugins/catalogEventBasedProviders.ts ++import { CatalogClient } from '@backstage/catalog-client'; ++import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; + import { EntityProvider } from '@backstage/plugin-catalog-node'; + import { EventSubscriber } from '@backstage/plugin-events-node'; + import { PluginEnvironment } from '../types'; + export default async function createCatalogEventBasedProviders( +- _: PluginEnvironment, ++ env: PluginEnvironment, + ): Promise> { + const providers: Array< + (EntityProvider & EventSubscriber) | Array + > = []; +- // add your event-based entity providers here ++ providers.push( ++ GithubEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ // optional: alternatively, use scheduler with schedule defined in app-config.yaml ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 30 }, ++ timeout: { minutes: 3 }, ++ }), ++ // optional: alternatively, use schedule ++ scheduler: env.scheduler, ++ }), ++ ); + return providers.flat(); + } +``` + ## Configuration To use the discovery provider, you'll need a GitHub integration @@ -111,13 +156,6 @@ catalog: branch: 'main' # string repository: '.*' # Regex validateLocationsExist: true # optional boolean - checkRepositoryFiltersForWebhook: - organization: 'backstage' # string - catalogPath: '/catalog-info.yaml' # string - filters: - branch: 'main' # string - repository: '.*' # Regex - checkRepositoryFiltersForWebhook: true # optional boolean enterpriseProviderId: host: ghe.example.net organization: 'backstage' # string @@ -160,11 +198,6 @@ This provider supports multiple organizations via unique provider IDs. Defaults to `false`. Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in conjunction with wildcards in the `catalogPath`. -- **`checkRepositoryFiltersForWebhook`** _(optional)_: - Whether to validate push events received from webhook. - This option enforce check the repository from push event to be matched against the filters configured in the entity provider. - Defaults to `false`. - It is disabled for default because you can configure the webhook only in expected repositories - **`schedule`** _(optional)_: - **`frequency`**: How often you want the task to run. The system does its best to avoid overlapping invocations. diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 9f0d060196..7a85d685a3 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -5,7 +5,6 @@ ```ts import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; -import { CatalogApi } from '@backstage/catalog-client'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -86,11 +85,9 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { static fromConfig( config: Config, options: { - catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; - tokenManager?: TokenManager; }, ): GithubEntityProvider[]; // (undocumented) diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 403b06681f..4d583d0730 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -34,7 +34,6 @@ export type QueryResponse = { type RepositoryOwnerResponse = { repositories?: Connection; - repository?: RepositoryResponse; }; export type OrganizationResponse = { @@ -301,60 +300,6 @@ export async function getOrganizationRepositories( return { repositories }; } -export async function getOrganizationRepository( - client: typeof graphql, - org: string, - catalogPath: string, - repoName: string, -): Promise<{ repository: RepositoryResponse | undefined }> { - let relativeCatalogPathRef: string; - // We must strip the leading slash or the query for objects does not work - if (catalogPath.startsWith('/')) { - relativeCatalogPathRef = catalogPath.substring(1); - } else { - relativeCatalogPathRef = catalogPath; - } - const catalogPathRef = `HEAD:${relativeCatalogPathRef}`; - const query = ` - query repositories($org: String!, $catalogPathRef: String!, $repoName: String!) { - repositoryOwner(login: $org) { - login - repository(name: $repoName) { - name - catalogInfoFile: object(expression: $catalogPathRef) { - __typename - ... on Blob { - id - text - } - } - url - isArchived - repositoryTopics(first: 100) { - nodes { - ... on RepositoryTopic { - topic { - name - } - } - } - } - defaultBranchRef { - name - } - } - } - }`; - - const response: QueryResponse = await client(query, { - org, - catalogPathRef, - repoName, - }); - - return { repository: response.repositoryOwner?.repository }; -} - /** * Gets all the users out of a Github organization. * diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index 1d7a71dd0c..fdf65c15a9 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { getVoidLogger, TokenManager } from '@backstage/backend-common'; +import { getVoidLogger } from '@backstage/backend-common'; import { PluginTaskScheduler, TaskInvocationDefinition, @@ -25,12 +25,10 @@ import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GithubEntityProvider } from './GithubEntityProvider'; import * as helpers from '../lib/github'; import { EventParams } from '@backstage/plugin-events-node'; -import { CatalogApi } from '@backstage/catalog-client'; jest.mock('../lib/github', () => { return { getOrganizationRepositories: jest.fn(), - getOrganizationRepository: jest.fn(), }; }); class PersistingTaskRunner implements TaskRunner { @@ -48,15 +46,6 @@ class PersistingTaskRunner implements TaskRunner { const logger = getVoidLogger(); -const mockCatalogApi: Partial = { - refreshEntity: jest.fn(), -}; - -const mockTokenManager: jest.Mocked = { - getToken: jest.fn(), - authenticate: jest.fn(), -}; - describe('GithubEntityProvider', () => { afterEach(() => jest.resetAllMocks()); @@ -752,30 +741,6 @@ describe('GithubEntityProvider', () => { }; await provider.connect(entityProviderConnection); - const mockGetOrganizationRepository = jest.spyOn( - helpers, - 'getOrganizationRepository', - ); - - mockGetOrganizationRepository.mockReturnValue( - Promise.resolve({ - repository: { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { nodes: [] }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - }), - ); - const event: EventParams = { topic: 'github.push', metadata: { @@ -790,6 +755,7 @@ describe('GithubEntityProvider', () => { stargazers: 0, master_branch: 'main', organization: 'test-org', + topics: [], }, created: true, deleted: false, @@ -865,30 +831,6 @@ describe('GithubEntityProvider', () => { }; await provider.connect(entityProviderConnection); - const mockGetOrganizationRepository = jest.spyOn( - helpers, - 'getOrganizationRepository', - ); - - mockGetOrganizationRepository.mockReturnValue( - Promise.resolve({ - repository: { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { nodes: [] }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - }), - ); - const event: EventParams = { topic: 'github.push', metadata: { @@ -903,6 +845,7 @@ describe('GithubEntityProvider', () => { stargazers: 0, master_branch: 'main', organization: 'test-org', + topics: [], }, created: true, deleted: false, @@ -967,12 +910,9 @@ describe('GithubEntityProvider', () => { }, }); - mockTokenManager.getToken.mockResolvedValue({ token: '' }); const provider = GithubEntityProvider.fromConfig(config, { logger, schedule, - tokenManager: mockTokenManager, - catalogApi: mockCatalogApi as unknown as CatalogApi, })[0]; const entityProviderConnection: EntityProviderConnection = { @@ -981,30 +921,6 @@ describe('GithubEntityProvider', () => { }; await provider.connect(entityProviderConnection); - const mockGetOrganizationRepository = jest.spyOn( - helpers, - 'getOrganizationRepository', - ); - - mockGetOrganizationRepository.mockReturnValue( - Promise.resolve({ - repository: { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { nodes: [] }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - }), - ); - const event: EventParams = { topic: 'github.push', metadata: { @@ -1019,6 +935,7 @@ describe('GithubEntityProvider', () => { stargazers: 0, master_branch: 'main', organization: 'test-org', + topics: [], }, created: true, deleted: false, @@ -1040,36 +957,34 @@ describe('GithubEntityProvider', () => { await provider.onEvent(event); - expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1); - expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith( - 'location:default/generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3', - { token: '' }, - ); + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + 'url:https://github.com/test-org/test-repo/tree/main/catalog-info.yaml', + ], + }); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); - it('should recover repository information when match filters from push event', async () => { + it('should process repository when match filters from push event', async () => { const schedule = new PersistingTaskRunner(); const config = new ConfigReader({ catalog: { providers: { github: { organization: 'test-org', - checkRepositoryFiltersForWebhook: true, filters: { branch: 'my-special-branch', + repository: 'test-repo', }, }, }, }, }); - mockTokenManager.getToken.mockResolvedValue({ token: '' }); const provider = GithubEntityProvider.fromConfig(config, { logger, schedule, - tokenManager: mockTokenManager, - catalogApi: mockCatalogApi as unknown as CatalogApi, })[0]; const entityProviderConnection: EntityProviderConnection = { @@ -1078,30 +993,6 @@ describe('GithubEntityProvider', () => { }; await provider.connect(entityProviderConnection); - const mockGetOrganizationRepository = jest.spyOn( - helpers, - 'getOrganizationRepository', - ); - - mockGetOrganizationRepository.mockReturnValue( - Promise.resolve({ - repository: { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { nodes: [] }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - }), - ); - const event: EventParams = { topic: 'github.push', metadata: { @@ -1110,12 +1001,13 @@ describe('GithubEntityProvider', () => { eventPayload: { ref: 'refs/heads/my-special-branch', repository: { - name: 'teste-1', + name: 'test-repo', url: 'https://github.com/test-org/test-repo', default_branch: 'main', stargazers: 0, master_branch: 'main', organization: 'test-org', + topics: [], }, created: true, deleted: false, @@ -1137,11 +1029,78 @@ describe('GithubEntityProvider', () => { await provider.onEvent(event); - expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1); - expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith( - 'location:default/generated-6ffd478ea33caf9af61fa75cc09b5aa7770470f0de', - { token: '' }, - ); + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.refresh).toHaveBeenCalledWith({ + keys: [ + 'url:https://github.com/test-org/test-repo/tree/my-special-branch/catalog-info.yaml', + ], + }); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); + }); + + it("should skip process when didn't match filters from push event", async () => { + const schedule = new PersistingTaskRunner(); + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + filters: { + repository: 'only-special-repository', + }, + }, + }, + }, + }); + + const provider = GithubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + await provider.connect(entityProviderConnection); + + const event: EventParams = { + topic: 'github.push', + metadata: { + 'x-github-event': 'push', + }, + eventPayload: { + ref: 'refs/heads/main', + repository: { + name: 'teste-1', + url: 'https://github.com/test-org/test-repo', + default_branch: 'main', + stargazers: 0, + master_branch: 'main', + organization: 'test-org', + topics: [], + }, + created: true, + deleted: false, + forced: false, + commits: [ + { + added: ['new-file.yaml'], + removed: [], + modified: [], + }, + { + added: [], + removed: [], + modified: ['catalog-info.yaml'], + }, + ], + }, + }; + + await provider.onEvent(event); + + expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts index 4fefb81dbb..2d3e1cd1af 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts @@ -40,20 +40,23 @@ import { readProviderConfigs, GithubEntityProviderConfig, } from './GithubEntityProviderConfig'; -import { - getOrganizationRepositories, - getOrganizationRepository, - RepositoryResponse, -} from '../lib/github'; +import { getOrganizationRepositories } from '../lib/github'; import { satisfiesTopicFilter } from '../lib/util'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; import { PushEvent, Commit } from '@octokit/webhooks-types'; -import { CatalogApi } from '@backstage/catalog-client'; -import { TokenManager } from '@backstage/backend-common'; -import { stringifyEntityRef } from '@backstage/catalog-model'; const TOPIC_REPO_PUSH = 'github.push'; + +type Repository = { + name: string; + url: string; + isArchived: boolean; + repositoryTopics: string[]; + defaultBranchRef: string | null; + isCatalogInfoFilePresent: boolean; +}; + /** * Discovers catalog files located in [GitHub](https://github.com). * The provider will search your GitHub account and register catalog files matching the configured path @@ -68,18 +71,14 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { private readonly integration: GithubIntegrationConfig; private readonly scheduleFn: () => Promise; private connection?: EntityProviderConnection; - private readonly catalogApi?: CatalogApi; - private readonly tokenManager?: TokenManager; private readonly githubCredentialsProvider: GithubCredentialsProvider; static fromConfig( config: Config, options: { - catalogApi?: CatalogApi; logger: Logger; schedule?: TaskRunner; scheduler?: PluginTaskScheduler; - tokenManager?: TokenManager; }, ): GithubEntityProvider[] { if (!options.schedule && !options.scheduler) { @@ -113,8 +112,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { integration, options.logger, taskRunner, - options.catalogApi, - options.tokenManager, ); }); } @@ -124,8 +121,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { integration: GithubIntegration, logger: Logger, taskRunner: TaskRunner, - catalogApi?: CatalogApi, - tokenManager?: TokenManager, ) { this.config = config; this.integration = integration.config; @@ -135,9 +130,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { this.scheduleFn = this.createScheduleFn(taskRunner); this.githubCredentialsProvider = SingleInstanceGithubCredentialsProvider.create(integration.config); - - this.catalogApi = catalogApi; - this.tokenManager = tokenManager; } /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ @@ -200,7 +192,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { } // go to the server and get all of the repositories - private async findCatalogFiles(): Promise { + private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; const catalogPath = this.config.catalogPath; @@ -215,45 +207,49 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { headers, }); - const { repositories } = await getOrganizationRepositories( - client, - organization, - catalogPath, - ); + const { repositories: repositoriesFromGithub } = + await getOrganizationRepositories(client, organization, catalogPath); + const repositories = repositoriesFromGithub.map(r => { + return { + url: r.url, + name: r.name, + defaultBranchRef: r.defaultBranchRef?.name || null, + repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name), + isArchived: r.isArchived, + isCatalogInfoFilePresent: + r.catalogInfoFile?.__typename === 'Blob' && + r.catalogInfoFile.text !== '', + }; + }); if (this.config.validateLocationsExist) { - return repositories.filter(repository => { - return ( - repository.catalogInfoFile?.__typename === 'Blob' && - repository.catalogInfoFile.text !== '' - ); - }); + return repositories.filter( + repository => repository.isCatalogInfoFilePresent, + ); } return repositories; } - private matchesFilters(repositories: RepositoryResponse[]) { + private matchesFilters(repositories: Repository[]) { const repositoryFilter = this.config.filters?.repository; const topicFilters = this.config.filters?.topic; const matchingRepositories = repositories.filter(r => { - const repoTopics: string[] = r.repositoryTopics.nodes.map( - node => node.topic.name, - ); + const repoTopics: string[] = r.repositoryTopics; return ( !r.isArchived && (!repositoryFilter || repositoryFilter.test(r.name)) && satisfiesTopicFilter(repoTopics, topicFilters) && - r.defaultBranchRef?.name + r.defaultBranchRef ); }); return matchingRepositories; } - private createLocationUrl(repository: RepositoryResponse): string { + private createLocationUrl(repository: Repository): string { const branch = - this.config.filters?.branch || repository.defaultBranchRef?.name || '-'; + this.config.filters?.branch || repository.defaultBranchRef || '-'; const catalogFile = this.config.catalogPath.startsWith('/') ? this.config.catalogPath.substring(1) : this.config.catalogPath; @@ -275,11 +271,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { return; } - if (params.metadata?.['x-github-event'] === 'push') { - await this.onRepoPush(params.eventPayload as PushEvent); - } - - return; + await this.onRepoPush(params.eventPayload as PushEvent); } /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */ @@ -304,23 +296,23 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { return; } - if (this.config.checkRepositoryFiltersForWebhook) { - const repository = await this.getRepositoryInfo(repoName); + const repository: Repository = { + url: event.repository.url, + name: event.repository.name, + defaultBranchRef: event.repository.default_branch, + repositoryTopics: event.repository.topics, + isArchived: event.repository.archived, + // we can consider this file present because + // only the catalog file will be recovered from the commits + isCatalogInfoFilePresent: true, + }; - if (!repository) { - this.logger.debug( - `skipping push event from repository ${repoName} because didn't find information in Github`, - ); - return; - } - - const matchingTargets = this.matchesFilters([repository]); - if (matchingTargets.length === 0) { - this.logger.debug( - `skipping push event from repository ${repoName} because didn't match provider filters`, - ); - return; - } + const matchingTargets = this.matchesFilters([repository]); + if (matchingTargets.length === 0) { + this.logger.debug( + `skipping push event from repository ${repoName} because didn't match provider filters`, + ); + return; } // the commit has information about the files (added,removed,modified) @@ -338,36 +330,26 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { event.commits, (commit: Commit) => [...commit.removed], ); - const modifiedCatalogFiles = this.collectFilesFromCommit( + const modified = this.collectFilesFromCommit( event.commits, (commit: Commit) => [...commit.modified], ); const limiter = limiterFactory(10); + const promises: Promise[] = []; - let modified = []; - let promises: Promise[] = []; - - if (this.catalogApi && this.tokenManager) { - modified = modifiedCatalogFiles - .map(filePath => `${event.repository.url}/blob/${branch}/${filePath}`) - .map(url => { - const location = GithubEntityProvider.toLocationSpec(url); - - return locationSpecToLocationEntity({ location }); - }); - - const { token } = await this.tokenManager.getToken(); - - promises = modified.map(entity => + if (modified.length > 0) { + const connection = this.connection; + promises.push( limiter(async () => - this.catalogApi!.refreshEntity(stringifyEntityRef(entity), { token }), + connection.refresh({ + keys: modified.map( + filePath => + `url:${event.repository.url}/tree/${branch}/${filePath}`, + ), + }), ), ); - } else { - this.logger.debug( - `skipping modified operation because is missing CatalogApi and/or TokenManager.`, - ); } if (added.length > 0 || removed.length > 0) { @@ -420,33 +402,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber { .filter(file => catalogFile.includes(file)); } - private async getRepositoryInfo( - repoName: string, - ): Promise { - const organization = this.config.organization; - const host = this.integration.host; - const catalogPath = this.config.catalogPath; - const orgUrl = `https://${host}/${organization}`; - - const { headers } = await this.githubCredentialsProvider.getCredentials({ - url: orgUrl, - }); - - const client = graphql.defaults({ - baseUrl: this.integration.apiBaseUrl, - headers, - }); - - const { repository } = await getOrganizationRepository( - client, - organization, - catalogPath, - repoName, - ); - - return repository; - } - private toDeferredEntities(targets: string[]): DeferredEntity[] { return targets .map(target => { diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 725771bb86..0d9b5ce65b 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -102,7 +102,6 @@ describe('readProviderConfigs', () => { id: 'providerOrganizationOnly', organization: 'test-org1', catalogPath: '/catalog-info.yaml', - checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -120,7 +119,6 @@ describe('readProviderConfigs', () => { organization: 'test-org2', catalogPath: 'custom/path/catalog-info.yaml', host: 'github.com', - checkRepositoryFiltersForWebhook: false, filters: { repository: undefined, branch: undefined, @@ -136,7 +134,6 @@ describe('readProviderConfigs', () => { id: 'providerWithRepositoryFilter', organization: 'test-org3', // organization catalogPath: '/catalog-info.yaml', // file - checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: /^repository.*filter$/, // repo @@ -153,7 +150,6 @@ describe('readProviderConfigs', () => { id: 'providerWithBranchFilter', organization: 'test-org4', catalogPath: '/catalog-info.yaml', - checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -170,7 +166,6 @@ describe('readProviderConfigs', () => { id: 'providerWithTopicFilter', organization: 'test-org5', catalogPath: '/catalog-info.yaml', - checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, @@ -187,7 +182,6 @@ describe('readProviderConfigs', () => { id: 'providerWithHost', organization: 'test-org1', catalogPath: '/catalog-info.yaml', - checkRepositoryFiltersForWebhook: false, host: 'ghe.internal.com', filters: { repository: undefined, @@ -204,7 +198,6 @@ describe('readProviderConfigs', () => { id: 'providerWithSchedule', organization: 'test-org1', catalogPath: '/catalog-info.yaml', - checkRepositoryFiltersForWebhook: false, host: 'github.com', filters: { repository: undefined, diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts index ef5df5733c..e888743258 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts @@ -34,7 +34,6 @@ export type GithubEntityProviderConfig = { topic?: GithubTopicFilters; }; validateLocationsExist: boolean; - checkRepositoryFiltersForWebhook: boolean; schedule?: TaskScheduleDefinition; }; @@ -82,9 +81,6 @@ function readProviderConfig( const validateLocationsExist = config?.getOptionalBoolean('validateLocationsExist') ?? false; - const checkRepositoryFiltersForWebhook = - config?.getOptionalBoolean('checkRepositoryFiltersForWebhook') ?? false; - const catalogPathContainsWildcard = catalogPath.includes('*'); if (validateLocationsExist && catalogPathContainsWildcard) { @@ -114,7 +110,6 @@ function readProviderConfig( }, schedule, validateLocationsExist, - checkRepositoryFiltersForWebhook, }; }