feat(events,catalog/github): handle github:push events

Signed-off-by: Rogerio Angeliski <angeliski@hotmail.com>

plugins/catalog-backend-module-github/api-report.md

@@ -5,12 +5,15 @@
 ```ts
 import { AnalyzeOptions } from '@backstage/plugin-catalog-backend';
 import { BackendFeature } from '@backstage/backend-plugin-api';
+import { CatalogApi } from '@backstage/catalog-client';
 import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
 import { Config } from '@backstage/config';
 import { Entity } from '@backstage/catalog-model';
 import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { EventParams } from '@backstage/plugin-events-node';
+import { EventSubscriber } from '@backstage/plugin-events-node';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GithubIntegrationConfig } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
@@ -76,22 +79,28 @@ export class GitHubEntityProvider implements EntityProvider {
 }
 
 // @public
-export class GithubEntityProvider implements EntityProvider {
+export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
   // (undocumented)
   static fromConfig(
     config: Config,
     options: {
+      catalogApi?: CatalogApi;
       logger: Logger;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
+      tokenManager?: TokenManager;
     },
   ): GithubEntityProvider[];
   // (undocumented)
   getProviderName(): string;
   // (undocumented)
+  onEvent(params: EventParams): Promise<void>;
+  // (undocumented)
   refresh(logger: Logger): Promise<void>;
+  // (undocumented)
+  supportsEventTopics(): string[];
 }
 
 // @alpha

plugins/catalog-backend-module-github/package.json

@@ -42,13 +42,16 @@
     "@backstage/errors": "workspace:^",
     "@backstage/integration": "workspace:^",
     "@backstage/plugin-catalog-backend": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
     "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^",
     "@backstage/types": "workspace:^",
     "@octokit/graphql": "^5.0.0",
     "@octokit/rest": "^19.0.3",
     "git-url-parse": "^13.0.0",
     "lodash": "^4.17.21",
     "node-fetch": "^2.6.7",
+    "p-limit": "^3.0.2",
     "uuid": "^8.0.0",
     "winston": "^3.2.1"
   },

plugins/catalog-backend-module-github/src/lib/github.ts

@@ -34,6 +34,7 @@ export type QueryResponse = {
 
 type RepositoryOwnerResponse = {
   repositories?: Connection<RepositoryResponse>;
+  repository?: RepositoryResponse;
 };
 
 export type OrganizationResponse = {
@@ -300,6 +301,60 @@ export async function getOrganizationRepositories(
   return { repositories };
 }
 
+export async function getOrganizationRepository(
+  client: typeof graphql,
+  org: string,
+  catalogPath: string,
+  repoName: string,
+): Promise<{ repository: RepositoryResponse | undefined }> {
+  let relativeCatalogPathRef: string;
+  // We must strip the leading slash or the query for objects does not work
+  if (catalogPath.startsWith('/')) {
+    relativeCatalogPathRef = catalogPath.substring(1);
+  } else {
+    relativeCatalogPathRef = catalogPath;
+  }
+  const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
+  const query = `
+    query repositories($org: String!, $catalogPathRef: String!, $repoName: String!) {
+      repositoryOwner(login: $org) {
+        login
+        repository(name: $repoName) {
+          name
+          catalogInfoFile: object(expression: $catalogPathRef) {
+            __typename
+            ... on Blob {
+              id
+              text
+            }
+          }
+          url
+          isArchived
+          repositoryTopics(first: 100) {
+            nodes {
+              ... on RepositoryTopic {
+                topic {
+                  name
+                }
+              }
+            }
+          }
+          defaultBranchRef {
+            name
+          }
+        }
+      }
+    }`;
+
+  const response: QueryResponse = await client(query, {
+    org,
+    catalogPathRef,
+    repoName,
+  });
+
+  return { repository: response.repositoryOwner?.repository };
+}
+
 /**
  * Gets all the users out of a Github organization.
  *

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { getVoidLogger } from '@backstage/backend-common';
+import { getVoidLogger, TokenManager } from '@backstage/backend-common';
 import {
   PluginTaskScheduler,
   TaskInvocationDefinition,
@@ -24,10 +24,13 @@ import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubEntityProvider } from './GithubEntityProvider';
 import * as helpers from '../lib/github';
+import { EventParams } from '@backstage/plugin-events-node';
+import { CatalogApi } from '@backstage/catalog-client';
 
 jest.mock('../lib/github', () => {
   return {
     getOrganizationRepositories: jest.fn(),
+    getOrganizationRepository: jest.fn(),
   };
 });
 class PersistingTaskRunner implements TaskRunner {
@@ -45,6 +48,15 @@ class PersistingTaskRunner implements TaskRunner {
 
 const logger = getVoidLogger();
 
+const mockCatalogApi: Partial<CatalogApi> = {
+  refreshEntity: jest.fn(),
+};
+
+const mockTokenManager: jest.Mocked<TokenManager> = {
+  getToken: jest.fn(),
+  authenticate: jest.fn(),
+};
+
 describe('GithubEntityProvider', () => {
   afterEach(() => jest.resetAllMocks());
 
@@ -716,4 +728,420 @@ describe('GithubEntityProvider', () => {
     expect(providers).toHaveLength(1);
     expect(providers[0].getProviderName()).toEqual('github-provider:default');
   });
+
+  it('apply delta update on added files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const mockGetOrganizationRepository = jest.spyOn(
+      helpers,
+      'getOrganizationRepository',
+    );
+
+    mockGetOrganizationRepository.mockReturnValue(
+      Promise.resolve({
+        repository: {
+          name: 'test-repo',
+          url: 'https://github.com/test-org/test-repo',
+          repositoryTopics: { nodes: [] },
+          isArchived: false,
+          defaultBranchRef: {
+            name: 'main',
+          },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
+        },
+      }),
+    );
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: ['catalog-info.yaml'],
+            removed: [],
+            modified: [],
+          },
+        ],
+      },
+    };
+    const url =
+      'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:default',
+      },
+    ];
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: expectedEntities,
+      removed: [],
+    });
+  });
+
+  it('apply delta update on removed files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const mockGetOrganizationRepository = jest.spyOn(
+      helpers,
+      'getOrganizationRepository',
+    );
+
+    mockGetOrganizationRepository.mockReturnValue(
+      Promise.resolve({
+        repository: {
+          name: 'test-repo',
+          url: 'https://github.com/test-org/test-repo',
+          repositoryTopics: { nodes: [] },
+          isArchived: false,
+          defaultBranchRef: {
+            name: 'main',
+          },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
+        },
+      }),
+    );
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: ['catalog-info.yaml'],
+            modified: [],
+          },
+        ],
+      },
+    };
+    const url =
+      'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:default',
+      },
+    ];
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: [],
+      removed: expectedEntities,
+    });
+  });
+
+  it('apply refresh call on modified files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    mockTokenManager.getToken.mockResolvedValue({ token: '' });
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      tokenManager: mockTokenManager,
+      catalogApi: mockCatalogApi as unknown as CatalogApi,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const mockGetOrganizationRepository = jest.spyOn(
+      helpers,
+      'getOrganizationRepository',
+    );
+
+    mockGetOrganizationRepository.mockReturnValue(
+      Promise.resolve({
+        repository: {
+          name: 'test-repo',
+          url: 'https://github.com/test-org/test-repo',
+          repositoryTopics: { nodes: [] },
+          isArchived: false,
+          defaultBranchRef: {
+            name: 'main',
+          },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
+        },
+      }),
+    );
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: [],
+            modified: ['catalog-info.yaml'],
+          },
+        ],
+      },
+    };
+
+    await provider.onEvent(event);
+
+    expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1);
+    expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith(
+      'location:default/generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
+      { token: '' },
+    );
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('should recover repository information when match filters from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+            checkRepositoryFiltersForWebhook: true,
+            filters: {
+              branch: 'my-special-branch',
+            },
+          },
+        },
+      },
+    });
+
+    mockTokenManager.getToken.mockResolvedValue({ token: '' });
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+      tokenManager: mockTokenManager,
+      catalogApi: mockCatalogApi as unknown as CatalogApi,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const mockGetOrganizationRepository = jest.spyOn(
+      helpers,
+      'getOrganizationRepository',
+    );
+
+    mockGetOrganizationRepository.mockReturnValue(
+      Promise.resolve({
+        repository: {
+          name: 'test-repo',
+          url: 'https://github.com/test-org/test-repo',
+          repositoryTopics: { nodes: [] },
+          isArchived: false,
+          defaultBranchRef: {
+            name: 'main',
+          },
+          catalogInfoFile: {
+            __typename: 'Blob',
+            id: 'abc123',
+            text: 'some yaml',
+          },
+        },
+      }),
+    );
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/my-special-branch',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: [],
+            modified: ['catalog-info.yaml'],
+          },
+        ],
+      },
+    };
+
+    await provider.onEvent(event);
+
+    expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1);
+    expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith(
+      'location:default/generated-6ffd478ea33caf9af61fa75cc09b5aa7770470f0de',
+      { token: '' },
+    );
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
 });

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -24,22 +24,36 @@ import {
   SingleInstanceGithubCredentialsProvider,
 } from '@backstage/integration';
 import {
+  DeferredEntity,
   EntityProvider,
   EntityProviderConnection,
-  LocationSpec,
   locationSpecToLocationEntity,
 } from '@backstage/plugin-catalog-backend';
 
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+
 import { graphql } from '@octokit/graphql';
 import * as uuid from 'uuid';
+import limiterFactory from 'p-limit';
 import { Logger } from 'winston';
 import {
   readProviderConfigs,
   GithubEntityProviderConfig,
 } from './GithubEntityProviderConfig';
-import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
+import {
+  getOrganizationRepositories,
+  getOrganizationRepository,
+  RepositoryResponse,
+} from '../lib/github';
 import { satisfiesTopicFilter } from '../lib/util';
 
+import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
+import { PushEvent, Commit } from '@octokit/webhooks-types';
+import { CatalogApi } from '@backstage/catalog-client';
+import { TokenManager } from '@backstage/backend-common';
+import { stringifyEntityRef } from '@backstage/catalog-model';
+
+const TOPIC_REPO_PUSH = 'github.push';
 /**
  * Discovers catalog files located in [GitHub](https://github.com).
  * The provider will search your GitHub account and register catalog files matching the configured path
@@ -48,20 +62,24 @@ import { satisfiesTopicFilter } from '../lib/util';
  *
  * @public
  */
-export class GithubEntityProvider implements EntityProvider {
+export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   private readonly config: GithubEntityProviderConfig;
   private readonly logger: Logger;
   private readonly integration: GithubIntegrationConfig;
   private readonly scheduleFn: () => Promise<void>;
   private connection?: EntityProviderConnection;
+  private readonly catalogApi?: CatalogApi;
+  private readonly tokenManager?: TokenManager;
   private readonly githubCredentialsProvider: GithubCredentialsProvider;
 
   static fromConfig(
     config: Config,
     options: {
+      catalogApi?: CatalogApi;
       logger: Logger;
       schedule?: TaskRunner;
       scheduler?: PluginTaskScheduler;
+      tokenManager?: TokenManager;
     },
   ): GithubEntityProvider[] {
     if (!options.schedule && !options.scheduler) {
@@ -95,6 +113,8 @@ export class GithubEntityProvider implements EntityProvider {
         integration,
         options.logger,
         taskRunner,
+        options.catalogApi,
+        options.tokenManager,
       );
     });
   }
@@ -104,6 +124,8 @@ export class GithubEntityProvider implements EntityProvider {
     integration: GithubIntegration,
     logger: Logger,
     taskRunner: TaskRunner,
+    catalogApi?: CatalogApi,
+    tokenManager?: TokenManager,
   ) {
     this.config = config;
     this.integration = integration.config;
@@ -113,6 +135,9 @@ export class GithubEntityProvider implements EntityProvider {
     this.scheduleFn = this.createScheduleFn(taskRunner);
     this.githubCredentialsProvider =
       SingleInstanceGithubCredentialsProvider.create(integration.config);
+
+    this.catalogApi = catalogApi;
+    this.tokenManager = tokenManager;
   }
 
   /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
@@ -242,6 +267,200 @@ export class GithubEntityProvider implements EntityProvider {
       presence: 'optional',
     };
   }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
+  async onEvent(params: EventParams): Promise<void> {
+    this.logger.debug(`Received event from ${params.topic}`);
+    if (params.topic !== TOPIC_REPO_PUSH) {
+      return;
+    }
+
+    if (params.metadata?.['x-github-event'] === 'push') {
+      await this.onRepoPush(params.eventPayload as PushEvent);
+    }
+
+    return;
+  }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
+  supportsEventTopics(): string[] {
+    return [TOPIC_REPO_PUSH];
+  }
+
+  private async onRepoPush(event: PushEvent) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const repoName = event.repository.name;
+    const repoUrl = event.repository.url;
+    this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`);
+
+    const branch =
+      this.config.filters?.branch || event.repository.default_branch;
+
+    if (!event.ref.includes(branch)) {
+      this.logger.debug(`skipping push event from ref ${event.ref}`);
+      return;
+    }
+
+    if (this.config.checkRepositoryFiltersForWebhook) {
+      const repository = await this.getRepositoryInfo(repoName);
+
+      if (!repository) {
+        this.logger.debug(
+          `skipping push event from repository ${repoName} because didn't find information in Github`,
+        );
+        return;
+      }
+
+      const matchingTargets = this.matchesFilters([repository]);
+      if (matchingTargets.length === 0) {
+        this.logger.debug(
+          `skipping push event from repository ${repoName} because didn't match provider filters`,
+        );
+        return;
+      }
+    }
+
+    // the commit has information about the files (added,removed,modified)
+    // so we will process the change based in this data
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push
+    const added = this.collectDeferredEntitiesFromCommit(
+      event.repository.url,
+      branch,
+      event.commits,
+      (commit: Commit) => [...commit.added],
+    );
+    const removed = this.collectDeferredEntitiesFromCommit(
+      event.repository.url,
+      branch,
+      event.commits,
+      (commit: Commit) => [...commit.removed],
+    );
+    const modifiedCatalogFiles = this.collectFilesFromCommit(
+      event.commits,
+      (commit: Commit) => [...commit.modified],
+    );
+
+    const limiter = limiterFactory(10);
+
+    let modified = [];
+    let promises: Promise<void>[] = [];
+
+    if (this.catalogApi && this.tokenManager) {
+      modified = modifiedCatalogFiles
+        .map(filePath => `${event.repository.url}/blob/${branch}/${filePath}`)
+        .map(url => {
+          const location = GithubEntityProvider.toLocationSpec(url);
+
+          return locationSpecToLocationEntity({ location });
+        });
+
+      const { token } = await this.tokenManager.getToken();
+
+      promises = modified.map(entity =>
+        limiter(async () =>
+          this.catalogApi!.refreshEntity(stringifyEntityRef(entity), { token }),
+        ),
+      );
+    } else {
+      this.logger.debug(
+        `skipping modified operation because is missing CatalogApi and/or TokenManager.`,
+      );
+    }
+
+    if (added.length > 0 || removed.length > 0) {
+      const connection = this.connection;
+      promises.push(
+        limiter(async () =>
+          connection.applyMutation({
+            type: 'delta',
+            added: added,
+            removed: removed,
+          }),
+        ),
+      );
+    }
+
+    await Promise.all(promises);
+    this.logger.info(
+      `Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`,
+    );
+  }
+
+  private collectDeferredEntitiesFromCommit(
+    repositoryUrl: string,
+    branch: string,
+    commits: Commit[],
+    transformOperation: (commit: Commit) => string[],
+  ): DeferredEntity[] {
+    const catalogFiles = this.collectFilesFromCommit(
+      commits,
+      transformOperation,
+    );
+    return this.toDeferredEntities(
+      catalogFiles.map(
+        filePath => `${repositoryUrl}/blob/${branch}/${filePath}`,
+      ),
+    );
+  }
+
+  private collectFilesFromCommit(
+    commits: Commit[],
+    transformOperation: (commit: Commit) => string[],
+  ): string[] {
+    const catalogFile = this.config.catalogPath.startsWith('/')
+      ? this.config.catalogPath.substring(1)
+      : this.config.catalogPath;
+
+    return commits
+      .map(transformOperation)
+      .flat()
+      .filter(file => catalogFile.includes(file));
+  }
+
+  private async getRepositoryInfo(
+    repoName: string,
+  ): Promise<RepositoryResponse | undefined> {
+    const organization = this.config.organization;
+    const host = this.integration.host;
+    const catalogPath = this.config.catalogPath;
+    const orgUrl = `https://${host}/${organization}`;
+
+    const { headers } = await this.githubCredentialsProvider.getCredentials({
+      url: orgUrl,
+    });
+
+    const client = graphql.defaults({
+      baseUrl: this.integration.apiBaseUrl,
+      headers,
+    });
+
+    const { repository } = await getOrganizationRepository(
+      client,
+      organization,
+      catalogPath,
+      repoName,
+    );
+
+    return repository;
+  }
+
+  private toDeferredEntities(targets: string[]): DeferredEntity[] {
+    return targets
+      .map(target => {
+        const location = GithubEntityProvider.toLocationSpec(target);
+
+        return locationSpecToLocationEntity({ location });
+      })
+      .map(entity => {
+        return {
+          locationKey: this.getProviderName(),
+          entity: entity,
+        };
+      });
+  }
 }
 
 /*

plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts

@@ -102,6 +102,7 @@ describe('readProviderConfigs', () => {
       id: 'providerOrganizationOnly',
       organization: 'test-org1',
       catalogPath: '/catalog-info.yaml',
+      checkRepositoryFiltersForWebhook: false,
       host: 'github.com',
       filters: {
         repository: undefined,
@@ -119,6 +120,7 @@ describe('readProviderConfigs', () => {
       organization: 'test-org2',
       catalogPath: 'custom/path/catalog-info.yaml',
       host: 'github.com',
+      checkRepositoryFiltersForWebhook: false,
       filters: {
         repository: undefined,
         branch: undefined,
@@ -134,6 +136,7 @@ describe('readProviderConfigs', () => {
       id: 'providerWithRepositoryFilter',
       organization: 'test-org3', // organization
       catalogPath: '/catalog-info.yaml', // file
+      checkRepositoryFiltersForWebhook: false,
       host: 'github.com',
       filters: {
         repository: /^repository.*filter$/, // repo
@@ -150,6 +153,7 @@ describe('readProviderConfigs', () => {
       id: 'providerWithBranchFilter',
       organization: 'test-org4',
       catalogPath: '/catalog-info.yaml',
+      checkRepositoryFiltersForWebhook: false,
       host: 'github.com',
       filters: {
         repository: undefined,
@@ -166,6 +170,7 @@ describe('readProviderConfigs', () => {
       id: 'providerWithTopicFilter',
       organization: 'test-org5',
       catalogPath: '/catalog-info.yaml',
+      checkRepositoryFiltersForWebhook: false,
       host: 'github.com',
       filters: {
         repository: undefined,
@@ -182,6 +187,7 @@ describe('readProviderConfigs', () => {
       id: 'providerWithHost',
       organization: 'test-org1',
       catalogPath: '/catalog-info.yaml',
+      checkRepositoryFiltersForWebhook: false,
       host: 'ghe.internal.com',
       filters: {
         repository: undefined,
@@ -198,6 +204,7 @@ describe('readProviderConfigs', () => {
       id: 'providerWithSchedule',
       organization: 'test-org1',
       catalogPath: '/catalog-info.yaml',
+      checkRepositoryFiltersForWebhook: false,
       host: 'github.com',
       filters: {
         repository: undefined,

plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts

@@ -34,6 +34,7 @@ export type GithubEntityProviderConfig = {
     topic?: GithubTopicFilters;
   };
   validateLocationsExist: boolean;
+  checkRepositoryFiltersForWebhook: boolean;
   schedule?: TaskScheduleDefinition;
 };
 
@@ -81,6 +82,9 @@ function readProviderConfig(
   const validateLocationsExist =
     config?.getOptionalBoolean('validateLocationsExist') ?? false;
 
+  const checkRepositoryFiltersForWebhook =
+    config?.getOptionalBoolean('checkRepositoryFiltersForWebhook') ?? false;
+
   const catalogPathContainsWildcard = catalogPath.includes('*');
 
   if (validateLocationsExist && catalogPathContainsWildcard) {
@@ -110,6 +114,7 @@ function readProviderConfig(
     },
     schedule,
     validateLocationsExist,
+    checkRepositoryFiltersForWebhook,
   };
 }