Merge pull request #14758 from angeliski/add-github-event-handler

Implements EventSubscriber in GithubEntityProvider

plugins/catalog-backend-module-github/api-report.md

@@ -11,6 +11,8 @@ import { Config } from '@backstage/config';
 import { Entity } from '@backstage/catalog-model';
 import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
+import { EventParams } from '@backstage/plugin-events-node';
+import { EventSubscriber } from '@backstage/plugin-events-node';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GithubIntegrationConfig } from '@backstage/integration';
 import { graphql } from '@octokit/graphql';
@@ -76,7 +78,7 @@ export class GitHubEntityProvider implements EntityProvider {
 }
 
 // @public
-export class GithubEntityProvider implements EntityProvider {
+export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;
   // (undocumented)
@@ -91,7 +93,11 @@ export class GithubEntityProvider implements EntityProvider {
   // (undocumented)
   getProviderName(): string;
   // (undocumented)
+  onEvent(params: EventParams): Promise<void>;
+  // (undocumented)
   refresh(logger: Logger): Promise<void>;
+  // (undocumented)
+  supportsEventTopics(): string[];
 }
 
 // @alpha

plugins/catalog-backend-module-github/package.json

@@ -42,7 +42,9 @@
     "@backstage/errors": "workspace:^",
     "@backstage/integration": "workspace:^",
     "@backstage/plugin-catalog-backend": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
     "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^",
     "@backstage/types": "workspace:^",
     "@octokit/graphql": "^5.0.0",
     "@octokit/rest": "^19.0.3",

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts

@@ -24,6 +24,7 @@ import { ConfigReader } from '@backstage/config';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubEntityProvider } from './GithubEntityProvider';
 import * as helpers from '../lib/github';
+import { EventParams } from '@backstage/plugin-events-node';
 
 jest.mock('../lib/github', () => {
   return {
@@ -716,4 +717,392 @@ describe('GithubEntityProvider', () => {
     expect(providers).toHaveLength(1);
     expect(providers[0].getProviderName()).toEqual('github-provider:default');
   });
+
+  it('apply delta update on added files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+          topics: [],
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: ['catalog-info.yaml'],
+            removed: [],
+            modified: [],
+          },
+        ],
+      },
+    };
+    const url =
+      'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:default',
+      },
+    ];
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: expectedEntities,
+      removed: [],
+    });
+  });
+
+  it('apply delta update on removed files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+          topics: [],
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: ['catalog-info.yaml'],
+            modified: [],
+          },
+        ],
+      },
+    };
+    const url =
+      'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
+    const expectedEntities = [
+      {
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Location',
+          metadata: {
+            annotations: {
+              'backstage.io/managed-by-location': `url:${url}`,
+              'backstage.io/managed-by-origin-location': `url:${url}`,
+            },
+            name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
+          },
+          spec: {
+            presence: 'optional',
+            target: `${url}`,
+            type: 'url',
+          },
+        },
+        locationKey: 'github-provider:default',
+      },
+    ];
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
+      type: 'delta',
+      added: [],
+      removed: expectedEntities,
+    });
+  });
+
+  it('apply refresh call on modified files from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+          topics: [],
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: [],
+            modified: ['catalog-info.yaml'],
+          },
+        ],
+      },
+    };
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
+      keys: [
+        'url:https://github.com/test-org/test-repo/tree/main/catalog-info.yaml',
+        'url:https://github.com/test-org/test-repo/blob/main/catalog-info.yaml',
+      ],
+    });
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it('should process repository when match filters from push event', async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+            filters: {
+              branch: 'my-special-branch',
+              repository: 'test-repo',
+            },
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/my-special-branch',
+        repository: {
+          name: 'test-repo',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+          topics: [],
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: [],
+            modified: ['catalog-info.yaml'],
+          },
+        ],
+      },
+    };
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
+    expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
+      keys: [
+        'url:https://github.com/test-org/test-repo/tree/my-special-branch/catalog-info.yaml',
+        'url:https://github.com/test-org/test-repo/blob/my-special-branch/catalog-info.yaml',
+      ],
+    });
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
+
+  it("should skip process when didn't match filters from push event", async () => {
+    const schedule = new PersistingTaskRunner();
+    const config = new ConfigReader({
+      catalog: {
+        providers: {
+          github: {
+            organization: 'test-org',
+            filters: {
+              repository: 'only-special-repository',
+            },
+          },
+        },
+      },
+    });
+
+    const provider = GithubEntityProvider.fromConfig(config, {
+      logger,
+      schedule,
+    })[0];
+
+    const entityProviderConnection: EntityProviderConnection = {
+      applyMutation: jest.fn(),
+      refresh: jest.fn(),
+    };
+    await provider.connect(entityProviderConnection);
+
+    const event: EventParams = {
+      topic: 'github.push',
+      metadata: {
+        'x-github-event': 'push',
+      },
+      eventPayload: {
+        ref: 'refs/heads/main',
+        repository: {
+          name: 'teste-1',
+          url: 'https://github.com/test-org/test-repo',
+          default_branch: 'main',
+          stargazers: 0,
+          master_branch: 'main',
+          organization: 'test-org',
+          topics: [],
+        },
+        created: true,
+        deleted: false,
+        forced: false,
+        commits: [
+          {
+            added: ['new-file.yaml'],
+            removed: [],
+            modified: [],
+          },
+          {
+            added: [],
+            removed: [],
+            modified: ['catalog-info.yaml'],
+          },
+        ],
+      },
+    };
+
+    await provider.onEvent(event);
+
+    expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
+    expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
+  });
 });

plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts

@@ -24,12 +24,14 @@ import {
   SingleInstanceGithubCredentialsProvider,
 } from '@backstage/integration';
 import {
+  DeferredEntity,
   EntityProvider,
   EntityProviderConnection,
-  LocationSpec,
   locationSpecToLocationEntity,
 } from '@backstage/plugin-catalog-backend';
 
+import { LocationSpec } from '@backstage/plugin-catalog-common';
+
 import { graphql } from '@octokit/graphql';
 import * as uuid from 'uuid';
 import { Logger } from 'winston';
@@ -37,9 +39,23 @@ import {
   readProviderConfigs,
   GithubEntityProviderConfig,
 } from './GithubEntityProviderConfig';
-import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
+import { getOrganizationRepositories } from '../lib/github';
 import { satisfiesTopicFilter } from '../lib/util';
 
+import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
+import { PushEvent, Commit } from '@octokit/webhooks-types';
+
+const TOPIC_REPO_PUSH = 'github.push';
+
+type Repository = {
+  name: string;
+  url: string;
+  isArchived: boolean;
+  repositoryTopics: string[];
+  defaultBranchRef?: string;
+  isCatalogInfoFilePresent: boolean;
+};
+
 /**
  * Discovers catalog files located in [GitHub](https://github.com).
  * The provider will search your GitHub account and register catalog files matching the configured path
@@ -48,7 +64,7 @@ import { satisfiesTopicFilter } from '../lib/util';
  *
  * @public
  */
-export class GithubEntityProvider implements EntityProvider {
+export class GithubEntityProvider implements EntityProvider, EventSubscriber {
   private readonly config: GithubEntityProviderConfig;
   private readonly logger: Logger;
   private readonly integration: GithubIntegrationConfig;
@@ -175,7 +191,7 @@ export class GithubEntityProvider implements EntityProvider {
   }
 
   // go to the server and get all of the repositories
-  private async findCatalogFiles(): Promise<RepositoryResponse[]> {
+  private async findCatalogFiles(): Promise<Repository[]> {
     const organization = this.config.organization;
     const host = this.integration.host;
     const catalogPath = this.config.catalogPath;
@@ -190,45 +206,49 @@ export class GithubEntityProvider implements EntityProvider {
       headers,
     });
 
-    const { repositories } = await getOrganizationRepositories(
-      client,
-      organization,
-      catalogPath,
-    );
+    const { repositories: repositoriesFromGithub } =
+      await getOrganizationRepositories(client, organization, catalogPath);
+    const repositories = repositoriesFromGithub.map(r => {
+      return {
+        url: r.url,
+        name: r.name,
+        defaultBranchRef: r.defaultBranchRef?.name,
+        repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name),
+        isArchived: r.isArchived,
+        isCatalogInfoFilePresent:
+          r.catalogInfoFile?.__typename === 'Blob' &&
+          r.catalogInfoFile.text !== '',
+      };
+    });
 
     if (this.config.validateLocationsExist) {
-      return repositories.filter(repository => {
-        return (
-          repository.catalogInfoFile?.__typename === 'Blob' &&
-          repository.catalogInfoFile.text !== ''
-        );
-      });
+      return repositories.filter(
+        repository => repository.isCatalogInfoFilePresent,
+      );
     }
 
     return repositories;
   }
 
-  private matchesFilters(repositories: RepositoryResponse[]) {
+  private matchesFilters(repositories: Repository[]) {
     const repositoryFilter = this.config.filters?.repository;
     const topicFilters = this.config.filters?.topic;
 
     const matchingRepositories = repositories.filter(r => {
-      const repoTopics: string[] = r.repositoryTopics.nodes.map(
-        node => node.topic.name,
-      );
+      const repoTopics: string[] = r.repositoryTopics;
       return (
         !r.isArchived &&
         (!repositoryFilter || repositoryFilter.test(r.name)) &&
         satisfiesTopicFilter(repoTopics, topicFilters) &&
-        r.defaultBranchRef?.name
+        r.defaultBranchRef
       );
     });
     return matchingRepositories;
   }
 
-  private createLocationUrl(repository: RepositoryResponse): string {
+  private createLocationUrl(repository: Repository): string {
     const branch =
-      this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
+      this.config.filters?.branch || repository.defaultBranchRef || '-';
     const catalogFile = this.config.catalogPath.startsWith('/')
       ? this.config.catalogPath.substring(1)
       : this.config.catalogPath;
@@ -242,6 +262,151 @@ export class GithubEntityProvider implements EntityProvider {
       presence: 'optional',
     };
   }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
+  async onEvent(params: EventParams): Promise<void> {
+    this.logger.debug(`Received event from ${params.topic}`);
+    if (params.topic !== TOPIC_REPO_PUSH) {
+      return;
+    }
+
+    await this.onRepoPush(params.eventPayload as PushEvent);
+  }
+
+  /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
+  supportsEventTopics(): string[] {
+    return [TOPIC_REPO_PUSH];
+  }
+
+  private async onRepoPush(event: PushEvent) {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const repoName = event.repository.name;
+    const repoUrl = event.repository.url;
+    this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`);
+
+    const branch =
+      this.config.filters?.branch || event.repository.default_branch;
+
+    if (!event.ref.includes(branch)) {
+      this.logger.debug(`skipping push event from ref ${event.ref}`);
+      return;
+    }
+
+    const repository: Repository = {
+      url: event.repository.url,
+      name: event.repository.name,
+      defaultBranchRef: event.repository.default_branch,
+      repositoryTopics: event.repository.topics,
+      isArchived: event.repository.archived,
+      // we can consider this file present because
+      // only the catalog file will be recovered from the commits
+      isCatalogInfoFilePresent: true,
+    };
+
+    const matchingTargets = this.matchesFilters([repository]);
+    if (matchingTargets.length === 0) {
+      this.logger.debug(
+        `skipping push event from repository ${repoName} because didn't match provider filters`,
+      );
+      return;
+    }
+
+    // the commit has information about the files (added,removed,modified)
+    // so we will process the change based in this data
+    // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push
+    const added = this.collectDeferredEntitiesFromCommit(
+      event.repository.url,
+      branch,
+      event.commits,
+      (commit: Commit) => [...commit.added],
+    );
+    const removed = this.collectDeferredEntitiesFromCommit(
+      event.repository.url,
+      branch,
+      event.commits,
+      (commit: Commit) => [...commit.removed],
+    );
+    const modified = this.collectFilesFromCommit(
+      event.commits,
+      (commit: Commit) => [...commit.modified],
+    );
+
+    if (modified.length > 0) {
+      await this.connection.refresh({
+        keys: [
+          ...modified.map(
+            filePath =>
+              `url:${event.repository.url}/tree/${branch}/${filePath}`,
+          ),
+          ...modified.map(
+            filePath =>
+              `url:${event.repository.url}/blob/${branch}/${filePath}`,
+          ),
+        ],
+      });
+    }
+
+    if (added.length > 0 || removed.length > 0) {
+      await this.connection.applyMutation({
+        type: 'delta',
+        added: added,
+        removed: removed,
+      });
+    }
+
+    this.logger.info(
+      `Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`,
+    );
+  }
+
+  private collectDeferredEntitiesFromCommit(
+    repositoryUrl: string,
+    branch: string,
+    commits: Commit[],
+    transformOperation: (commit: Commit) => string[],
+  ): DeferredEntity[] {
+    const catalogFiles = this.collectFilesFromCommit(
+      commits,
+      transformOperation,
+    );
+    return this.toDeferredEntities(
+      catalogFiles.map(
+        filePath => `${repositoryUrl}/blob/${branch}/${filePath}`,
+      ),
+    );
+  }
+
+  private collectFilesFromCommit(
+    commits: Commit[],
+    transformOperation: (commit: Commit) => string[],
+  ): string[] {
+    const catalogFile = this.config.catalogPath.startsWith('/')
+      ? this.config.catalogPath.substring(1)
+      : this.config.catalogPath;
+
+    return commits
+      .map(transformOperation)
+      .flat()
+      .filter(file => catalogFile.includes(file));
+  }
+
+  private toDeferredEntities(targets: string[]): DeferredEntity[] {
+    return targets
+      .map(target => {
+        const location = GithubEntityProvider.toLocationSpec(target);
+
+        return locationSpecToLocationEntity({ location });
+      })
+      .map(entity => {
+        return {
+          locationKey: this.getProviderName(),
+          entity: entity,
+        };
+      });
+  }
 }
 
 /*