Merge pull request #14758 from angeliski/add-github-event-handler

Implements EventSubscriber in GithubEntityProvider
This commit is contained in:
Johan Haals
2022-11-25 15:54:52 +01:00
committed by GitHub
7 changed files with 648 additions and 24 deletions
+14
View File
@@ -0,0 +1,14 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Handle GitHub `push` events at the `GithubEntityProvider` by subscribing to the topic `github.push.`
Implements `EventSubscriber` to receive events for the topic `github.push`.
On `github.push`, the affected repository will be refreshed.
This includes adding new Location entities, refreshing existing ones,
and removing obsolete ones.
Please find more information at
https://backstage.io/docs/integrations/github/discovery#installation-with-events-support
+47 -1
View File
@@ -14,7 +14,7 @@ organization and register entities matching the configured path. This can be
useful as an alternative to static locations or manually adding things to the
catalog. This is the preferred method for ingesting entities into the catalog.
## Installation
## Installation without Events Support
You will have to add the provider in the catalog initialization code of your
backend. They are not installed by default, therefore you have to add a
@@ -53,6 +53,52 @@ And then add the entity provider to your catalog builder:
}
```
## Installation with Events Support
Please follow the installation instructions at
- https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md
- https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md
Additionally, you need to decide how you want to receive events from external sources like
- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
Set up your provider
```diff
// packages/backend/src/plugins/catalogEventBasedProviders.ts
+import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { PluginEnvironment } from '../types';
export default async function createCatalogEventBasedProviders(
- _: PluginEnvironment,
+ env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
- // add your event-based entity providers here
+ providers.push(
+ GithubEntityProvider.fromConfig(env.config, {
+ logger: env.logger,
+ // optional: alternatively, use scheduler with schedule defined in app-config.yaml
+ schedule: env.scheduler.createScheduledTaskRunner({
+ frequency: { minutes: 30 },
+ timeout: { minutes: 3 },
+ }),
+ // optional: alternatively, use schedule
+ scheduler: env.scheduler,
+ }),
+ );
return providers.flat();
}
```
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `push` events.
## Configuration
To use the discovery provider, you'll need a GitHub integration
@@ -11,6 +11,8 @@ import { Config } from '@backstage/config';
import { Entity } from '@backstage/catalog-model';
import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { EventParams } from '@backstage/plugin-events-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { GithubCredentialsProvider } from '@backstage/integration';
import { GithubIntegrationConfig } from '@backstage/integration';
import { graphql } from '@octokit/graphql';
@@ -76,7 +78,7 @@ export class GitHubEntityProvider implements EntityProvider {
}
// @public
export class GithubEntityProvider implements EntityProvider {
export class GithubEntityProvider implements EntityProvider, EventSubscriber {
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
@@ -91,7 +93,11 @@ export class GithubEntityProvider implements EntityProvider {
// (undocumented)
getProviderName(): string;
// (undocumented)
onEvent(params: EventParams): Promise<void>;
// (undocumented)
refresh(logger: Logger): Promise<void>;
// (undocumented)
supportsEventTopics(): string[];
}
// @alpha
@@ -42,7 +42,9 @@
"@backstage/errors": "workspace:^",
"@backstage/integration": "workspace:^",
"@backstage/plugin-catalog-backend": "workspace:^",
"@backstage/plugin-catalog-common": "workspace:^",
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/plugin-events-node": "workspace:^",
"@backstage/types": "workspace:^",
"@octokit/graphql": "^5.0.0",
"@octokit/rest": "^19.0.3",
@@ -24,6 +24,7 @@ import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GithubEntityProvider } from './GithubEntityProvider';
import * as helpers from '../lib/github';
import { EventParams } from '@backstage/plugin-events-node';
jest.mock('../lib/github', () => {
return {
@@ -716,4 +717,392 @@ describe('GithubEntityProvider', () => {
expect(providers).toHaveLength(1);
expect(providers[0].getProviderName()).toEqual('github-provider:default');
});
it('apply delta update on added files from push event', async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/main',
repository: {
name: 'teste-1',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: ['catalog-info.yaml'],
removed: [],
modified: [],
},
],
},
};
const url =
'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:default',
},
];
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: expectedEntities,
removed: [],
});
});
it('apply delta update on removed files from push event', async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/main',
repository: {
name: 'teste-1',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: [],
removed: ['catalog-info.yaml'],
modified: [],
},
],
},
};
const url =
'https://github.com/test-org/test-repo/blob/main/catalog-info.yaml';
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:default',
},
];
await provider.onEvent(event);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expectedEntities,
});
});
it('apply refresh call on modified files from push event', async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/main',
repository: {
name: 'teste-1',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: [],
removed: [],
modified: ['catalog-info.yaml'],
},
],
},
};
await provider.onEvent(event);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
keys: [
'url:https://github.com/test-org/test-repo/tree/main/catalog-info.yaml',
'url:https://github.com/test-org/test-repo/blob/main/catalog-info.yaml',
],
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
it('should process repository when match filters from push event', async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
filters: {
branch: 'my-special-branch',
repository: 'test-repo',
},
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/my-special-branch',
repository: {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: [],
removed: [],
modified: ['catalog-info.yaml'],
},
],
},
};
await provider.onEvent(event);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
keys: [
'url:https://github.com/test-org/test-repo/tree/my-special-branch/catalog-info.yaml',
'url:https://github.com/test-org/test-repo/blob/my-special-branch/catalog-info.yaml',
],
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
it("should skip process when didn't match filters from push event", async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
filters: {
repository: 'only-special-repository',
},
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/main',
repository: {
name: 'teste-1',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: [],
removed: [],
modified: ['catalog-info.yaml'],
},
],
},
};
await provider.onEvent(event);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
});
@@ -24,12 +24,14 @@ import {
SingleInstanceGithubCredentialsProvider,
} from '@backstage/integration';
import {
DeferredEntity,
EntityProvider,
EntityProviderConnection,
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-common';
import { graphql } from '@octokit/graphql';
import * as uuid from 'uuid';
import { Logger } from 'winston';
@@ -37,9 +39,23 @@ import {
readProviderConfigs,
GithubEntityProviderConfig,
} from './GithubEntityProviderConfig';
import { getOrganizationRepositories, RepositoryResponse } from '../lib/github';
import { getOrganizationRepositories } from '../lib/github';
import { satisfiesTopicFilter } from '../lib/util';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import { PushEvent, Commit } from '@octokit/webhooks-types';
const TOPIC_REPO_PUSH = 'github.push';
type Repository = {
name: string;
url: string;
isArchived: boolean;
repositoryTopics: string[];
defaultBranchRef?: string;
isCatalogInfoFilePresent: boolean;
};
/**
* Discovers catalog files located in [GitHub](https://github.com).
* The provider will search your GitHub account and register catalog files matching the configured path
@@ -48,7 +64,7 @@ import { satisfiesTopicFilter } from '../lib/util';
*
* @public
*/
export class GithubEntityProvider implements EntityProvider {
export class GithubEntityProvider implements EntityProvider, EventSubscriber {
private readonly config: GithubEntityProviderConfig;
private readonly logger: Logger;
private readonly integration: GithubIntegrationConfig;
@@ -175,7 +191,7 @@ export class GithubEntityProvider implements EntityProvider {
}
// go to the server and get all of the repositories
private async findCatalogFiles(): Promise<RepositoryResponse[]> {
private async findCatalogFiles(): Promise<Repository[]> {
const organization = this.config.organization;
const host = this.integration.host;
const catalogPath = this.config.catalogPath;
@@ -190,45 +206,49 @@ export class GithubEntityProvider implements EntityProvider {
headers,
});
const { repositories } = await getOrganizationRepositories(
client,
organization,
catalogPath,
);
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
const repositories = repositoriesFromGithub.map(r => {
return {
url: r.url,
name: r.name,
defaultBranchRef: r.defaultBranchRef?.name,
repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name),
isArchived: r.isArchived,
isCatalogInfoFilePresent:
r.catalogInfoFile?.__typename === 'Blob' &&
r.catalogInfoFile.text !== '',
};
});
if (this.config.validateLocationsExist) {
return repositories.filter(repository => {
return (
repository.catalogInfoFile?.__typename === 'Blob' &&
repository.catalogInfoFile.text !== ''
);
});
return repositories.filter(
repository => repository.isCatalogInfoFilePresent,
);
}
return repositories;
}
private matchesFilters(repositories: RepositoryResponse[]) {
private matchesFilters(repositories: Repository[]) {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const matchingRepositories = repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics.nodes.map(
node => node.topic.name,
);
const repoTopics: string[] = r.repositoryTopics;
return (
!r.isArchived &&
(!repositoryFilter || repositoryFilter.test(r.name)) &&
satisfiesTopicFilter(repoTopics, topicFilters) &&
r.defaultBranchRef?.name
r.defaultBranchRef
);
});
return matchingRepositories;
}
private createLocationUrl(repository: RepositoryResponse): string {
private createLocationUrl(repository: Repository): string {
const branch =
this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
this.config.filters?.branch || repository.defaultBranchRef || '-';
const catalogFile = this.config.catalogPath.startsWith('/')
? this.config.catalogPath.substring(1)
: this.config.catalogPath;
@@ -242,6 +262,151 @@ export class GithubEntityProvider implements EntityProvider {
presence: 'optional',
};
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
async onEvent(params: EventParams): Promise<void> {
this.logger.debug(`Received event from ${params.topic}`);
if (params.topic !== TOPIC_REPO_PUSH) {
return;
}
await this.onRepoPush(params.eventPayload as PushEvent);
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
supportsEventTopics(): string[] {
return [TOPIC_REPO_PUSH];
}
private async onRepoPush(event: PushEvent) {
if (!this.connection) {
throw new Error('Not initialized');
}
const repoName = event.repository.name;
const repoUrl = event.repository.url;
this.logger.debug(`handle github:push event for ${repoName} - ${repoUrl}`);
const branch =
this.config.filters?.branch || event.repository.default_branch;
if (!event.ref.includes(branch)) {
this.logger.debug(`skipping push event from ref ${event.ref}`);
return;
}
const repository: Repository = {
url: event.repository.url,
name: event.repository.name,
defaultBranchRef: event.repository.default_branch,
repositoryTopics: event.repository.topics,
isArchived: event.repository.archived,
// we can consider this file present because
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
};
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping push event from repository ${repoName} because didn't match provider filters`,
);
return;
}
// the commit has information about the files (added,removed,modified)
// so we will process the change based in this data
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#push
const added = this.collectDeferredEntitiesFromCommit(
event.repository.url,
branch,
event.commits,
(commit: Commit) => [...commit.added],
);
const removed = this.collectDeferredEntitiesFromCommit(
event.repository.url,
branch,
event.commits,
(commit: Commit) => [...commit.removed],
);
const modified = this.collectFilesFromCommit(
event.commits,
(commit: Commit) => [...commit.modified],
);
if (modified.length > 0) {
await this.connection.refresh({
keys: [
...modified.map(
filePath =>
`url:${event.repository.url}/tree/${branch}/${filePath}`,
),
...modified.map(
filePath =>
`url:${event.repository.url}/blob/${branch}/${filePath}`,
),
],
});
}
if (added.length > 0 || removed.length > 0) {
await this.connection.applyMutation({
type: 'delta',
added: added,
removed: removed,
});
}
this.logger.info(
`Processed Github push event: added ${added.length} - removed ${removed.length} - modified ${modified.length}`,
);
}
private collectDeferredEntitiesFromCommit(
repositoryUrl: string,
branch: string,
commits: Commit[],
transformOperation: (commit: Commit) => string[],
): DeferredEntity[] {
const catalogFiles = this.collectFilesFromCommit(
commits,
transformOperation,
);
return this.toDeferredEntities(
catalogFiles.map(
filePath => `${repositoryUrl}/blob/${branch}/${filePath}`,
),
);
}
private collectFilesFromCommit(
commits: Commit[],
transformOperation: (commit: Commit) => string[],
): string[] {
const catalogFile = this.config.catalogPath.startsWith('/')
? this.config.catalogPath.substring(1)
: this.config.catalogPath;
return commits
.map(transformOperation)
.flat()
.filter(file => catalogFile.includes(file));
}
private toDeferredEntities(targets: string[]): DeferredEntity[] {
return targets
.map(target => {
const location = GithubEntityProvider.toLocationSpec(target);
return locationSpecToLocationEntity({ location });
})
.map(entity => {
return {
locationKey: this.getProviderName(),
entity: entity,
};
});
}
}
/*
+2
View File
@@ -4577,7 +4577,9 @@ __metadata:
"@backstage/errors": "workspace:^"
"@backstage/integration": "workspace:^"
"@backstage/plugin-catalog-backend": "workspace:^"
"@backstage/plugin-catalog-common": "workspace:^"
"@backstage/plugin-catalog-node": "workspace:^"
"@backstage/plugin-events-node": "workspace:^"
"@backstage/types": "workspace:^"
"@octokit/graphql": ^5.0.0
"@octokit/rest": ^19.0.3