feat(catalog,github): support repository events
The provider adds a subscription to the topic `github.repository`. Hereby, it supports events of type `repository` with actions - `archived` - `deleted` - `edited` - `renamed` - `transferred` - `unarchived` Actions skipped as they don't require entity changes: - `created` - `privatized` - `publicized` If the config option `validateLocationsExist` is enabled, an API request is necessary and will be executed. This affects the actions `renamed`, `transferred`, and `unarchive` of event type `repository`. Catalog entities related to the `GithubEntityProvider` instance will be adjusted according to action and its meaning for them. Closes: #21906 Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github': patch
|
||||
---
|
||||
|
||||
Adds support for `repository` events.
|
||||
|
||||
The provider adds a subscription to the topic `github.repository`.
|
||||
|
||||
Hereby, it supports events of type `repository` with actions
|
||||
|
||||
- `archived`
|
||||
- `deleted`
|
||||
- `edited`
|
||||
- `renamed`
|
||||
- `transferred`
|
||||
- `unarchived`
|
||||
|
||||
Actions skipped as they don't require entity changes:
|
||||
|
||||
- `created`
|
||||
- `privatized`
|
||||
- `publicized`
|
||||
|
||||
If the config option `validateLocationsExist` is enabled, an API request
|
||||
is necessary and will be executed.
|
||||
This affects the actions `renamed`, `transferred`, and `unarchive`
|
||||
of event type `repository`.
|
||||
|
||||
Catalog entities related to the `GithubEntityProvider` instance will be adjusted
|
||||
according to action and its meaning for them.
|
||||
@@ -40,8 +40,9 @@ backend.add(import('@backstage/plugin-catalog-backend-module-github/alpha'));
|
||||
## Events Support
|
||||
|
||||
The catalog module for GitHub comes with events support enabled.
|
||||
This will make it subscribe to its relevant topics (`github.push`)
|
||||
and expects these events to be published via the `EventsService`.
|
||||
This will make it subscribe to its relevant topics (`github.push`,
|
||||
`github.repository`) and expects these events to be published
|
||||
via the `EventsService`.
|
||||
|
||||
Additionally, you should install the
|
||||
[event router by `events-backend-module-github`](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md)
|
||||
@@ -55,7 +56,15 @@ You can decide between the following options (extensible):
|
||||
- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
|
||||
- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
|
||||
|
||||
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `push` events.
|
||||
You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks).
|
||||
|
||||
The webhook(s) will need to be configured to react to `push` and
|
||||
`repository` events.
|
||||
|
||||
Certain actions like `transferred` by the `repository` event type
|
||||
will not be supported when you use repository webhooks.
|
||||
Please check the GitHubs documentation for these event types and
|
||||
its actions.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ export type QueryResponse = {
|
||||
|
||||
type RepositoryOwnerResponse = {
|
||||
repositories?: Connection<RepositoryResponse>;
|
||||
repository?: RepositoryResponse;
|
||||
};
|
||||
|
||||
export type OrganizationResponse = {
|
||||
@@ -509,6 +510,61 @@ export async function getOrganizationRepositories(
|
||||
return { repositories };
|
||||
}
|
||||
|
||||
export async function getOrganizationRepository(
|
||||
client: typeof graphql,
|
||||
org: string,
|
||||
repoName: string,
|
||||
catalogPath: string,
|
||||
): Promise<RepositoryResponse | null> {
|
||||
let relativeCatalogPathRef: string;
|
||||
// We must strip the leading slash or the query for objects does not work
|
||||
if (catalogPath.startsWith('/')) {
|
||||
relativeCatalogPathRef = catalogPath.substring(1);
|
||||
} else {
|
||||
relativeCatalogPathRef = catalogPath;
|
||||
}
|
||||
const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
|
||||
const query = `
|
||||
query repository($org: String!, $repoName: String!, $catalogPathRef: String!) {
|
||||
repositoryOwner(login: $org) {
|
||||
repository(name: $repoName) {
|
||||
name
|
||||
catalogInfoFile: object(expression: $catalogPathRef) {
|
||||
__typename
|
||||
... on Blob {
|
||||
id
|
||||
text
|
||||
}
|
||||
}
|
||||
url
|
||||
isArchived
|
||||
isFork
|
||||
visibility
|
||||
repositoryTopics(first: 100) {
|
||||
nodes {
|
||||
... on RepositoryTopic {
|
||||
topic {
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
defaultBranchRef {
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
const response: QueryResponse = await client(query, {
|
||||
org,
|
||||
repoName,
|
||||
catalogPathRef,
|
||||
});
|
||||
|
||||
return response.repositoryOwner?.repository || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets all the users out of a Github organization.
|
||||
*
|
||||
|
||||
@@ -29,7 +29,12 @@ import { GithubEntityProvider } from './GithubEntityProvider';
|
||||
import * as helpers from '../lib/github';
|
||||
import { EventParams } from '@backstage/plugin-events-node';
|
||||
import { mockServices } from '@backstage/backend-test-utils';
|
||||
import { Commit, PushEvent } from '@octokit/webhooks-types';
|
||||
import {
|
||||
Commit,
|
||||
PushEvent,
|
||||
RepositoryEvent,
|
||||
RepositoryRenamedEvent,
|
||||
} from '@octokit/webhooks-types';
|
||||
|
||||
jest.mock('../lib/github', () => {
|
||||
return {
|
||||
@@ -657,7 +662,7 @@ describe('GithubEntityProvider', () => {
|
||||
|
||||
describe('on event', () => {
|
||||
const createExpectedEntitiesForEvent = (
|
||||
event: EventParams<PushEvent>,
|
||||
event: EventParams<PushEvent | RepositoryEvent>,
|
||||
options?: { branch?: string; catalogFilePath?: string },
|
||||
): DeferredEntity[] => {
|
||||
const url = `${event.eventPayload.repository.url}/blob/${
|
||||
@@ -948,5 +953,543 @@ describe('GithubEntityProvider', () => {
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository event', () => {
|
||||
const createRepoEvent = (
|
||||
action: RepositoryEvent['action'],
|
||||
): EventParams<RepositoryEvent> => {
|
||||
const repo = {
|
||||
name: 'test-repo',
|
||||
url: 'https://github.com/test-org/test-repo',
|
||||
default_branch: 'main',
|
||||
master_branch: 'main',
|
||||
organization: 'test-org',
|
||||
topics: [],
|
||||
archived: action === 'archived',
|
||||
private: action !== 'publicized',
|
||||
} as Partial<RepositoryEvent['repository']>;
|
||||
|
||||
const event = {
|
||||
action,
|
||||
repository: repo as RepositoryEvent['repository'],
|
||||
} as RepositoryEvent;
|
||||
|
||||
if (action === 'renamed') {
|
||||
(event as RepositoryRenamedEvent).changes = {
|
||||
repository: {
|
||||
name: {
|
||||
from: `old-${event.repository.name}`,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
topic: 'github.repository',
|
||||
metadata: {
|
||||
'x-github-event': 'repository',
|
||||
},
|
||||
eventPayload: event,
|
||||
};
|
||||
};
|
||||
|
||||
describe('on repository archived event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('archived');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update removing entities', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('archived');
|
||||
const expectedEntities = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: expectedEntities,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository created event', () => {
|
||||
it('skip', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('created');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository deleted event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('deleted');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update removing entities', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('deleted');
|
||||
const expectedEntities = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: expectedEntities,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository edited event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('edited');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update removing entities with non-matching filters', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
filters: {
|
||||
topic: {
|
||||
include: ['backstage-include'],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('edited');
|
||||
const expectedEntities = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: expectedEntities,
|
||||
});
|
||||
});
|
||||
|
||||
it('apply no delta update with matching filters', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
filters: {
|
||||
topic: {
|
||||
include: ['backstage-include'],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('edited');
|
||||
event.eventPayload.repository.topics = ['backstage-include'];
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository privatized event', () => {
|
||||
it('skip', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('privatized');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository publicized event', () => {
|
||||
it('skip', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('publicized');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository renamed event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('renamed');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update removing entities with non-matching filters', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
filters: {
|
||||
repository: 'other-.*',
|
||||
},
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent(
|
||||
'renamed',
|
||||
) as EventParams<RepositoryRenamedEvent>;
|
||||
const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`;
|
||||
const expectedEntitiesRemoved =
|
||||
createExpectedEntitiesForUrl(urlOldRepo);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: expectedEntitiesRemoved,
|
||||
});
|
||||
});
|
||||
|
||||
it('apply delta update removing and adding entities with matching filters', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent(
|
||||
'renamed',
|
||||
) as EventParams<RepositoryRenamedEvent>;
|
||||
const urlOldRepo = `https://github.com/${event.eventPayload.repository.organization}/${event.eventPayload.changes.repository.name.from}/blob/main/catalog-info.yaml`;
|
||||
const expectedEntitiesRemoved =
|
||||
createExpectedEntitiesForUrl(urlOldRepo);
|
||||
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
2,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: expectedEntitiesRemoved,
|
||||
});
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: expectedEntitiesAdded,
|
||||
removed: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository transferred event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('transferred');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply no delta update with non-matching filters', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
filters: {
|
||||
topic: {
|
||||
include: ['backstage-include'],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('transferred');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update adding entities with matching filters', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('transferred');
|
||||
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: expectedEntitiesAdded,
|
||||
removed: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('on repository unarchived event', () => {
|
||||
it('skip on non-matching org', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
organization: 'other-org',
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('unarchived');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply no delta update with non-matching filters', async () => {
|
||||
const config = createSingleProviderConfig({
|
||||
providerConfig: {
|
||||
filters: {
|
||||
topic: {
|
||||
include: ['backstage-include'],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('unarchived');
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('apply delta update adding entities with matching filters', async () => {
|
||||
const config = createSingleProviderConfig();
|
||||
const provider = createProviders(config)[0];
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const event = createRepoEvent('unarchived');
|
||||
const expectedEntitiesAdded = createExpectedEntitiesForEvent(event);
|
||||
|
||||
await provider.onEvent(event);
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(
|
||||
1,
|
||||
);
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'delta',
|
||||
added: expectedEntitiesAdded,
|
||||
removed: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -38,7 +38,11 @@ import {
|
||||
GithubEntityProviderConfig,
|
||||
readProviderConfigs,
|
||||
} from './GithubEntityProviderConfig';
|
||||
import { getOrganizationRepositories } from '../lib/github';
|
||||
import {
|
||||
getOrganizationRepositories,
|
||||
getOrganizationRepository,
|
||||
RepositoryResponse,
|
||||
} from '../lib/github';
|
||||
import {
|
||||
satisfiesForkFilter,
|
||||
satisfiesTopicFilter,
|
||||
@@ -50,11 +54,21 @@ import {
|
||||
EventsService,
|
||||
EventSubscriber,
|
||||
} from '@backstage/plugin-events-node';
|
||||
import { Commit, PushEvent } from '@octokit/webhooks-types';
|
||||
import {
|
||||
Commit,
|
||||
PushEvent,
|
||||
RepositoryArchivedEvent,
|
||||
RepositoryDeletedEvent,
|
||||
RepositoryEditedEvent,
|
||||
RepositoryEvent,
|
||||
RepositoryRenamedEvent,
|
||||
RepositoryTransferredEvent,
|
||||
RepositoryUnarchivedEvent,
|
||||
} from '@octokit/webhooks-types';
|
||||
import { Minimatch } from 'minimatch';
|
||||
import { LoggerService } from '@backstage/backend-plugin-api';
|
||||
|
||||
const TOPIC_REPO_PUSH = 'github.push';
|
||||
const EVENT_TOPICS = ['github.push', 'github.repository'];
|
||||
|
||||
type Repository = {
|
||||
name: string;
|
||||
@@ -157,7 +171,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
this.connection = connection;
|
||||
await this.events?.subscribe({
|
||||
id: this.getProviderName(),
|
||||
topics: [TOPIC_REPO_PUSH],
|
||||
topics: EVENT_TOPICS,
|
||||
onEvent: params => this.onEvent(params),
|
||||
});
|
||||
return await this.scheduleFn();
|
||||
@@ -194,15 +208,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
|
||||
const targets = await this.findCatalogFiles();
|
||||
const matchingTargets = this.matchesFilters(targets);
|
||||
const entities = matchingTargets
|
||||
.map(repository => this.createLocationUrl(repository))
|
||||
.map(GithubEntityProvider.toLocationSpec)
|
||||
.map(location => {
|
||||
return {
|
||||
locationKey: this.getProviderName(),
|
||||
entity: locationSpecToLocationEntity({ location }),
|
||||
};
|
||||
});
|
||||
const entities = this.toDeferredEntitiesFromRepos(matchingTargets);
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
@@ -214,38 +220,32 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
);
|
||||
}
|
||||
|
||||
// go to the server and get all of the repositories
|
||||
private async findCatalogFiles(): Promise<Repository[]> {
|
||||
private async createGraphqlClient() {
|
||||
const organization = this.config.organization;
|
||||
const host = this.integration.host;
|
||||
const catalogPath = this.config.catalogPath;
|
||||
const orgUrl = `https://${host}/${organization}`;
|
||||
|
||||
const { headers } = await this.githubCredentialsProvider.getCredentials({
|
||||
url: orgUrl,
|
||||
});
|
||||
|
||||
const client = graphql.defaults({
|
||||
return graphql.defaults({
|
||||
baseUrl: this.integration.apiBaseUrl,
|
||||
headers,
|
||||
});
|
||||
}
|
||||
|
||||
// go to the server and get all repositories
|
||||
private async findCatalogFiles(): Promise<Repository[]> {
|
||||
const organization = this.config.organization;
|
||||
const catalogPath = this.config.catalogPath;
|
||||
const client = await this.createGraphqlClient();
|
||||
|
||||
const { repositories: repositoriesFromGithub } =
|
||||
await getOrganizationRepositories(client, organization, catalogPath);
|
||||
const repositories = repositoriesFromGithub.map(r => {
|
||||
return {
|
||||
url: r.url,
|
||||
name: r.name,
|
||||
defaultBranchRef: r.defaultBranchRef?.name,
|
||||
repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name),
|
||||
isArchived: r.isArchived,
|
||||
isFork: r.isFork,
|
||||
isCatalogInfoFilePresent:
|
||||
r.catalogInfoFile?.__typename === 'Blob' &&
|
||||
r.catalogInfoFile.text !== '',
|
||||
visibility: r.visibility,
|
||||
};
|
||||
});
|
||||
const repositories = repositoriesFromGithub.map(
|
||||
this.createRepoFromGithubResponse,
|
||||
);
|
||||
|
||||
if (this.config.validateLocationsExist) {
|
||||
return repositories.filter(
|
||||
@@ -256,13 +256,13 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
return repositories;
|
||||
}
|
||||
|
||||
private matchesFilters(repositories: Repository[]) {
|
||||
private matchesFilters(repositories: Repository[]): Repository[] {
|
||||
const repositoryFilter = this.config.filters?.repository;
|
||||
const topicFilters = this.config.filters?.topic;
|
||||
const allowForks = this.config.filters?.allowForks ?? true;
|
||||
const visibilities = this.config.filters?.visibility ?? [];
|
||||
|
||||
const matchingRepositories = repositories.filter(r => {
|
||||
return repositories.filter(r => {
|
||||
const repoTopics: string[] = r.repositoryTopics;
|
||||
return (
|
||||
!r.isArchived &&
|
||||
@@ -273,7 +273,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
r.defaultBranchRef
|
||||
);
|
||||
});
|
||||
return matchingRepositories;
|
||||
}
|
||||
|
||||
private createLocationUrl(repository: Repository): string {
|
||||
@@ -296,23 +295,34 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.onEvent} */
|
||||
async onEvent(params: EventParams): Promise<void> {
|
||||
this.logger.debug(`Received event from ${params.topic}`);
|
||||
if (params.topic !== TOPIC_REPO_PUSH) {
|
||||
return;
|
||||
}
|
||||
if (EVENT_TOPICS.some(topic => topic === params.topic)) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
await this.onRepoPush(params.eventPayload as PushEvent);
|
||||
switch (params.topic) {
|
||||
case 'github.push':
|
||||
await this.onPush(params.eventPayload as PushEvent);
|
||||
return;
|
||||
|
||||
case 'github.repository':
|
||||
await this.onRepoChange(params.eventPayload as RepositoryEvent);
|
||||
return;
|
||||
|
||||
default: // should never be reached
|
||||
this.logger.warn(
|
||||
`Missing implementation for event of topic ${params.topic}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
|
||||
supportsEventTopics(): string[] {
|
||||
return [TOPIC_REPO_PUSH];
|
||||
return EVENT_TOPICS;
|
||||
}
|
||||
|
||||
private async onRepoPush(event: PushEvent) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
private async onPush(event: PushEvent) {
|
||||
if (this.config.organization !== event.repository.organization) {
|
||||
this.logger.debug(
|
||||
`skipping push event from organization ${event.repository.organization}`,
|
||||
@@ -332,18 +342,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
return;
|
||||
}
|
||||
|
||||
const repository: Repository = {
|
||||
url: event.repository.url,
|
||||
name: event.repository.name,
|
||||
defaultBranchRef: event.repository.default_branch,
|
||||
repositoryTopics: event.repository.topics,
|
||||
isArchived: event.repository.archived,
|
||||
isFork: event.repository.fork,
|
||||
// we can consider this file present because
|
||||
// only the catalog file will be recovered from the commits
|
||||
isCatalogInfoFilePresent: true,
|
||||
visibility: event.repository.visibility,
|
||||
};
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
|
||||
const matchingTargets = this.matchesFilters([repository]);
|
||||
if (matchingTargets.length === 0) {
|
||||
@@ -378,7 +377,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
? this.config.catalogPath.substring(1)
|
||||
: this.config.catalogPath;
|
||||
|
||||
await this.connection.refresh({
|
||||
await this.connection!.refresh({
|
||||
keys: [
|
||||
...new Set([
|
||||
...modified.map(
|
||||
@@ -396,7 +395,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
}
|
||||
|
||||
if (added.length > 0 || removed.length > 0) {
|
||||
await this.connection.applyMutation({
|
||||
await this.connection!.applyMutation({
|
||||
type: 'delta',
|
||||
added: added,
|
||||
removed: removed,
|
||||
@@ -408,6 +407,263 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
);
|
||||
}
|
||||
|
||||
private async onRepoChange(event: RepositoryEvent) {
|
||||
if (this.config.organization !== event.repository.organization) {
|
||||
this.logger.debug(
|
||||
`skipping repository event from organization ${event.repository.organization}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const action = event.action;
|
||||
switch (action) {
|
||||
case 'archived':
|
||||
await this.onRepoArchived(event as RepositoryArchivedEvent);
|
||||
return;
|
||||
|
||||
// A repository was created.
|
||||
case 'created':
|
||||
// skip these events
|
||||
return;
|
||||
|
||||
case 'deleted':
|
||||
await this.onRepoDeleted(event as RepositoryDeletedEvent);
|
||||
return;
|
||||
|
||||
case 'edited':
|
||||
await this.onRepoEdited(event as RepositoryEditedEvent);
|
||||
return;
|
||||
|
||||
// The visibility of a repository was changed to `private`.
|
||||
case 'privatized':
|
||||
// skip these events
|
||||
return;
|
||||
|
||||
// The visibility of a repository was changed to `public`.
|
||||
case 'publicized':
|
||||
// skip these events
|
||||
return;
|
||||
|
||||
case 'renamed':
|
||||
await this.onRepoRenamed(event as RepositoryRenamedEvent);
|
||||
return;
|
||||
|
||||
case 'transferred':
|
||||
await this.onRepoTransferred(event as RepositoryTransferredEvent);
|
||||
return;
|
||||
|
||||
case 'unarchived':
|
||||
await this.onRepoUnarchived(event as RepositoryUnarchivedEvent);
|
||||
return;
|
||||
|
||||
default: // should never be reached
|
||||
this.logger.warn(
|
||||
`Missing implementation for event of topic repository with action ${action}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A repository was archived.
|
||||
*
|
||||
* Removes all entities associated with the repository.
|
||||
*
|
||||
* @param event - The repository archived event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoArchived(event: RepositoryArchivedEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
await this.removeEntitiesForRepo(repository);
|
||||
this.logger.debug(
|
||||
`Removed entities for archived repository ${repository.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A repository was deleted.
|
||||
*
|
||||
* Removes all entities associated with the repository.
|
||||
*
|
||||
* @param event - The repository deleted event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoDeleted(event: RepositoryDeletedEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
await this.removeEntitiesForRepo(repository);
|
||||
this.logger.debug(
|
||||
`Removed entities for deleted repository ${repository.name}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The topics, default branch, description, or homepage of a repository was changed.
|
||||
*
|
||||
* We are interested in potential topic changes as these can be used as part of the filters.
|
||||
*
|
||||
* Removes all entities associated with the repository if the repository no longer matches the filters.
|
||||
*
|
||||
* @param event - The repository edited event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoEdited(event: RepositoryEditedEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
|
||||
const matchingTargets = this.matchesFilters([repository]);
|
||||
if (matchingTargets.length === 0) {
|
||||
await this.removeEntitiesForRepo(repository);
|
||||
}
|
||||
// else: repository is (still) matching the filters, so we don't need to do anything
|
||||
}
|
||||
|
||||
/**
|
||||
* The name of a repository was changed.
|
||||
*
|
||||
* Removes all entities associated with the repository's old name.
|
||||
* Creates new entities for the repository's new name if it still matches the filters.
|
||||
*
|
||||
* @param event - The repository renamed event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoRenamed(event: RepositoryRenamedEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
const oldRepoName = event.changes.repository.name.from;
|
||||
const urlParts = event.repository.url.split('/');
|
||||
urlParts[urlParts.length - 1] = oldRepoName;
|
||||
const oldRepoUrl = urlParts.join('/');
|
||||
const oldRepository: Repository = {
|
||||
...repository,
|
||||
name: oldRepoName,
|
||||
url: oldRepoUrl,
|
||||
};
|
||||
await this.removeEntitiesForRepo(oldRepository);
|
||||
|
||||
const matchingTargets = this.matchesFilters([repository]);
|
||||
if (matchingTargets.length === 0) {
|
||||
this.logger.debug(
|
||||
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.addEntitiesForRepo(repository);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ownership of the repository was transferred to a user or organization account.
|
||||
* This event is only sent to the account where the ownership is transferred.
|
||||
* To receive the `repository.transferred` event, the new owner account must have the GitHub App installed,
|
||||
* and the App must be subscribed to "Repository" events.
|
||||
*
|
||||
* Creates new entities for the repository if it matches the filters.
|
||||
*
|
||||
* @param event - The repository unarchived event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoTransferred(event: RepositoryTransferredEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
|
||||
const matchingTargets = this.matchesFilters([repository]);
|
||||
if (matchingTargets.length === 0) {
|
||||
this.logger.debug(
|
||||
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.addEntitiesForRepo(repository);
|
||||
}
|
||||
|
||||
/**
|
||||
* A previously archived repository was unarchived.
|
||||
*
|
||||
* Creates new entities for the repository if it matches the filters.
|
||||
*
|
||||
* @param event - The repository unarchived event.
|
||||
* @private
|
||||
*/
|
||||
private async onRepoUnarchived(event: RepositoryUnarchivedEvent) {
|
||||
const repository = this.createRepoFromEvent(event);
|
||||
|
||||
const matchingTargets = this.matchesFilters([repository]);
|
||||
if (matchingTargets.length === 0) {
|
||||
this.logger.debug(
|
||||
`skipping repository transferred event for repository ${repository.name} because it didn't match provider filters`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await this.addEntitiesForRepo(repository);
|
||||
}
|
||||
|
||||
private async removeEntitiesForRepo(repository: Repository) {
|
||||
const removed = this.toDeferredEntitiesFromRepos([repository]);
|
||||
await this.connection!.applyMutation({
|
||||
type: 'delta',
|
||||
added: [],
|
||||
removed: removed,
|
||||
});
|
||||
}
|
||||
|
||||
private async addEntitiesForRepo(repository: Repository) {
|
||||
if (this.config.validateLocationsExist) {
|
||||
const organization = this.config.organization;
|
||||
const catalogPath = this.config.catalogPath;
|
||||
const client = await this.createGraphqlClient();
|
||||
|
||||
const repositoryFromGithub = await getOrganizationRepository(
|
||||
client,
|
||||
organization,
|
||||
repository.name,
|
||||
catalogPath,
|
||||
).then(r => (r ? this.createRepoFromGithubResponse(r) : null));
|
||||
|
||||
if (!repositoryFromGithub?.isCatalogInfoFilePresent) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const added = this.toDeferredEntitiesFromRepos([repository]);
|
||||
await this.connection!.applyMutation({
|
||||
type: 'delta',
|
||||
added: added,
|
||||
removed: [],
|
||||
});
|
||||
}
|
||||
|
||||
private createRepoFromEvent(event: RepositoryEvent | PushEvent): Repository {
|
||||
return {
|
||||
url: event.repository.url,
|
||||
name: event.repository.name,
|
||||
defaultBranchRef: event.repository.default_branch,
|
||||
repositoryTopics: event.repository.topics,
|
||||
isArchived: event.repository.archived,
|
||||
isFork: event.repository.fork,
|
||||
// we can consider this file present because
|
||||
// only the catalog file will be recovered from the commits
|
||||
isCatalogInfoFilePresent: true,
|
||||
visibility: event.repository.visibility,
|
||||
};
|
||||
}
|
||||
|
||||
private createRepoFromGithubResponse(
|
||||
repositoryResponse: RepositoryResponse,
|
||||
): Repository {
|
||||
return {
|
||||
url: repositoryResponse.url,
|
||||
name: repositoryResponse.name,
|
||||
defaultBranchRef: repositoryResponse.defaultBranchRef?.name,
|
||||
repositoryTopics: repositoryResponse.repositoryTopics.nodes.map(
|
||||
t => t.topic.name,
|
||||
),
|
||||
isArchived: repositoryResponse.isArchived,
|
||||
isFork: repositoryResponse.isFork,
|
||||
isCatalogInfoFilePresent:
|
||||
repositoryResponse.catalogInfoFile?.__typename === 'Blob' &&
|
||||
repositoryResponse.catalogInfoFile.text !== '',
|
||||
visibility: repositoryResponse.visibility,
|
||||
};
|
||||
}
|
||||
|
||||
private collectDeferredEntitiesFromCommit(
|
||||
repositoryUrl: string,
|
||||
branch: string,
|
||||
@@ -454,6 +710,20 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private toDeferredEntitiesFromRepos(
|
||||
repositories: Repository[],
|
||||
): DeferredEntity[] {
|
||||
return repositories
|
||||
.map(repository => this.createLocationUrl(repository))
|
||||
.map(GithubEntityProvider.toLocationSpec)
|
||||
.map(location => {
|
||||
return {
|
||||
locationKey: this.getProviderName(),
|
||||
entity: locationSpecToLocationEntity({ location }),
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user