Merge pull request #17285 from kuangp/feat/githubMultiOrgEntityProvider
feat(GithubMultiOrgEntityProvider): implement provider
This commit is contained in:
Fredrik Adelöw
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github': patch
|
||||
---
|
||||
|
||||
Implement `GithubMultiOrgEntityProvider` for sourcing group and user entities from multiple GitHub organizations
|
||||
@@ -132,6 +132,42 @@ export type GithubMultiOrgConfig = Array<{
|
||||
userNamespace: string | undefined;
|
||||
}>;
|
||||
|
||||
// @public
|
||||
export class GithubMultiOrgEntityProvider implements EntityProvider {
|
||||
constructor(options: {
|
||||
id: string;
|
||||
gitHubConfig: GithubIntegrationConfig;
|
||||
githubCredentialsProvider: GithubCredentialsProvider;
|
||||
githubUrl: string;
|
||||
logger: Logger;
|
||||
orgs?: string[];
|
||||
userTransformer?: UserTransformer;
|
||||
teamTransformer?: TeamTransformer;
|
||||
});
|
||||
// (undocumented)
|
||||
connect(connection: EntityProviderConnection): Promise<void>;
|
||||
// (undocumented)
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: GithubMultiOrgEntityProviderOptions,
|
||||
): GithubMultiOrgEntityProvider;
|
||||
// (undocumented)
|
||||
getProviderName(): string;
|
||||
read(options?: { logger?: Logger }): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface GithubMultiOrgEntityProviderOptions {
|
||||
githubCredentialsProvider?: GithubCredentialsProvider;
|
||||
githubUrl: string;
|
||||
id: string;
|
||||
logger: Logger;
|
||||
orgs?: string[];
|
||||
schedule?: 'manual' | TaskRunner;
|
||||
teamTransformer?: TeamTransformer;
|
||||
userTransformer?: UserTransformer;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
|
||||
constructor(options: {
|
||||
|
||||
@@ -26,6 +26,8 @@ export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor'
|
||||
export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
|
||||
export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
|
||||
export { GithubEntityProvider } from './providers/GithubEntityProvider';
|
||||
export { GithubMultiOrgEntityProvider } from './providers/GithubMultiOrgEntityProvider';
|
||||
export type { GithubMultiOrgEntityProviderOptions } from './providers/GithubMultiOrgEntityProvider';
|
||||
export { GithubOrgEntityProvider } from './providers/GithubOrgEntityProvider';
|
||||
export type { GithubOrgEntityProviderOptions } from './providers/GithubOrgEntityProvider';
|
||||
export {
|
||||
|
||||
+716
@@ -0,0 +1,716 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
|
||||
import {
|
||||
GithubCredentialsProvider,
|
||||
GithubIntegrationConfig,
|
||||
} from '@backstage/integration';
|
||||
import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
|
||||
import { graphql } from '@octokit/graphql';
|
||||
import {
|
||||
GithubMultiOrgEntityProvider,
|
||||
withLocations,
|
||||
} from './GithubMultiOrgEntityProvider';
|
||||
|
||||
jest.mock('@octokit/graphql');
|
||||
|
||||
const getAllInstallationsMock = jest.fn();
|
||||
jest.mock('@backstage/integration', () => ({
|
||||
...jest.requireActual('@backstage/integration'),
|
||||
GithubAppCredentialsMux: function mockGithubAppCredentialsMux() {
|
||||
return {
|
||||
getAllInstallations: getAllInstallationsMock,
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
describe('GithubMultiOrgEntityProvider', () => {
|
||||
describe('read', () => {
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('should read specified orgs', async () => {
|
||||
const mockClient = jest.fn();
|
||||
|
||||
mockClient
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
membersWithRole: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
login: 'a',
|
||||
name: 'b',
|
||||
bio: 'c',
|
||||
email: 'd',
|
||||
avatarUrl: 'e',
|
||||
},
|
||||
{
|
||||
login: 'x',
|
||||
name: 'y',
|
||||
bio: 'z',
|
||||
email: 'w',
|
||||
avatarUrl: 'v',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
teams: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
slug: 'team',
|
||||
combinedSlug: 'orgA/team',
|
||||
name: 'Team',
|
||||
description: 'The one and only team',
|
||||
avatarUrl: 'http://example.com/team.jpeg',
|
||||
parentTeam: {
|
||||
slug: 'parent',
|
||||
combinedSlug: '',
|
||||
members: { pageInfo: { hasNextPage: false }, nodes: [] },
|
||||
},
|
||||
members: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [{ login: 'a' }, { login: 'x' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
membersWithRole: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
login: 'a',
|
||||
name: 'b',
|
||||
bio: 'c',
|
||||
email: 'd',
|
||||
avatarUrl: 'e',
|
||||
},
|
||||
{
|
||||
login: 'x',
|
||||
name: 'y',
|
||||
bio: 'z',
|
||||
email: 'w',
|
||||
avatarUrl: 'v',
|
||||
},
|
||||
{
|
||||
login: 'q',
|
||||
name: 'r',
|
||||
bio: 's',
|
||||
email: 't',
|
||||
avatarUrl: 'u',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
teams: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
slug: 'team',
|
||||
combinedSlug: 'orgB/team',
|
||||
name: 'Team',
|
||||
description: 'The one and only team',
|
||||
avatarUrl: 'http://example.com/team.jpeg',
|
||||
parentTeam: {
|
||||
slug: 'parent',
|
||||
combinedSlug: '',
|
||||
members: { pageInfo: { hasNextPage: false }, nodes: [] },
|
||||
},
|
||||
members: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [{ login: 'a' }, { login: 'q' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
(graphql.defaults as jest.Mock).mockReturnValue(mockClient);
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
const gitHubConfig: GithubIntegrationConfig = {
|
||||
host: 'https://github.com',
|
||||
};
|
||||
|
||||
const mockGetCredentials = jest.fn().mockReturnValue({
|
||||
headers: { token: 'blah' },
|
||||
type: 'app',
|
||||
});
|
||||
|
||||
const githubCredentialsProvider: GithubCredentialsProvider = {
|
||||
getCredentials: mockGetCredentials,
|
||||
};
|
||||
|
||||
const entityProvider = new GithubMultiOrgEntityProvider({
|
||||
id: 'my-id',
|
||||
gitHubConfig,
|
||||
githubCredentialsProvider,
|
||||
githubUrl: 'https://github.com',
|
||||
logger,
|
||||
orgs: ['orgA', 'orgB'],
|
||||
});
|
||||
|
||||
entityProvider.connect(entityProviderConnection);
|
||||
|
||||
await entityProvider.read();
|
||||
|
||||
expect(mockGetCredentials).toHaveBeenCalledWith({
|
||||
url: 'https://github.com/orgA',
|
||||
});
|
||||
expect(mockGetCredentials).toHaveBeenCalledWith({
|
||||
url: 'https://github.com/orgB',
|
||||
});
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
entities: [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/a',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/a',
|
||||
'github.com/user-login': 'a',
|
||||
},
|
||||
description: 'c',
|
||||
name: 'a',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orga/team', 'orgb/team'],
|
||||
profile: {
|
||||
displayName: 'b',
|
||||
email: 'd',
|
||||
picture: 'e',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/x',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/x',
|
||||
'github.com/user-login': 'x',
|
||||
},
|
||||
description: 'z',
|
||||
name: 'x',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orga/team'],
|
||||
profile: {
|
||||
displayName: 'y',
|
||||
email: 'w',
|
||||
picture: 'v',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/q',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/q',
|
||||
'github.com/user-login': 'q',
|
||||
},
|
||||
description: 's',
|
||||
name: 'q',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orgb/team'],
|
||||
profile: {
|
||||
displayName: 'r',
|
||||
email: 't',
|
||||
picture: 'u',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/orgs/orgA/teams/team',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/orgs/orgA/teams/team',
|
||||
'github.com/team-slug': 'orgA/team',
|
||||
},
|
||||
namespace: 'orga',
|
||||
name: 'team',
|
||||
description: 'The one and only team',
|
||||
},
|
||||
spec: {
|
||||
children: [],
|
||||
parent: 'parent',
|
||||
profile: {
|
||||
displayName: 'Team',
|
||||
picture: 'http://example.com/team.jpeg',
|
||||
},
|
||||
type: 'team',
|
||||
members: ['default/a', 'default/x'],
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/orgs/orgB/teams/team',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/orgs/orgB/teams/team',
|
||||
'github.com/team-slug': 'orgB/team',
|
||||
},
|
||||
namespace: 'orgb',
|
||||
name: 'team',
|
||||
description: 'The one and only team',
|
||||
},
|
||||
spec: {
|
||||
children: [],
|
||||
parent: 'parent',
|
||||
profile: {
|
||||
displayName: 'Team',
|
||||
picture: 'http://example.com/team.jpeg',
|
||||
},
|
||||
type: 'team',
|
||||
members: ['default/a', 'default/q'],
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
],
|
||||
type: 'full',
|
||||
});
|
||||
});
|
||||
|
||||
it('should read every accessible org', async () => {
|
||||
getAllInstallationsMock.mockResolvedValue([
|
||||
{
|
||||
target_type: 'Organization',
|
||||
account: {
|
||||
login: 'orgC',
|
||||
},
|
||||
},
|
||||
{
|
||||
target_type: 'Organization',
|
||||
account: {
|
||||
login: 'orgD',
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const mockClient = jest.fn();
|
||||
|
||||
mockClient
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
membersWithRole: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
login: 'a',
|
||||
name: 'b',
|
||||
bio: 'c',
|
||||
email: 'd',
|
||||
avatarUrl: 'e',
|
||||
},
|
||||
{
|
||||
login: 'x',
|
||||
name: 'y',
|
||||
bio: 'z',
|
||||
email: 'w',
|
||||
avatarUrl: 'v',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
teams: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
slug: 'team',
|
||||
combinedSlug: 'orgC/team',
|
||||
name: 'Team',
|
||||
description: 'The one and only team',
|
||||
avatarUrl: 'http://example.com/team.jpeg',
|
||||
parentTeam: {
|
||||
slug: 'parent',
|
||||
combinedSlug: '',
|
||||
members: { pageInfo: { hasNextPage: false }, nodes: [] },
|
||||
},
|
||||
members: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [{ login: 'a' }, { login: 'x' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
membersWithRole: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
login: 'a',
|
||||
name: 'b',
|
||||
bio: 'c',
|
||||
email: 'd',
|
||||
avatarUrl: 'e',
|
||||
},
|
||||
{
|
||||
login: 'x',
|
||||
name: 'y',
|
||||
bio: 'z',
|
||||
email: 'w',
|
||||
avatarUrl: 'v',
|
||||
},
|
||||
{
|
||||
login: 'q',
|
||||
name: 'r',
|
||||
bio: 's',
|
||||
email: 't',
|
||||
avatarUrl: 'u',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
organization: {
|
||||
teams: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [
|
||||
{
|
||||
slug: 'team',
|
||||
combinedSlug: 'orgD/team',
|
||||
name: 'Team',
|
||||
description: 'The one and only team',
|
||||
avatarUrl: 'http://example.com/team.jpeg',
|
||||
parentTeam: {
|
||||
slug: 'parent',
|
||||
combinedSlug: '',
|
||||
members: { pageInfo: { hasNextPage: false }, nodes: [] },
|
||||
},
|
||||
members: {
|
||||
pageInfo: { hasNextPage: false },
|
||||
nodes: [{ login: 'a' }, { login: 'q' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
(graphql.defaults as jest.Mock).mockReturnValue(mockClient);
|
||||
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
|
||||
const logger = getVoidLogger();
|
||||
const gitHubConfig: GithubIntegrationConfig = {
|
||||
host: 'https://github.com',
|
||||
};
|
||||
|
||||
const mockGetCredentials = jest.fn().mockReturnValue({
|
||||
headers: { token: 'blah' },
|
||||
type: 'app',
|
||||
});
|
||||
|
||||
const githubCredentialsProvider: GithubCredentialsProvider = {
|
||||
getCredentials: mockGetCredentials,
|
||||
};
|
||||
|
||||
const entityProvider = new GithubMultiOrgEntityProvider({
|
||||
id: 'my-id',
|
||||
gitHubConfig,
|
||||
githubCredentialsProvider,
|
||||
githubUrl: 'https://github.com',
|
||||
logger,
|
||||
});
|
||||
|
||||
entityProvider.connect(entityProviderConnection);
|
||||
|
||||
await entityProvider.read();
|
||||
|
||||
expect(mockGetCredentials).toHaveBeenCalledWith({
|
||||
url: 'https://github.com/orgC',
|
||||
});
|
||||
expect(mockGetCredentials).toHaveBeenCalledWith({
|
||||
url: 'https://github.com/orgD',
|
||||
});
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
|
||||
entities: [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/a',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/a',
|
||||
'github.com/user-login': 'a',
|
||||
},
|
||||
description: 'c',
|
||||
name: 'a',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orgc/team', 'orgd/team'],
|
||||
profile: {
|
||||
displayName: 'b',
|
||||
email: 'd',
|
||||
picture: 'e',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/x',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/x',
|
||||
'github.com/user-login': 'x',
|
||||
},
|
||||
description: 'z',
|
||||
name: 'x',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orgc/team'],
|
||||
profile: {
|
||||
displayName: 'y',
|
||||
email: 'w',
|
||||
picture: 'v',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/q',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/q',
|
||||
'github.com/user-login': 'q',
|
||||
},
|
||||
description: 's',
|
||||
name: 'q',
|
||||
},
|
||||
spec: {
|
||||
memberOf: ['orgd/team'],
|
||||
profile: {
|
||||
displayName: 'r',
|
||||
email: 't',
|
||||
picture: 'u',
|
||||
},
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/orgs/orgC/teams/team',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/orgs/orgC/teams/team',
|
||||
'github.com/team-slug': 'orgC/team',
|
||||
},
|
||||
namespace: 'orgc',
|
||||
name: 'team',
|
||||
description: 'The one and only team',
|
||||
},
|
||||
spec: {
|
||||
children: [],
|
||||
parent: 'parent',
|
||||
profile: {
|
||||
displayName: 'Team',
|
||||
picture: 'http://example.com/team.jpeg',
|
||||
},
|
||||
type: 'team',
|
||||
members: ['default/a', 'default/x'],
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://https://github.com/orgs/orgD/teams/team',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://https://github.com/orgs/orgD/teams/team',
|
||||
'github.com/team-slug': 'orgD/team',
|
||||
},
|
||||
namespace: 'orgd',
|
||||
name: 'team',
|
||||
description: 'The one and only team',
|
||||
},
|
||||
spec: {
|
||||
children: [],
|
||||
parent: 'parent',
|
||||
profile: {
|
||||
displayName: 'Team',
|
||||
picture: 'http://example.com/team.jpeg',
|
||||
},
|
||||
type: 'team',
|
||||
members: ['default/a', 'default/q'],
|
||||
},
|
||||
},
|
||||
locationKey: 'github-multi-org-provider:my-id',
|
||||
},
|
||||
],
|
||||
type: 'full',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('withLocations', () => {
|
||||
it('should set location for user', () => {
|
||||
const entity: UserEntity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
name: 'user-name',
|
||||
annotations: {
|
||||
'github.com/user-login': 'githubuser',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
memberOf: [],
|
||||
},
|
||||
};
|
||||
|
||||
expect(withLocations('https://github.com', entity)).toEqual({
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'User',
|
||||
metadata: {
|
||||
name: 'user-name',
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://github.com/githubuser',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://github.com/githubuser',
|
||||
'github.com/user-login': 'githubuser',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
memberOf: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should set location for group', () => {
|
||||
const entity: GroupEntity = {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
name: 'group-name',
|
||||
annotations: {
|
||||
'github.com/team-slug': 'backstage/mygroup',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'team',
|
||||
children: [],
|
||||
},
|
||||
};
|
||||
|
||||
expect(withLocations('https://github.com', entity)).toEqual({
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Group',
|
||||
metadata: {
|
||||
name: 'group-name',
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location':
|
||||
'url:https://github.com/orgs/backstage/teams/mygroup',
|
||||
'backstage.io/managed-by-origin-location':
|
||||
'url:https://github.com/orgs/backstage/teams/mygroup',
|
||||
'github.com/team-slug': 'backstage/mygroup',
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
type: 'team',
|
||||
children: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,394 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TaskRunner } from '@backstage/backend-tasks';
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
DEFAULT_NAMESPACE,
|
||||
Entity,
|
||||
GroupEntity,
|
||||
stringifyEntityRef,
|
||||
} from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
DefaultGithubCredentialsProvider,
|
||||
GithubAppCredentialsMux,
|
||||
GithubCredentialsProvider,
|
||||
GithubIntegrationConfig,
|
||||
ScmIntegrations,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
} from '@backstage/plugin-catalog-node';
|
||||
import { graphql } from '@octokit/graphql';
|
||||
import { merge } from 'lodash';
|
||||
import * as uuid from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
import {
|
||||
assignGroupsToUsers,
|
||||
buildOrgHierarchy,
|
||||
defaultOrganizationTeamTransformer,
|
||||
getOrganizationTeams,
|
||||
getOrganizationUsers,
|
||||
GithubTeam,
|
||||
TeamTransformer,
|
||||
TransformerContext,
|
||||
UserTransformer,
|
||||
} from '../lib';
|
||||
import {
|
||||
ANNOTATION_GITHUB_TEAM_SLUG,
|
||||
ANNOTATION_GITHUB_USER_LOGIN,
|
||||
} from '../lib/annotation';
|
||||
import { splitTeamSlug } from '../lib/util';
|
||||
|
||||
/**
|
||||
* Options for {@link GithubMultiOrgEntityProvider}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface GithubMultiOrgEntityProviderOptions {
|
||||
/**
|
||||
* A unique, stable identifier for this provider.
|
||||
*
|
||||
* @example "production"
|
||||
*/
|
||||
id: string;
|
||||
|
||||
/**
|
||||
* The target that this provider should consume.
|
||||
*
|
||||
* @example "https://mycompany.github.com"
|
||||
*/
|
||||
githubUrl: string;
|
||||
|
||||
/**
|
||||
* The list of the GitHub orgs to consume. By default will consume all accessible
|
||||
* orgs on the given GitHub instance (support for GitHub App integration only).
|
||||
*/
|
||||
orgs?: string[];
|
||||
|
||||
/**
|
||||
* The refresh schedule to use.
|
||||
*
|
||||
* @defaultValue "manual"
|
||||
* @remarks
|
||||
*
|
||||
* If you pass in 'manual', you are responsible for calling the `read` method
|
||||
* manually at some interval.
|
||||
*
|
||||
* But more commonly you will pass in the result of
|
||||
* {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
|
||||
* to enable automatic scheduling of tasks.
|
||||
*/
|
||||
schedule?: 'manual' | TaskRunner;
|
||||
|
||||
/**
|
||||
* The logger to use.
|
||||
*/
|
||||
logger: Logger;
|
||||
|
||||
/**
|
||||
* Optionally supply a custom credentials provider, replacing the default one.
|
||||
*/
|
||||
githubCredentialsProvider?: GithubCredentialsProvider;
|
||||
|
||||
/**
|
||||
* Optionally include a user transformer for transforming from GitHub users to User Entities
|
||||
*/
|
||||
userTransformer?: UserTransformer;
|
||||
|
||||
/**
|
||||
* Optionally include a team transformer for transforming from GitHub teams to Group Entities.
|
||||
* By default groups will be namespaced according to their GitHub org.
|
||||
*/
|
||||
teamTransformer?: TeamTransformer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingests org data (users and groups) from GitHub.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GithubMultiOrgEntityProvider implements EntityProvider {
|
||||
private connection?: EntityProviderConnection;
|
||||
private scheduleFn?: () => Promise<void>;
|
||||
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: GithubMultiOrgEntityProviderOptions,
|
||||
) {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const gitHubConfig = integrations.github.byUrl(options.githubUrl)?.config;
|
||||
|
||||
if (!gitHubConfig) {
|
||||
throw new Error(
|
||||
`There is no GitHub integration that matches ${options.githubUrl}. Please add a configuration entry for it under integrations.github.`,
|
||||
);
|
||||
}
|
||||
|
||||
const logger = options.logger.child({
|
||||
target: options.githubUrl,
|
||||
});
|
||||
|
||||
const provider = new GithubMultiOrgEntityProvider({
|
||||
id: options.id,
|
||||
gitHubConfig,
|
||||
githubCredentialsProvider:
|
||||
options.githubCredentialsProvider ||
|
||||
DefaultGithubCredentialsProvider.fromIntegrations(integrations),
|
||||
githubUrl: new URL(options.githubUrl).origin,
|
||||
logger,
|
||||
orgs: options.orgs,
|
||||
userTransformer: options.userTransformer,
|
||||
teamTransformer: options.teamTransformer,
|
||||
});
|
||||
|
||||
provider.schedule(options.schedule);
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly options: {
|
||||
id: string;
|
||||
gitHubConfig: GithubIntegrationConfig;
|
||||
githubCredentialsProvider: GithubCredentialsProvider;
|
||||
githubUrl: string;
|
||||
logger: Logger;
|
||||
orgs?: string[];
|
||||
userTransformer?: UserTransformer;
|
||||
teamTransformer?: TeamTransformer;
|
||||
},
|
||||
) {}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
|
||||
getProviderName() {
|
||||
return `GithubMultiOrgEntityProvider:${this.options.id}`;
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
|
||||
async connect(connection: EntityProviderConnection) {
|
||||
this.connection = connection;
|
||||
await this.scheduleFn?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one single complete ingestion. This is only necessary if you use
|
||||
* manual scheduling.
|
||||
*/
|
||||
async read(options?: { logger?: Logger }) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
const logger = options?.logger ?? this.options.logger;
|
||||
const { markReadComplete } = trackProgress(logger);
|
||||
|
||||
const allUsersMap = new Map();
|
||||
const allGroups: Entity[] = [];
|
||||
|
||||
const orgsToProcess = this.options.orgs?.length
|
||||
? this.options.orgs
|
||||
: await this.getAllOrgs(this.options.gitHubConfig);
|
||||
|
||||
for (const org of orgsToProcess) {
|
||||
try {
|
||||
const { headers, type: tokenType } =
|
||||
await this.options.githubCredentialsProvider.getCredentials({
|
||||
url: `${this.options.githubUrl}/${org}`,
|
||||
});
|
||||
const client = graphql.defaults({
|
||||
baseUrl: this.options.gitHubConfig.apiBaseUrl,
|
||||
headers,
|
||||
});
|
||||
|
||||
logger.info(`Reading GitHub users and teams for org: ${org}`);
|
||||
|
||||
const { users } = await getOrganizationUsers(
|
||||
client,
|
||||
org,
|
||||
tokenType,
|
||||
this.options.userTransformer,
|
||||
);
|
||||
|
||||
const { groups } = await getOrganizationTeams(
|
||||
client,
|
||||
org,
|
||||
this.defaultMultiOrgTeamTransformer.bind(this),
|
||||
);
|
||||
|
||||
// Grab current users from `allUsersMap` if they already exist in our
|
||||
// pending users so we can append to their group membership relations
|
||||
const pendingUsers = users.map(u => {
|
||||
const userRef = stringifyEntityRef(u);
|
||||
if (!allUsersMap.has(userRef)) {
|
||||
allUsersMap.set(userRef, u);
|
||||
}
|
||||
|
||||
return allUsersMap.get(userRef);
|
||||
});
|
||||
|
||||
assignGroupsToUsers(pendingUsers, groups);
|
||||
buildOrgHierarchy(groups);
|
||||
|
||||
allGroups.push(...groups);
|
||||
} catch (e) {
|
||||
logger.error(`Failed to read GitHub org data for ${org}: ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
const allUsers = Array.from(allUsersMap.values());
|
||||
|
||||
const { markCommitComplete } = markReadComplete({ allUsers, allGroups });
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
entities: [...allUsers, ...allGroups].map(entity => ({
|
||||
locationKey: `github-multi-org-provider:${this.options.id}`,
|
||||
entity: withLocations(
|
||||
`https://${this.options.gitHubConfig.host}`,
|
||||
entity,
|
||||
),
|
||||
})),
|
||||
});
|
||||
|
||||
markCommitComplete();
|
||||
}
|
||||
|
||||
private schedule(schedule: GithubMultiOrgEntityProviderOptions['schedule']) {
|
||||
if (!schedule || schedule === 'manual') {
|
||||
return;
|
||||
}
|
||||
|
||||
this.scheduleFn = async () => {
|
||||
const id = `${this.getProviderName()}:refresh`;
|
||||
await schedule.run({
|
||||
id,
|
||||
fn: async () => {
|
||||
const logger = this.options.logger.child({
|
||||
class: GithubMultiOrgEntityProvider.prototype.constructor.name,
|
||||
taskId: id,
|
||||
taskInstanceId: uuid.v4(),
|
||||
});
|
||||
|
||||
try {
|
||||
await this.read({ logger });
|
||||
} catch (error) {
|
||||
logger.error(`${this.getProviderName()} refresh failed`, error);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
private async defaultMultiOrgTeamTransformer(
|
||||
team: GithubTeam,
|
||||
ctx: TransformerContext,
|
||||
): Promise<GroupEntity | undefined> {
|
||||
const result = this.options.teamTransformer
|
||||
? await this.options.teamTransformer(team, ctx)
|
||||
: await defaultOrganizationTeamTransformer(team, ctx);
|
||||
|
||||
if (result) {
|
||||
result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US');
|
||||
// Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here
|
||||
result.spec.members = team.members.map(
|
||||
user => `${DEFAULT_NAMESPACE}/${user.login}`,
|
||||
);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Note: Does not support usage of PATs
|
||||
private async getAllOrgs(
|
||||
gitHubConfig: GithubIntegrationConfig,
|
||||
): Promise<string[]> {
|
||||
const githubAppMux = new GithubAppCredentialsMux(gitHubConfig);
|
||||
const installs = await githubAppMux.getAllInstallations();
|
||||
|
||||
return installs
|
||||
.map(install =>
|
||||
install.target_type === 'Organization' &&
|
||||
install.account &&
|
||||
install.account.login
|
||||
? install.account.login
|
||||
: undefined,
|
||||
)
|
||||
.filter(Boolean) as string[];
|
||||
}
|
||||
}
|
||||
|
||||
// Helps wrap the timing and logging behaviors
|
||||
function trackProgress(logger: Logger) {
|
||||
let timestamp = Date.now();
|
||||
let summary: string;
|
||||
|
||||
logger.info('Reading GitHub users and groups');
|
||||
|
||||
function markReadComplete(read: {
|
||||
allUsers: unknown[];
|
||||
allGroups: unknown[];
|
||||
}) {
|
||||
summary = `${read.allUsers.length} GitHub users and ${read.allGroups.length} GitHub groups`;
|
||||
const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
|
||||
timestamp = Date.now();
|
||||
logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
|
||||
return { markCommitComplete };
|
||||
}
|
||||
|
||||
function markCommitComplete() {
|
||||
const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
|
||||
logger.info(`Committed ${summary} in ${commitDuration} seconds.`);
|
||||
}
|
||||
|
||||
return { markReadComplete };
|
||||
}
|
||||
|
||||
// Makes sure that emitted entities have a proper location
|
||||
export function withLocations(baseUrl: string, entity: Entity): Entity {
|
||||
const login =
|
||||
entity.metadata.annotations?.[ANNOTATION_GITHUB_USER_LOGIN] ||
|
||||
entity.metadata.name;
|
||||
|
||||
let org = entity.metadata.namespace;
|
||||
let team = entity.metadata.name;
|
||||
const slug = entity.metadata.annotations?.[ANNOTATION_GITHUB_TEAM_SLUG];
|
||||
if (slug) {
|
||||
const [slugOrg, slugTeam] = splitTeamSlug(slug);
|
||||
org = slugOrg;
|
||||
team = slugTeam;
|
||||
}
|
||||
|
||||
const location =
|
||||
entity.kind === 'Group'
|
||||
? `url:${baseUrl}/orgs/${org}/teams/${team}`
|
||||
: `url:${baseUrl}/${login}`;
|
||||
return merge(
|
||||
{
|
||||
metadata: {
|
||||
annotations: {
|
||||
[ANNOTATION_LOCATION]: location,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: location,
|
||||
},
|
||||
},
|
||||
},
|
||||
entity,
|
||||
) as Entity;
|
||||
}
|
||||
Reference in New Issue
Block a user