paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

Merge pull request #24700 from backstage/github-multi-org-fix-namespace

Browse Source 
catalog-backend-module-github-org: fix default namespace for groups
This commit is contained in:
Vincenzo Scamporlino
2024-05-13 22:10:11 +02:00
committed by GitHub
parent 7d9369cc88 5f3f8a86fd
commit 95ee6d6a24
9 changed files with 452 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/red-mangos-fly.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github-org': patch
---
Fixed an issue where the `catalog-backend-module-github-org` would not correctly create groups using `default` as namespace in case a single organization was configured.
.changeset/wild-cats-hug.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Added `alwaysUseDefaultNamespace` option to `GithubMultiOrgEntityProvider`.
If set to true, the provider will use `default` as the namespace for all group entities. Groups with the same name across different orgs will be considered the same group.
docs/backend-system/building-backends/08-migrating.md
+67 -8
View File
@@ -452,7 +452,7 @@ catalog:
/* highlight-add-end */
```
To migrate `GithubMultiOrgEntityProvider` and `GithubOrgEntityProvider` to the new backend system, add a reference to `@backstage/plugin-catalog-backend-module-github-org`.
To migrate `GithubMultiOrgEntityProvider` or `GithubOrgEntityProvider` to the new backend system, add a reference to `@backstage/plugin-catalog-backend-module-github-org`.
```ts title="packages/backend/src/index.ts"
backend.add(import('@backstage/plugin-catalog-backend/alpha'));
@@ -461,20 +461,79 @@ backend.add(import('@backstage/plugin-catalog-backend-module-github-org'));
/* highlight-add-end */
```
If you were providing a `schedule` in code, this now needs to be set via configuration.
All other Github configuration in `app-config.yaml` remains the same.
##### GithubOrgEntityProvider
If you were using `GithubOrgEntityProvider` you might have been configured in code like this:
```ts title="packages/backend/src/plugins/catalog.ts"
// The org URL below needs to match a configured integrations.github entry
// specified in your app-config.
builder.addEntityProvider(
GithubOrgEntityProvider.fromConfig(env.config, {
id: 'production',
orgUrl: 'https://github.com/backstage',
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 60 },
timeout: { minutes: 15 },
}),
}),
);
```
This now needs to be set via configuration. The options defined above are now set in `app-config.yaml` instead as shown below:
```yaml title="app-config.yaml"
catalog:
/* highlight-add-start */
providers:
githubOrg:
yourProviderId:
# ...
/* highlight-add-start */
- id: production
githubUrl: 'https://github.com',
orgs: ['backstage']
schedule:
frequency: PT30M
timeout: PT3M
/* highlight-add-end */
timeout: PT15M
/* highlight-add-end */
```
##### GithubMultiOrgEntityProvider
If you were using `GithubMultiOrgEntityProvider` you might have been configured in code like this:
```ts title="packages/backend/src/plugins/catalog.ts"
// The GitHub URL below needs to match a configured integrations.github entry
// specified in your app-config.
builder.addEntityProvider(
GithubMultiOrgEntityProvider.fromConfig(env.config, {
id: 'production',
githubUrl: 'https://github.com',
// Set the following to list the GitHub orgs you wish to ingest from. You can
// also omit this option to ingest all orgs accessible by your GitHub integration
orgs: ['org-a', 'org-b'],
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 60 },
timeout: { minutes: 15 },
}),
}),
);
```
This now needs to be set via configuration. The options defined above are now set in `app-config.yaml` instead as shown below:
```yaml title="app-config.yaml"
catalog:
/* highlight-add-start */
providers:
githubOrg:
- id: production
githubUrl: 'https://github.com',
orgs: ['org-a', 'org-b'],
schedule:
frequency: PT30M
timeout: PT15M
/* highlight-add-end */
```
If you were providing transformers, these can be configured by extending `githubOrgEntityProviderTransformsExtensionPoint`
docs/integrations/github/org.md
+1 -1
View File
@@ -45,7 +45,7 @@ Next add the basic configuration to `app-config.yaml`
catalog:
providers:
githubOrg:
id: github
id: production
githubUrl: https://github.com
orgs: ['organization-1', 'organization-2', 'organization-3']
schedule:
plugins/catalog-backend-module-github-org/src/GithubOrgEntityCleanerProvider.ts
+42
View File
@@ -0,0 +1,42 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LoggerService } from '@backstage/backend-plugin-api';
import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-node';
export class GithubOrgEntityCleanerProvider implements EntityProvider {
logger: LoggerService;
constructor(private readonly options: { id: string; logger: LoggerService }) {
this.logger = options.logger.child({ target: this.getProviderName() });
}
getProviderName() {
return `GithubOrgEntityProvider:${this.options.id}`;
}
async connect(connection: EntityProviderConnection) {
// Clean up any existing entities
connection
.applyMutation({
type: 'full',
entities: [],
})
.catch(error => {
this.logger.error('Failed to clean up entities', error);
});
}
}
plugins/catalog-backend-module-github-org/src/module.ts
+10 -1
View File
@@ -31,6 +31,7 @@ import {
} from '@backstage/plugin-catalog-backend-module-github';
import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
import { eventsServiceRef } from '@backstage/plugin-events-node';
import { GithubOrgEntityCleanerProvider } from './GithubOrgEntityCleanerProvider';
/**
* Interface for {@link githubOrgEntityProviderTransformsExtensionPoint}.
@@ -99,8 +100,14 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({
logger: coreServices.logger,
scheduler: coreServices.scheduler,
},
async init({ catalog, config, events, logger, scheduler }) {
for (const definition of readDefinitionsFromConfig(config)) {
const definitions = readDefinitionsFromConfig(config);
for (const definition of definitions) {
catalog.addEntityProvider(
new GithubOrgEntityCleanerProvider({ id: definition.id, logger }),
);
catalog.addEntityProvider(
GithubMultiOrgEntityProvider.fromConfig(config, {
id: definition.id,
@@ -113,6 +120,8 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({
logger,
userTransformer,
teamTransformer,
alwaysUseDefaultNamespace:
definitions.length === 1 && definition.orgs?.length === 1,
}),
);
}
plugins/catalog-backend-module-github/api-report.md
+2
View File
@@ -150,6 +150,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
orgs?: string[];
userTransformer?: UserTransformer;
teamTransformer?: TeamTransformer;
alwaysUseDefaultNamespace?: boolean;
});
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
@@ -165,6 +166,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
// @public
export interface GithubMultiOrgEntityProviderOptions {
alwaysUseDefaultNamespace?: boolean;
events?: EventsService;
githubCredentialsProvider?: GithubCredentialsProvider;
githubUrl: string;
plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts
+303 -15
View File
@@ -68,26 +68,24 @@ describe('GithubMultiOrgEntityProvider', () => {
headers: { token: 'blah' },
type: 'app',
});
const githubCredentialsProvider: GithubCredentialsProvider = {
getCredentials: mockGetCredentials,
};
entityProvider = new GithubMultiOrgEntityProvider({
id: 'my-id',
gitHubConfig,
githubCredentialsProvider,
githubUrl: 'https://github.com',
logger,
orgs: ['orgA', 'orgB'], // only include for tests that require it
});
entityProvider.connect(entityProviderConnection);
});
afterEach(() => jest.resetAllMocks());
it('should read specified orgs', async () => {
entityProvider = new GithubMultiOrgEntityProvider({
id: 'my-id',
gitHubConfig,
githubCredentialsProvider: {
getCredentials: mockGetCredentials,
},
githubUrl: 'https://github.com',
logger,
orgs: ['orgA', 'orgB'],
});
await entityProvider.connect(entityProviderConnection);
mockClient
.mockResolvedValueOnce({
organization: {
@@ -348,6 +346,19 @@ describe('GithubMultiOrgEntityProvider', () => {
});
it('should read every accessible org', async () => {
entityProvider = new GithubMultiOrgEntityProvider({
id: 'my-id',
gitHubConfig,
githubCredentialsProvider: {
getCredentials: mockGetCredentials,
},
githubUrl: 'https://github.com',
logger,
orgs: ['orgA', 'orgB'],
});
await entityProvider.connect(entityProviderConnection);
getAllInstallationsMock.mockResolvedValue([
{
target_type: 'Organization',
@@ -636,7 +647,284 @@ describe('GithubMultiOrgEntityProvider', () => {
});
});
it('should use the default namespace if options.alwaysUseDefaultNamespace is provided', async () => {
mockClient
.mockResolvedValueOnce({
organization: {
membersWithRole: {
pageInfo: { hasNextPage: false },
nodes: [
{
login: 'a',
name: 'b',
bio: 'c',
email: 'd',
avatarUrl: 'e',
},
{
login: 'x',
name: 'y',
bio: 'z',
email: 'w',
avatarUrl: 'v',
},
],
},
},
})
.mockResolvedValueOnce({
organization: {
teams: {
pageInfo: { hasNextPage: false },
nodes: [
{
slug: 'team',
combinedSlug: 'orgC/team',
name: 'Team',
description: 'The one and only team',
avatarUrl: 'http://example.com/team.jpeg',
parentTeam: {
slug: 'parent',
combinedSlug: '',
members: { pageInfo: { hasNextPage: false }, nodes: [] },
},
members: {
pageInfo: { hasNextPage: false },
nodes: [{ login: 'a' }, { login: 'x' }],
},
},
],
},
},
})
.mockResolvedValueOnce({
organization: {
membersWithRole: {
pageInfo: { hasNextPage: false },
nodes: [
{
login: 'a',
name: 'b',
bio: 'c',
email: 'd',
avatarUrl: 'e',
},
{
login: 'x',
name: 'y',
bio: 'z',
email: 'w',
avatarUrl: 'v',
},
{
login: 'q',
name: 'r',
bio: 's',
email: 't',
avatarUrl: 'u',
},
],
},
},
})
.mockResolvedValueOnce({
organization: {
teams: {
pageInfo: { hasNextPage: false },
nodes: [
{
slug: 'team',
combinedSlug: 'orgD/team',
name: 'Team',
description: 'The one and only team',
avatarUrl: 'http://example.com/team.jpeg',
parentTeam: {
slug: 'parent',
combinedSlug: '',
members: { pageInfo: { hasNextPage: false }, nodes: [] },
},
members: {
pageInfo: { hasNextPage: false },
nodes: [{ login: 'a' }, { login: 'q' }],
},
},
],
},
},
});
(graphql.defaults as jest.Mock).mockReturnValue(mockClient);
entityProvider = new GithubMultiOrgEntityProvider({
id: 'my-id',
gitHubConfig,
githubCredentialsProvider: {
getCredentials: mockGetCredentials,
},
githubUrl: 'https://github.com',
logger,
orgs: ['orgA', 'orgB'],
alwaysUseDefaultNamespace: true,
});
await entityProvider.connect(entityProviderConnection);
await entityProvider.read();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
entities: [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/a',
'backstage.io/managed-by-origin-location':
'url:https://github.com/a',
'github.com/user-login': 'a',
},
description: 'c',
name: 'a',
},
spec: {
memberOf: ['team'],
profile: {
displayName: 'b',
email: 'd',
picture: 'e',
},
},
},
locationKey: 'github-multi-org-provider:my-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/x',
'backstage.io/managed-by-origin-location':
'url:https://github.com/x',
'github.com/user-login': 'x',
},
description: 'z',
name: 'x',
},
spec: {
memberOf: ['team'],
profile: {
displayName: 'y',
email: 'w',
picture: 'v',
},
},
},
locationKey: 'github-multi-org-provider:my-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/q',
'backstage.io/managed-by-origin-location':
'url:https://github.com/q',
'github.com/user-login': 'q',
},
description: 's',
name: 'q',
},
spec: {
memberOf: ['team'],
profile: {
displayName: 'r',
email: 't',
picture: 'u',
},
},
},
locationKey: 'github-multi-org-provider:my-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/orgs/orgC/teams/team',
'backstage.io/managed-by-origin-location':
'url:https://github.com/orgs/orgC/teams/team',
'github.com/team-slug': 'orgC/team',
},
name: 'team',
description: 'The one and only team',
},
spec: {
children: [],
parent: 'parent',
profile: {
displayName: 'Team',
picture: 'http://example.com/team.jpeg',
},
type: 'team',
members: ['default/a', 'default/x'],
},
},
locationKey: 'github-multi-org-provider:my-id',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://github.com/orgs/orgD/teams/team',
'backstage.io/managed-by-origin-location':
'url:https://github.com/orgs/orgD/teams/team',
'github.com/team-slug': 'orgD/team',
},
name: 'team',
description: 'The one and only team',
},
spec: {
children: [],
parent: 'parent',
profile: {
displayName: 'Team',
picture: 'http://example.com/team.jpeg',
},
type: 'team',
members: ['default/a', 'default/q'],
},
},
locationKey: 'github-multi-org-provider:my-id',
},
],
type: 'full',
});
});
it('should not call applyMutation if an error is thrown', async () => {
entityProvider = new GithubMultiOrgEntityProvider({
id: 'my-id',
gitHubConfig,
githubCredentialsProvider: {
getCredentials: mockGetCredentials,
},
githubUrl: 'https://github.com',
logger,
orgs: ['orgA', 'orgB'],
});
await entityProvider.connect(entityProviderConnection);
mockClient.mockImplementationOnce(() => {
throw new Error('Network Error');
});
plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts
+15 -1
View File
@@ -143,6 +143,15 @@ export interface GithubMultiOrgEntityProviderOptions {
*/
githubCredentialsProvider?: GithubCredentialsProvider;
/**
* Use the default namespace for groups. By default, groups will be namespaced according to their GitHub org.
*
* @remarks
*
* If set to true, groups with the same name across different orgs will be considered the same group.
*/
alwaysUseDefaultNamespace?: boolean;
/**
* Optionally include a user transformer for transforming from GitHub users to User Entities
*/
@@ -198,6 +207,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
userTransformer: options.userTransformer,
teamTransformer: options.teamTransformer,
events: options.events,
alwaysUseDefaultNamespace: options.alwaysUseDefaultNamespace,
});
provider.schedule(options.schedule);
@@ -216,6 +226,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
orgs?: string[];
userTransformer?: UserTransformer;
teamTransformer?: TeamTransformer;
alwaysUseDefaultNamespace?: boolean;
},
) {}
@@ -846,7 +857,10 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
const result = await defaultOrganizationTeamTransformer(team, ctx);
if (result && result.spec) {
result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US');
if (!this.options.alwaysUseDefaultNamespace) {
result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US');
}
// Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here
result.spec.members = team.members.map(
user => `${DEFAULT_NAMESPACE}/${user.login}`,