diff --git a/.changeset/red-mangos-fly.md b/.changeset/red-mangos-fly.md new file mode 100644 index 0000000000..73af978be2 --- /dev/null +++ b/.changeset/red-mangos-fly.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-github-org': patch +--- + +Fixed an issue where the `catalog-backend-module-github-org` would not correctly create groups using `default` as namespace in case a single organization was configured. diff --git a/.changeset/wild-cats-hug.md b/.changeset/wild-cats-hug.md new file mode 100644 index 0000000000..e33d399dab --- /dev/null +++ b/.changeset/wild-cats-hug.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +--- + +Added `alwaysUseDefaultNamespace` option to `GithubMultiOrgEntityProvider`. + +If set to true, the provider will use `default` as the namespace for all group entities. Groups with the same name across different orgs will be considered the same group. diff --git a/docs/backend-system/building-backends/08-migrating.md b/docs/backend-system/building-backends/08-migrating.md index 0fbb368449..fc0bf35267 100644 --- a/docs/backend-system/building-backends/08-migrating.md +++ b/docs/backend-system/building-backends/08-migrating.md @@ -452,7 +452,7 @@ catalog: /* highlight-add-end */ ``` -To migrate `GithubMultiOrgEntityProvider` and `GithubOrgEntityProvider` to the new backend system, add a reference to `@backstage/plugin-catalog-backend-module-github-org`. +To migrate `GithubMultiOrgEntityProvider` or `GithubOrgEntityProvider` to the new backend system, add a reference to `@backstage/plugin-catalog-backend-module-github-org`. ```ts title="packages/backend/src/index.ts" backend.add(import('@backstage/plugin-catalog-backend/alpha')); @@ -461,20 +461,79 @@ backend.add(import('@backstage/plugin-catalog-backend-module-github-org')); /* highlight-add-end */ ``` -If you were providing a `schedule` in code, this now needs to be set via configuration. -All other Github configuration in `app-config.yaml` remains the same. +##### GithubOrgEntityProvider + +If you were using `GithubOrgEntityProvider` you might have been configured in code like this: + +```ts title="packages/backend/src/plugins/catalog.ts" +// The org URL below needs to match a configured integrations.github entry +// specified in your app-config. +builder.addEntityProvider( + GithubOrgEntityProvider.fromConfig(env.config, { + id: 'production', + orgUrl: 'https://github.com/backstage', + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + }), +); +``` + +This now needs to be set via configuration. The options defined above are now set in `app-config.yaml` instead as shown below: ```yaml title="app-config.yaml" catalog: + /* highlight-add-start */ providers: githubOrg: - yourProviderId: - # ... - /* highlight-add-start */ + - id: production + githubUrl: 'https://github.com', + orgs: ['backstage'] schedule: frequency: PT30M - timeout: PT3M - /* highlight-add-end */ + timeout: PT15M + /* highlight-add-end */ +``` + +##### GithubMultiOrgEntityProvider + +If you were using `GithubMultiOrgEntityProvider` you might have been configured in code like this: + +```ts title="packages/backend/src/plugins/catalog.ts" +// The GitHub URL below needs to match a configured integrations.github entry +// specified in your app-config. +builder.addEntityProvider( + GithubMultiOrgEntityProvider.fromConfig(env.config, { + id: 'production', + githubUrl: 'https://github.com', + // Set the following to list the GitHub orgs you wish to ingest from. You can + // also omit this option to ingest all orgs accessible by your GitHub integration + orgs: ['org-a', 'org-b'], + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, + }), + }), +); +``` + +This now needs to be set via configuration. The options defined above are now set in `app-config.yaml` instead as shown below: + +```yaml title="app-config.yaml" +catalog: + /* highlight-add-start */ + providers: + githubOrg: + - id: production + githubUrl: 'https://github.com', + orgs: ['org-a', 'org-b'], + schedule: + frequency: PT30M + timeout: PT15M + /* highlight-add-end */ ``` If you were providing transformers, these can be configured by extending `githubOrgEntityProviderTransformsExtensionPoint` diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index 0b33cfa184..94eb551c7c 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -45,7 +45,7 @@ Next add the basic configuration to `app-config.yaml` catalog: providers: githubOrg: - id: github + id: production githubUrl: https://github.com orgs: ['organization-1', 'organization-2', 'organization-3'] schedule: diff --git a/plugins/catalog-backend-module-github-org/src/GithubOrgEntityCleanerProvider.ts b/plugins/catalog-backend-module-github-org/src/GithubOrgEntityCleanerProvider.ts new file mode 100644 index 0000000000..a00cd504d7 --- /dev/null +++ b/plugins/catalog-backend-module-github-org/src/GithubOrgEntityCleanerProvider.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { LoggerService } from '@backstage/backend-plugin-api'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; + +export class GithubOrgEntityCleanerProvider implements EntityProvider { + logger: LoggerService; + constructor(private readonly options: { id: string; logger: LoggerService }) { + this.logger = options.logger.child({ target: this.getProviderName() }); + } + + getProviderName() { + return `GithubOrgEntityProvider:${this.options.id}`; + } + async connect(connection: EntityProviderConnection) { + // Clean up any existing entities + connection + .applyMutation({ + type: 'full', + entities: [], + }) + .catch(error => { + this.logger.error('Failed to clean up entities', error); + }); + } +} diff --git a/plugins/catalog-backend-module-github-org/src/module.ts b/plugins/catalog-backend-module-github-org/src/module.ts index 0baa768644..31d4cef1c0 100644 --- a/plugins/catalog-backend-module-github-org/src/module.ts +++ b/plugins/catalog-backend-module-github-org/src/module.ts @@ -31,6 +31,7 @@ import { } from '@backstage/plugin-catalog-backend-module-github'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { GithubOrgEntityCleanerProvider } from './GithubOrgEntityCleanerProvider'; /** * Interface for {@link githubOrgEntityProviderTransformsExtensionPoint}. @@ -99,8 +100,14 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({ logger: coreServices.logger, scheduler: coreServices.scheduler, }, + async init({ catalog, config, events, logger, scheduler }) { - for (const definition of readDefinitionsFromConfig(config)) { + const definitions = readDefinitionsFromConfig(config); + + for (const definition of definitions) { + catalog.addEntityProvider( + new GithubOrgEntityCleanerProvider({ id: definition.id, logger }), + ); catalog.addEntityProvider( GithubMultiOrgEntityProvider.fromConfig(config, { id: definition.id, @@ -113,6 +120,8 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({ logger, userTransformer, teamTransformer, + alwaysUseDefaultNamespace: + definitions.length === 1 && definition.orgs?.length === 1, }), ); } diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index b53d1b7b17..0b68170255 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -150,6 +150,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { orgs?: string[]; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + alwaysUseDefaultNamespace?: boolean; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -165,6 +166,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { // @public export interface GithubMultiOrgEntityProviderOptions { + alwaysUseDefaultNamespace?: boolean; events?: EventsService; githubCredentialsProvider?: GithubCredentialsProvider; githubUrl: string; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts index 0542e64d64..c89901fcb3 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.test.ts @@ -68,26 +68,24 @@ describe('GithubMultiOrgEntityProvider', () => { headers: { token: 'blah' }, type: 'app', }); - - const githubCredentialsProvider: GithubCredentialsProvider = { - getCredentials: mockGetCredentials, - }; - - entityProvider = new GithubMultiOrgEntityProvider({ - id: 'my-id', - gitHubConfig, - githubCredentialsProvider, - githubUrl: 'https://github.com', - logger, - orgs: ['orgA', 'orgB'], // only include for tests that require it - }); - - entityProvider.connect(entityProviderConnection); }); afterEach(() => jest.resetAllMocks()); it('should read specified orgs', async () => { + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider: { + getCredentials: mockGetCredentials, + }, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + await entityProvider.connect(entityProviderConnection); + mockClient .mockResolvedValueOnce({ organization: { @@ -348,6 +346,19 @@ describe('GithubMultiOrgEntityProvider', () => { }); it('should read every accessible org', async () => { + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider: { + getCredentials: mockGetCredentials, + }, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + await entityProvider.connect(entityProviderConnection); + getAllInstallationsMock.mockResolvedValue([ { target_type: 'Organization', @@ -636,7 +647,284 @@ describe('GithubMultiOrgEntityProvider', () => { }); }); + it('should use the default namespace if options.alwaysUseDefaultNamespace is provided', async () => { + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgC/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'x' }], + }, + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + { + login: 'x', + name: 'y', + bio: 'z', + email: 'w', + avatarUrl: 'v', + }, + { + login: 'q', + name: 'r', + bio: 's', + email: 't', + avatarUrl: 'u', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'orgD/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }, { login: 'q' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider: { + getCredentials: mockGetCredentials, + }, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + alwaysUseDefaultNamespace: true, + }); + + await entityProvider.connect(entityProviderConnection); + await entityProvider.read(); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/x', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/x', + 'github.com/user-login': 'x', + }, + description: 'z', + name: 'x', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'y', + email: 'w', + picture: 'v', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/q', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/q', + 'github.com/user-login': 'q', + }, + description: 's', + name: 'q', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'r', + email: 't', + picture: 'u', + }, + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgC/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgC/teams/team', + 'github.com/team-slug': 'orgC/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/x'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/orgD/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/orgD/teams/team', + 'github.com/team-slug': 'orgD/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['default/a', 'default/q'], + }, + }, + locationKey: 'github-multi-org-provider:my-id', + }, + ], + type: 'full', + }); + }); + it('should not call applyMutation if an error is thrown', async () => { + entityProvider = new GithubMultiOrgEntityProvider({ + id: 'my-id', + gitHubConfig, + githubCredentialsProvider: { + getCredentials: mockGetCredentials, + }, + githubUrl: 'https://github.com', + logger, + orgs: ['orgA', 'orgB'], + }); + + await entityProvider.connect(entityProviderConnection); + mockClient.mockImplementationOnce(() => { throw new Error('Network Error'); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index f07d033ecd..756b9902c6 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -143,6 +143,15 @@ export interface GithubMultiOrgEntityProviderOptions { */ githubCredentialsProvider?: GithubCredentialsProvider; + /** + * Use the default namespace for groups. By default, groups will be namespaced according to their GitHub org. + * + * @remarks + * + * If set to true, groups with the same name across different orgs will be considered the same group. + */ + alwaysUseDefaultNamespace?: boolean; + /** * Optionally include a user transformer for transforming from GitHub users to User Entities */ @@ -198,6 +207,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, events: options.events, + alwaysUseDefaultNamespace: options.alwaysUseDefaultNamespace, }); provider.schedule(options.schedule); @@ -216,6 +226,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { orgs?: string[]; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; + alwaysUseDefaultNamespace?: boolean; }, ) {} @@ -846,7 +857,10 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { const result = await defaultOrganizationTeamTransformer(team, ctx); if (result && result.spec) { - result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); + if (!this.options.alwaysUseDefaultNamespace) { + result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); + } + // Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here result.spec.members = team.members.map( user => `${DEFAULT_NAMESPACE}/${user.login}`,