From 7418d09737194211b329b1a8300ccb14b10b83ed Mon Sep 17 00:00:00 2001 From: Rogerio Angeliski Date: Mon, 12 Dec 2022 19:22:44 +0000 Subject: [PATCH] refct(events,catalog/github): remove catalog api dependency Signed-off-by: Rogerio Angeliski --- docs/integrations/github/org.md | 10 - .../api-report.md | 7 - .../src/lib/github.test.ts | 172 ------ .../src/lib/github.ts | 55 -- .../providers/GithubOrgEntityProvider.test.ts | 576 +++++++++--------- .../src/providers/GithubOrgEntityProvider.ts | 199 +----- 6 files changed, 306 insertions(+), 713 deletions(-) diff --git a/docs/integrations/github/org.md b/docs/integrations/github/org.md index bfda79d1f2..0aa9997bae 100644 --- a/docs/integrations/github/org.md +++ b/docs/integrations/github/org.md @@ -76,7 +76,6 @@ Set up your provider ```diff // packages/backend/src/plugins/catalogEventBasedProviders.ts -+import { CatalogClient } from '@backstage/catalog-client'; +import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EventSubscriber } from '@backstage/plugin-events-node'; @@ -94,8 +93,6 @@ Set up your provider + id: 'production', + orgUrl: 'https://github.com/backstage', + logger: env.logger, -+ catalogApi: new CatalogClient({ discoveryApi: env.discovery }), -+ tokenManager: env.tokenManager, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 60 }, + timeout: { minutes: 15 }, @@ -106,9 +103,6 @@ Set up your provider } ``` -**Attention:** -`catalogApi` and `tokenManager` are required at this variant compared to the one without events support. - You can check the official docs to [configure your webhook](https://docs.github.com/en/developers/webhooks-and-events/webhooks/creating-webhooks) and to [secure your request](https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks). The webhook will need to be configured to forward `organization`,`team` and `membership` events. @@ -165,10 +159,6 @@ There is also a `defaultUserTransformer` and `defaultOrganizationTeamTransformer You could use these and simply decorate the response from the default transformation if you only need to change a few properties. -**Attention:** -When you use the Events Support with a `TeamTransformer`, you need to ensure your entity will have the -annotation `github.com/team-id` with the value from `team.databaseId`. Without this, the `team.edit` event could fail to update - ### Resolving GitHub users via organization email When you authenticate users you should resolve them to an entity within the diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index bdf42aa6b3..4b07ffd593 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -5,7 +5,6 @@ ```ts import { AnalyzeOptions } from '@backstage/plugin-catalog-backend'; import { BackendFeature } from '@backstage/backend-plugin-api'; -import { CatalogApi } from '@backstage/catalog-client'; import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; @@ -189,8 +188,6 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; - catalogApi?: CatalogApi; - tokenManager?: TokenManager; }); // (undocumented) connect(connection: EntityProviderConnection): Promise; @@ -213,16 +210,12 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions; // @public export interface GithubOrgEntityProviderOptions { - // (undocumented) - catalogApi?: CatalogApi; githubCredentialsProvider?: GithubCredentialsProvider; id: string; logger: Logger; orgUrl: string; schedule?: 'manual' | TaskRunner; teamTransformer?: TeamTransformer; - // (undocumented) - tokenManager?: TokenManager; userTransformer?: UserTransformer; } diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 593450aeda..a9bcf2c49b 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -31,9 +31,6 @@ import { GithubTeam, createAddEntitiesOperation, createRemoveEntitiesOperation, - createReplaceEntitiesOperation, - removeUserFromGroup, - addUserToGroup, } from './github'; import fetch from 'node-fetch'; @@ -638,173 +635,4 @@ describe('github', () => { }); }); }); - - describe('createReplaceEntitiesOperation', () => { - it('create a function to replace deferred entities to a delta operation', () => { - const operation = createReplaceEntitiesOperation('my-id', 'host'); - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - expect(operation('org', [userEntity])).toEqual({ - removed: [ - { - locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - }); - }); - }); - - describe('removeUserFromGroup', () => { - it('should remove a user from the group', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - const { user, group } = removeUserFromGroup(userEntity, groupEntity); - - expect(user.spec.memberOf).toEqual([]); - expect(group.spec.members).toEqual(['user-1']); - }); - }); - - describe('addUserToGroup', () => { - it('should add a user to the group', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - const { user, group } = addUserToGroup(userEntity, groupEntity); - expect(user.spec.memberOf).toEqual(['new-team']); - expect(group.spec.members).toEqual(['user-1', 'githubuser']); - }); - it('should add a user to the group when the fields are not defined', () => { - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: {}, - }; - - const { user, group } = addUserToGroup(userEntity, groupEntity); - - expect(user.spec.memberOf).toEqual(['new-team']); - expect(group.spec.members).toEqual(['githubuser']); - }); - }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 22c405d7d3..d33ef4fb34 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -27,7 +27,6 @@ import { import { withLocations } from '../providers/GithubOrgEntityProvider'; import { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { merge, omit } from 'lodash'; // Graphql types @@ -419,11 +418,6 @@ export type DeferredEntitiesBuilder = ( entities: Entity[], ) => { added: DeferredEntity[]; removed: DeferredEntity[] }; -export type EntityUpdateOperation = ( - user: UserEntity, - group: GroupEntity, -) => { user: UserEntity; group: GroupEntity }; - export const createAddEntitiesOperation = (id: string, host: string) => (org: string, entities: Entity[]) => ({ removed: [], @@ -441,52 +435,3 @@ export const createRemoveEntitiesOperation = entity: withLocations(`https://${host}`, org, entity), })), }); - -export const createReplaceEntitiesOperation = - (id: string, host: string) => (org: string, entities: Entity[]) => { - const entitiesToReplace = entities.map(entity => ({ - locationKey: `github-org-provider:${id}`, - entity: withLocations(`https://${host}`, org, entity), - })); - - return { - removed: entitiesToReplace, - added: entitiesToReplace, - }; - }; - -export function removeUserFromGroup(user: UserEntity, group: GroupEntity) { - return { - group: merge(omit(group, 'spec.members'), { - spec: { - members: group?.spec?.members?.filter(m => m !== user.metadata.name), - }, - }), - - user: merge(omit(user, 'spec.memberOf'), { - spec: { - memberOf: user?.spec?.memberOf?.filter(m => m !== group.metadata.name), - }, - }), - }; -} - -export function addUserToGroup(user: UserEntity, group: GroupEntity) { - if (!group.spec?.members) { - group.spec = { - ...group.spec, - members: [], - }; - } - - if (!user.spec?.memberOf) { - user.spec = { - ...user.spec, - memberOf: [], - }; - } - - group.spec?.members?.push(user.metadata.name); - user.spec?.memberOf?.push(group.metadata.name); - return { group, user }; -} diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts index a13215d91b..270f184d7a 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.test.ts @@ -14,9 +14,8 @@ * limitations under the License. */ -import { getVoidLogger, TokenManager } from '@backstage/backend-common'; +import { getVoidLogger } from '@backstage/backend-common'; import { GroupEntity, UserEntity } from '@backstage/catalog-model'; -import { CatalogApi } from '@backstage/catalog-client'; import { GithubCredentialsProvider, GithubIntegrationConfig, @@ -28,19 +27,8 @@ import { GithubOrgEntityProvider, withLocations, } from './GithubOrgEntityProvider'; -import { cloneDeep } from 'lodash'; jest.mock('@octokit/graphql'); -const tokenManager: jest.Mocked = { - getToken: jest.fn(), - authenticate: jest.fn(), -}; - -const mockCatalogApi = { - getEntityByRef: jest.fn(), - getEntities: jest.fn(), -}; -const catalogApi = mockCatalogApi as Partial as CatalogApi; describe('GithubOrgEntityProvider', () => { describe('read', () => { @@ -273,7 +261,6 @@ describe('GithubOrgEntityProvider', () => { const githubCredentialsProvider: GithubCredentialsProvider = { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', @@ -281,8 +268,6 @@ describe('GithubOrgEntityProvider', () => { orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -363,16 +348,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -453,15 +434,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -547,15 +525,12 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); entityProvider.connect(entityProviderConnection); @@ -642,17 +617,61 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); + entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -680,71 +699,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/mygroup"', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/mygroup"', - }, - name: 'mygroup', - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const expectedEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - description: 'description from the new team', - annotations: { - 'backstage.io/edit-url': - 'https://github.com/orgs/test-org/teams/new-team/edit', - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'github.com/team-slug': 'test-org/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - profile: { - displayName: 'New Team', - }, - }, - }; - - mockCatalogApi.getEntities.mockResolvedValue({ - items: [cloneDeep(entity)], - }); await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, + locationKey: 'github-org-provider:my-id', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: entity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedEntity, }, ], + type: 'full', }); }); @@ -768,18 +784,60 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -804,109 +862,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - const expectedGroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const expectedUserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - mockCatalogApi.getEntityByRef - .mockResolvedValueOnce(cloneDeep(groupEntity)) - .mockResolvedValueOnce(cloneDeep(userEntity)); - await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, locationKey: 'github-org-provider:my-id', - entity: groupEntity, }, { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedGroupEntity, - }, - { - locationKey: 'github-org-provider:my-id', - entity: expectedUserEntity, }, ], + type: 'full', }); }); @@ -930,18 +947,60 @@ describe('GithubOrgEntityProvider', () => { getCredentials: mockGetCredentials, }; - tokenManager.getToken.mockResolvedValue({ token: 'my-token' }); - const entityProvider = new GithubOrgEntityProvider({ id: 'my-id', githubCredentialsProvider, orgUrl: 'https://github.com/backstage', gitHubConfig, logger, - tokenManager, - catalogApi, }); + const mockClient = jest.fn(); + + mockClient + .mockResolvedValueOnce({ + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }) + .mockResolvedValueOnce({ + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + name: 'Team', + description: 'The one and only team', + avatarUrl: 'http://example.com/team.jpeg', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'a' }], + }, + }, + ], + }, + }, + }); + + (graphql.defaults as jest.Mock).mockReturnValue(mockClient); entityProvider.connect(entityProviderConnection); const event: EventParams = { @@ -966,109 +1025,68 @@ describe('GithubOrgEntityProvider', () => { }, }; - const groupEntity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1', 'githubuser'], - }, - }; - - const userEntity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: ['new-team'], - }, - }; - - const expectedGroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'new-team', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/orgs/test-org/teams/new-team', - }, - }, - spec: { - type: 'team', - children: [], - members: ['user-1'], - }, - }; - - const expectedUserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name: 'githubuser', - annotations: { - 'backstage.io/managed-by-location': - 'url:https://github.com/githubuser', - 'backstage.io/managed-by-origin-location': - 'url:https://github.com/githubuser', - 'github.com/user-login': 'githubuser', - }, - }, - spec: { - memberOf: [], - }, - }; - - mockCatalogApi.getEntityByRef - .mockResolvedValueOnce(cloneDeep(groupEntity)) - .mockResolvedValueOnce(cloneDeep(userEntity)); - await entityProvider.onEvent(event); + await new Promise(process.nextTick); expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [ + entities: [ { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/a', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/a', + 'github.com/user-login': 'a', + }, + description: 'c', + name: 'a', + }, + spec: { + memberOf: ['team'], + profile: { + displayName: 'b', + email: 'd', + picture: 'e', + }, + }, + }, locationKey: 'github-org-provider:my-id', - entity: groupEntity, }, { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'backstage.io/managed-by-origin-location': + 'url:https://github.com/orgs/backstage/teams/team', + 'github.com/team-slug': 'blah/team', + }, + name: 'team', + description: 'The one and only team', + }, + spec: { + children: [], + parent: 'parent', + profile: { + displayName: 'Team', + picture: 'http://example.com/team.jpeg', + }, + type: 'team', + members: ['a'], + }, + }, locationKey: 'github-org-provider:my-id', - entity: userEntity, - }, - ], - added: [ - { - locationKey: 'github-org-provider:my-id', - entity: expectedGroupEntity, - }, - { - locationKey: 'github-org-provider:my-id', - entity: expectedUserEntity, }, ], + type: 'full', }); }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 7d7941f5ec..9b11f81697 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -18,10 +18,7 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, - DEFAULT_NAMESPACE, Entity, - GroupEntity, - UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -43,12 +40,8 @@ import { OrganizationMemberAddedEvent, OrganizationMemberRemovedEvent, TeamEvent, - TeamEditedEvent, - MembershipEvent, } from '@octokit/webhooks-types'; -import { CatalogApi } from '@backstage/catalog-client'; -import { TokenManager } from '@backstage/backend-common'; -import { merge, omit } from 'lodash'; +import { merge } from 'lodash'; import * as uuid from 'uuid'; import { Logger } from 'winston'; import { @@ -63,13 +56,9 @@ import { } from '../lib'; import { TeamTransformer, UserTransformer } from '../lib'; import { - addUserToGroup, - removeUserFromGroup, createAddEntitiesOperation, createRemoveEntitiesOperation, - createReplaceEntitiesOperation, DeferredEntitiesBuilder, - EntityUpdateOperation, } from '../lib/github'; /** @@ -126,9 +115,6 @@ export interface GithubOrgEntityProviderOptions { * Optionally include a team transformer for transforming from GitHub teams to Group Entities */ teamTransformer?: TeamTransformer; - - catalogApi?: CatalogApi; - tokenManager?: TokenManager; } /** @@ -143,8 +129,6 @@ export class GithubOrgEntityProvider private connection?: EntityProviderConnection; private scheduleFn?: () => Promise; - private eventConfigErrorThrown = false; - static fromConfig(config: Config, options: GithubOrgEntityProviderOptions) { const integrations = ScmIntegrations.fromConfig(config); const gitHubConfig = integrations.github.byUrl(options.orgUrl)?.config; @@ -169,8 +153,6 @@ export class GithubOrgEntityProvider DefaultGithubCredentialsProvider.fromIntegrations(integrations), userTransformer: options.userTransformer, teamTransformer: options.teamTransformer, - catalogApi: options.catalogApi, - tokenManager: options.tokenManager, }); provider.schedule(options.schedule); @@ -187,8 +169,6 @@ export class GithubOrgEntityProvider githubCredentialsProvider?: GithubCredentialsProvider; userTransformer?: UserTransformer; teamTransformer?: TeamTransformer; - catalogApi?: CatalogApi; - tokenManager?: TokenManager; }, ) { this.credentialsProvider = @@ -266,11 +246,6 @@ export class GithubOrgEntityProvider const { logger } = this.options; logger.debug(`Received event from ${params.topic}`); - if (!this.canHandleEvents()) { - logger.debug(`Skipping event ${params.topic}`); - return; - } - const addEntitiesOperation = createAddEntitiesOperation( this.options.id, this.options.gitHubConfig.host, @@ -279,10 +254,6 @@ export class GithubOrgEntityProvider this.options.id, this.options.gitHubConfig.host, ); - const replaceEntitiesOperation = createReplaceEntitiesOperation( - this.options.id, - this.options.gitHubConfig.host, - ); // handle change users in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#organization @@ -312,26 +283,14 @@ export class GithubOrgEntityProvider : removeEntitiesOperation; await this.onTeamChangeInOrganization(teamEvent, createDeltaOperation); } else if (teamEvent.action === 'edited') { - await this.onTeamEditedInOrganization( - teamEvent, - replaceEntitiesOperation, - ); + this.onEventToRebuildOrg(); } } // handle change membership in the org // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership if (params.topic.includes('membership')) { - const membershipEvent = params.eventPayload as MembershipEvent; - const updateOperation = - membershipEvent.action === 'added' - ? addUserToGroup - : removeUserFromGroup; - await this.onMemberChangeToTeam( - membershipEvent, - replaceEntitiesOperation, - updateOperation, - ); + this.onEventToRebuildOrg(); } return; @@ -342,22 +301,6 @@ export class GithubOrgEntityProvider return ['github.organization', 'github.team', 'github.membership']; } - private canHandleEvents(): boolean { - if (this.options.catalogApi && this.options.tokenManager) { - return true; - } - - // throw only once - if (!this.eventConfigErrorThrown) { - this.eventConfigErrorThrown = true; - throw new Error( - `${this.getProviderName()} not well configured to handle ${this.supportsEventTopics()}. Missing CatalogApi and/or TokenManager.`, - ); - } - - return false; - } - private async onTeamChangeInOrganization( event: TeamEvent, createDeltaOperation: DeferredEntitiesBuilder, @@ -412,71 +355,14 @@ export class GithubOrgEntityProvider }); } - private async onTeamEditedInOrganization( - event: TeamEditedEvent, - createDeltaOperation: DeferredEntitiesBuilder, - ) { - if (!this.connection) { - throw new Error('Not initialized'); - } - - const { name } = event.changes; - const org = event.organization.login; - - const { token } = await this.options.tokenManager!.getToken(); - const groups = await this.options.catalogApi!.getEntities( - { - filter: [ - { - kind: 'Group', - [`metadata.annotations.github.com/team-id`]: `${event.team.id}`, - }, - ], - fields: ['apiVersion', 'kind', 'metadata', 'spec'], - }, - { token }, - ); - - if (groups.items.length === 0) { - this.options.logger.debug( - `Skipping event because couldn't found group with 'github.com/node-id':${event.team.node_id}`, - ); - return; - } - - const group = groups.items[0] as GroupEntity; - const { removed } = createDeltaOperation(org, [group]); - const { name: newTeamName, html_url: url, description, slug } = event.team; - - if (name) { - merge(group, { - metadata: { - name: slug, - annotations: { - [`backstage.io/edit-url`]: `${url}/edit`, - [`github.com/team-slug`]: `${org}/${slug}`, - [ANNOTATION_LOCATION]: `url:${url}`, - [ANNOTATION_ORIGIN_LOCATION]: `url:${url}`, - }, - }, - spec: { - profile: { - displayName: newTeamName, - }, - }, + private onEventToRebuildOrg() { + try { + this.read(this.options).then(() => { + this.options.logger.debug('Event process finished'); }); + } catch (e) { + this.options.logger.debug(`Event process error ${e}`); } - - if (description) { - group.metadata.description = description; - } - - const { added } = createDeltaOperation(org, [group]); - await this.connection.applyMutation({ - type: 'delta', - removed, - added, - }); } private async onMemberChangeInOrganization( @@ -523,73 +409,6 @@ export class GithubOrgEntityProvider }); } - private async onMemberChangeToTeam( - event: MembershipEvent, - createDeltaOperation: DeferredEntitiesBuilder, - updateEntities: EntityUpdateOperation, - ) { - if (!this.connection) { - throw new Error('Not initialized'); - } - - // The docs are saying I will receive the slug for the removed event, but the types don't reflect that, - // so I will just check to be sure the slug is there - // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#membership - if (!('slug' in event.team)) { - return; - } - const teamSlug = event.team.slug; - const userSlug = event.member.login; - - const { token } = await this.options.tokenManager!.getToken(); - const org = event.organization.login; - let group = (await this.options.catalogApi!.getEntityByRef( - { - kind: 'Group', - namespace: DEFAULT_NAMESPACE, - name: teamSlug, - }, - { token }, - )) as GroupEntity; - - if (!group) { - this.options.logger.debug( - `Skipping event because couldn't found group ':${teamSlug} for namespace ${DEFAULT_NAMESPACE}`, - ); - return; - } - - let user = (await this.options.catalogApi!.getEntityByRef( - { - kind: 'User', - namespace: DEFAULT_NAMESPACE, - name: userSlug, - }, - { token }, - )) as UserEntity; - - group = omit(group, 'relations'); - - if (!user) { - this.options.logger.debug( - `Skipping event because couldn't found user ':${userSlug} for namespace ${DEFAULT_NAMESPACE}`, - ); - return; - } - user = omit(user, 'relations'); - - const { removed } = createDeltaOperation(org, [group, user]); - - const result = updateEntities(user, group); - - const { added } = createDeltaOperation(org, [result.group, result.user]); - await this.connection.applyMutation({ - type: 'delta', - removed, - added, - }); - } - private schedule(schedule: GithubOrgEntityProviderOptions['schedule']) { if (!schedule || schedule === 'manual') { return;