From b420b8012461084b245d142c9203f1b139ac9326 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 20 Jul 2023 16:09:59 +1000 Subject: [PATCH 1/6] relax github transform types Signed-off-by: Jack Grigg --- .../catalog-model/src/entity/conditions.ts | 14 ++++ .../src/lib/defaultTransformers.ts | 10 +-- .../src/lib/github.test.ts | 2 +- .../src/lib/github.ts | 32 ++++---- .../GithubMultiOrgReaderProcessor.ts | 28 ++++--- .../processors/GithubOrgReaderProcessor.ts | 17 +++-- .../providers/GithubMultiOrgEntityProvider.ts | 75 +++++++++++-------- .../src/providers/GithubOrgEntityProvider.ts | 63 ++++++++++------ 8 files changed, 150 insertions(+), 91 deletions(-) diff --git a/packages/catalog-model/src/entity/conditions.ts b/packages/catalog-model/src/entity/conditions.ts index d761695b5e..7e798ccb31 100644 --- a/packages/catalog-model/src/entity/conditions.ts +++ b/packages/catalog-model/src/entity/conditions.ts @@ -50,6 +50,14 @@ export function isDomainEntity(entity: Entity): entity is DomainEntity { export function isGroupEntity(entity: Entity): entity is GroupEntity { return entity.kind.toLocaleUpperCase('en-US') === 'GROUP'; } +/** + * @public + */ +export function areGroupEntities( + entities: Entity[], +): entities is GroupEntity[] { + return entities.every(e => isGroupEntity(e)); +} /** * @public */ @@ -74,3 +82,9 @@ export function isSystemEntity(entity: Entity): entity is SystemEntity { export function isUserEntity(entity: Entity): entity is UserEntity { return entity.kind.toLocaleUpperCase('en-US') === 'USER'; } +/** + * @public + */ +export function areUserEntities(entities: Entity[]): entities is UserEntity[] { + return entities.every(e => isUserEntity(e)); +} diff --git a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts index 7669fd5b47..71613d88d2 100644 --- a/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts +++ b/plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model'; import { graphql } from '@octokit/graphql'; import { ANNOTATION_GITHUB_TEAM_SLUG, @@ -34,24 +34,24 @@ export interface TransformerContext { } /** - * Transformer for GitHub users to UserEntity + * Transformer for GitHub users to an Entity * * @public */ export type UserTransformer = ( item: GithubUser, ctx: TransformerContext, -) => Promise; +) => Promise; /** - * Transformer for GitHub Team to GroupEntity + * Transformer for GitHub Team to an Entity * * @public */ export type TeamTransformer = ( item: GithubTeam, ctx: TransformerContext, -) => Promise; +) => Promise; /** * Default transformer for GitHub users to UserEntity diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 7a0a29e098..a0559ea070 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -444,7 +444,7 @@ describe('github', () => { customTeamTransformer, ); - expect(teams.groups).toHaveLength(1); + expect(teams.teams).toHaveLength(1); expect(teams).toEqual(output); }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 412b751c38..6b490f363a 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { GithubCredentialType } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { @@ -139,7 +139,7 @@ export async function getOrganizationUsers( org: string, tokenType: GithubCredentialType, userTransformer: UserTransformer = defaultUserTransformer, -): Promise<{ users: UserEntity[] }> { +): Promise<{ users: Entity[] }> { const query = ` query users($org: String!, $email: Boolean!, $cursor: String) { organization(login: $org) { @@ -188,7 +188,7 @@ export async function getOrganizationTeams( org: string, teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, ): Promise<{ - groups: GroupEntity[]; + teams: Entity[]; }> { const query = ` query teams($org: String!, $cursor: String) { @@ -222,7 +222,7 @@ export async function getOrganizationTeams( const materialisedTeams = async ( item: GithubTeamResponse, ctx: TransformerContext, - ): Promise => { + ): Promise => { const memberNames: GithubUser[] = []; if (!item.members.pageInfo.hasNextPage) { @@ -247,7 +247,7 @@ export async function getOrganizationTeams( return await teamTransformer(team, ctx); }; - const groups = await queryWithPaging( + const teams = await queryWithPaging( client, query, org, @@ -256,7 +256,7 @@ export async function getOrganizationTeams( { org }, ); - return { groups }; + return { teams }; } export async function getOrganizationTeamsFromUsers( @@ -265,7 +265,7 @@ export async function getOrganizationTeamsFromUsers( userLogins: string[], teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, ): Promise<{ - groups: GroupEntity[]; + teams: Entity[]; }> { const query = ` query teams($org: String!, $cursor: String, $userLogins: [String!] = "") { @@ -306,7 +306,7 @@ export async function getOrganizationTeamsFromUsers( const materialisedTeams = async ( item: GithubTeamResponse, ctx: TransformerContext, - ): Promise => { + ): Promise => { const memberNames: GithubUser[] = []; if (!item.members.pageInfo.hasNextPage) { @@ -331,7 +331,7 @@ export async function getOrganizationTeamsFromUsers( return await teamTransformer(team, ctx); }; - const groups = await queryWithPaging( + const teams = await queryWithPaging( client, query, org, @@ -340,7 +340,7 @@ export async function getOrganizationTeamsFromUsers( { org, userLogins }, ); - return { groups }; + return { teams }; } export async function getOrganizationsFromUser( @@ -377,7 +377,7 @@ export async function getOrganizationTeam( teamSlug: string, teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer, ): Promise<{ - group: GroupEntity; + team: Entity; }> { const query = ` query teams($org: String!, $teamSlug: String!) { @@ -401,7 +401,7 @@ export async function getOrganizationTeam( const materialisedTeam = async ( item: GithubTeamResponse, ctx: TransformerContext, - ): Promise => { + ): Promise => { const memberNames: GithubUser[] = []; if (!item.members.pageInfo.hasNextPage) { @@ -432,17 +432,17 @@ export async function getOrganizationTeam( }); if (!response.organization?.team) - throw new Error(`Found no match for group ${teamSlug}`); + throw new Error(`Found no match for team ${teamSlug}`); - const group = await materialisedTeam(response.organization?.team, { + const team = await materialisedTeam(response.organization?.team, { query, client, org, }); - if (!group) throw new Error(`Can't transform for group ${teamSlug}`); + if (!team) throw new Error(`Can't transform for team ${teamSlug}`); - return { group }; + return { team }; } export async function getOrganizationRepositories( diff --git a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts index cf5e484fc2..8d19f9fca6 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts @@ -16,9 +16,11 @@ import { DEFAULT_NAMESPACE, - GroupEntity, + Entity, + areGroupEntities, + areUserEntities, + isGroupEntity, stringifyEntityRef, - UserEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -147,7 +149,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { client, orgConfig.name, tokenType, - async (githubUser, ctx): Promise => { + async (githubUser, ctx): Promise => { const result = this.options.userTransformer ? await this.options.userTransformer(githubUser, ctx) : await defaultUserTransformer(githubUser, ctx); @@ -160,15 +162,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { }, ); - const { groups } = await getOrganizationTeams( + const { teams } = await getOrganizationTeams( client, orgConfig.name, - async (team, ctx): Promise => { + async (team, ctx): Promise => { const result = this.options.teamTransformer ? await this.options.teamTransformer(team, ctx) : await defaultOrganizationTeamTransformer(team, ctx); - if (result) { + if (result && isGroupEntity(result)) { result.metadata.namespace = orgConfig.groupNamespace; // Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here result.spec.members = team.members.map( @@ -185,7 +187,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( - `Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`, + `Read ${users.length} GitHub users and ${teams.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`, ); // Grab current users from `allUsersMap` if they already exist in our @@ -199,11 +201,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { return allUsersMap.get(userRef); }); - assignGroupsToUsers(pendingUsers, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(pendingUsers)) { + assignGroupsToUsers(pendingUsers, teams); + } + } - for (const group of groups) { - emit(processingResult.entity(location, group)); + for (const team of teams) { + emit(processingResult.entity(location, team)); } } catch (e) { this.logger.error( diff --git a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts index 654deb3079..295cf5ecd5 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts @@ -37,6 +37,7 @@ import { getOrganizationUsers, parseGithubOrgUrl, } from '../lib'; +import { areGroupEntities, areUserEntities } from '@backstage/catalog-model'; type GraphQL = typeof graphql; @@ -101,19 +102,23 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { this.logger.info('Reading GitHub users and groups'); const { users } = await getOrganizationUsers(client, org, tokenType); - const { groups } = await getOrganizationTeams(client, org); + const { teams } = await getOrganizationTeams(client, org); const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( - `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`, + `Read ${users.length} GitHub users and ${teams.length} GitHub teams in ${duration} seconds`, ); - assignGroupsToUsers(users, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(users)) { + assignGroupsToUsers(users, teams); + } + } // Done! - for (const group of groups) { - emit(processingResult.entity(location, group)); + for (const team of teams) { + emit(processingResult.entity(location, team)); } for (const user of users) { emit(processingResult.entity(location, user)); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index 4d343493f6..901e7f5185 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -18,9 +18,11 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, + areGroupEntities, + areUserEntities, DEFAULT_NAMESPACE, Entity, - GroupEntity, + isGroupEntity, parseEntityRef, stringifyEntityRef, UserEntity, @@ -240,7 +242,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { const { markReadComplete } = trackProgress(logger); const allUsersMap = new Map(); - const allGroups: Entity[] = []; + const allTeams: Entity[] = []; const orgsToProcess = this.options.orgs?.length ? this.options.orgs @@ -266,7 +268,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { this.options.userTransformer, ); - const { groups } = await getOrganizationTeams( + const { teams } = await getOrganizationTeams( client, org, this.defaultMultiOrgTeamTransformer.bind(this), @@ -283,10 +285,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { return allUsersMap.get(userRef); }); - assignGroupsToUsers(pendingUsers, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(pendingUsers)) { + assignGroupsToUsers(pendingUsers, teams); + } + } - allGroups.push(...groups); + allTeams.push(...teams); } catch (e) { logger.error(`Failed to read GitHub org data for ${org}: ${e}`); } @@ -294,11 +300,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { const allUsers = Array.from(allUsersMap.values()); - const { markCommitComplete } = markReadComplete({ allUsers, allGroups }); + const { markCommitComplete } = markReadComplete({ allUsers, allTeams }); await this.connection.applyMutation({ type: 'full', - entities: [...allUsers, ...allGroups].map(entity => ({ + entities: [...allUsers, ...allTeams].map(entity => ({ locationKey: `github-multi-org-provider:${this.options.id}`, entity: withLocations( `https://${this.options.gitHubConfig.host}`, @@ -422,7 +428,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { this.options.userTransformer, ); - const { groups } = await getOrganizationTeams( + const { teams } = await getOrganizationTeams( client, org, this.defaultMultiOrgTeamTransformer.bind(this), @@ -440,7 +446,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { headers: orgHeaders, }); - const { groups: userGroups } = await getOrganizationTeamsFromUsers( + const { teams: userTeams } = await getOrganizationTeamsFromUsers( orgClient, userOrg, users.map( @@ -451,12 +457,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { this.defaultMultiOrgTeamTransformer.bind(this), ); - assignGroupsToUsers(users, userGroups); + if (areGroupEntities(userTeams) && areUserEntities(users)) { + assignGroupsToUsers(users, userTeams); + } } const { added, removed } = this.createAddEntitiesOperation([ ...users, - ...groups, + ...teams, ]); await this.connection.applyMutation({ type: 'delta', @@ -536,14 +544,16 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { headers: orgHeaders, }); - const { groups } = await getOrganizationTeamsFromUsers( + const { teams } = await getOrganizationTeamsFromUsers( orgClient, userOrg, [login], this.defaultMultiOrgTeamTransformer.bind(this), ); - assignGroupsToUsers([user], groups); + if (areGroupEntities(teams)) { + assignGroupsToUsers([user], teams); + } } } @@ -623,7 +633,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { }); const teamSlug = event.team.slug; - const { group } = await getOrganizationTeam( + const { team } = await getOrganizationTeam( client, org, teamSlug, @@ -637,10 +647,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { this.options.userTransformer, ); - const usersFromChangedGroup = - group.spec.members?.map(m => - stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })), - ) || []; + const usersFromChangedGroup = isGroupEntity(team) + ? team.spec.members?.map(m => + stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })), + ) || [] + : []; const usersToRebuild = users.filter(u => usersFromChangedGroup.includes(stringifyEntityRef(u)), ); @@ -656,7 +667,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { headers: orgHeaders, }); - const { groups } = await getOrganizationTeamsFromUsers( + const { teams } = await getOrganizationTeamsFromUsers( orgClient, userOrg, usersToRebuild.map( @@ -667,7 +678,9 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { this.defaultMultiOrgTeamTransformer.bind(this), ); - assignGroupsToUsers(usersToRebuild, groups); + if (areGroupEntities(teams) && areUserEntities(usersToRebuild)) { + assignGroupsToUsers(usersToRebuild, teams); + } } const oldName = event.changes.name?.from || ''; @@ -693,7 +706,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { const { removed } = this.createRemoveEntitiesOperation([oldGroup]); const { added } = this.createAddEntitiesOperation([ ...usersToRebuild, - group, + team, ]); await this.connection.applyMutation({ type: 'delta', @@ -729,7 +742,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { }); const teamSlug = event.team.slug; - const { group } = await getOrganizationTeam( + const { team } = await getOrganizationTeam( client, org, teamSlug, @@ -765,17 +778,19 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { headers: orgHeaders, }); - const { groups } = await getOrganizationTeamsFromUsers( + const { teams } = await getOrganizationTeamsFromUsers( orgClient, userOrg, [login], this.defaultMultiOrgTeamTransformer.bind(this), ); - assignGroupsToUsers([user], groups); + if (areGroupEntities(teams)) { + assignGroupsToUsers([user], teams); + } } - const { added, removed } = this.createAddEntitiesOperation([user, group]); + const { added, removed } = this.createAddEntitiesOperation([user, team]); await this.connection.applyMutation({ type: 'delta', removed, @@ -812,14 +827,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider { private async defaultMultiOrgTeamTransformer( team: GithubTeam, ctx: TransformerContext, - ): Promise { + ): Promise { if (this.options.teamTransformer) { return await this.options.teamTransformer(team, ctx); } const result = await defaultOrganizationTeamTransformer(team, ctx); - if (result) { + if (result && result.spec) { result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US'); // Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here result.spec.members = team.members.map( @@ -885,9 +900,9 @@ function trackProgress(logger: Logger) { function markReadComplete(read: { allUsers: unknown[]; - allGroups: unknown[]; + allTeams: unknown[]; }) { - summary = `${read.allUsers.length} GitHub users and ${read.allGroups.length} GitHub groups`; + summary = `${read.allUsers.length} GitHub users and ${read.allTeams.length} GitHub groups`; const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); timestamp = Date.now(); logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index bf82607c6b..1381e7689d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -19,6 +19,9 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, + areGroupEntities, + areUserEntities, + isGroupEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { @@ -223,20 +226,24 @@ export class GithubOrgEntityProvider tokenType, this.options.userTransformer, ); - const { groups } = await getOrganizationTeams( + const { teams } = await getOrganizationTeams( client, org, this.options.teamTransformer, ); - assignGroupsToUsers(users, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(users)) { + assignGroupsToUsers(users, teams); + } + } - const { markCommitComplete } = markReadComplete({ users, groups }); + const { markCommitComplete } = markReadComplete({ users, teams }); await this.connection.applyMutation({ type: 'full', - entities: [...users, ...groups].map(entity => ({ + entities: [...users, ...teams].map(entity => ({ locationKey: `github-org-provider:${this.options.id}`, entity: withLocations( `https://${this.options.gitHubConfig.host}`, @@ -340,7 +347,7 @@ export class GithubOrgEntityProvider }); const { org } = parseGithubOrgUrl(this.options.orgUrl); - const { group } = await getOrganizationTeam( + const { team } = await getOrganizationTeam( client, org, teamSlug, @@ -354,20 +361,28 @@ export class GithubOrgEntityProvider this.options.userTransformer, ); - const usersFromChangedGroup = group.spec.members || []; + if (!isGroupEntity(team)) { + return; + } + + const usersFromChangedGroup = team.spec.members || []; const usersToRebuild = users.filter(u => usersFromChangedGroup.includes(u.metadata.name), ); - const { groups } = await getOrganizationTeamsFromUsers( + const { teams } = await getOrganizationTeamsFromUsers( client, org, usersToRebuild.map(u => u.metadata.name), this.options.teamTransformer, ); - assignGroupsToUsers(usersToRebuild, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(usersToRebuild)) { + assignGroupsToUsers(usersToRebuild, teams); + } + } const oldName = event.changes.name?.from || event.team.name; const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-'); @@ -380,14 +395,14 @@ export class GithubOrgEntityProvider const { removed } = createDeltaOperation(org, [ { - ...group, + ...team, metadata: { name: oldSlug, description: oldDescriptionSlug, }, }, ]); - const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]); + const { added } = createDeltaOperation(org, [...usersToRebuild, ...teams]); await this.connection.applyMutation({ type: 'delta', removed, @@ -423,7 +438,7 @@ export class GithubOrgEntityProvider }); const { org } = parseGithubOrgUrl(this.options.orgUrl); - const { group } = await getOrganizationTeam( + const { team } = await getOrganizationTeam( client, org, teamSlug, @@ -439,7 +454,7 @@ export class GithubOrgEntityProvider const usersToRebuild = users.filter(u => u.metadata.name === userLogin); - const { groups } = await getOrganizationTeamsFromUsers( + const { teams } = await getOrganizationTeamsFromUsers( client, org, [userLogin], @@ -447,16 +462,20 @@ export class GithubOrgEntityProvider ); // we include group because the removed event need to update the old group too - if (!groups.some(g => g.metadata.name === group.metadata.name)) { - groups.push(group); + if (!teams.some(t => t.metadata.name === team.metadata.name)) { + teams.push(team); } - assignGroupsToUsers(usersToRebuild, groups); - buildOrgHierarchy(groups); + if (areGroupEntities(teams)) { + buildOrgHierarchy(teams); + if (areUserEntities(usersToRebuild)) { + assignGroupsToUsers(usersToRebuild, teams); + } + } const { added, removed } = createDeltaOperation(org, [ ...usersToRebuild, - ...groups, + ...teams, ]); await this.connection.applyMutation({ type: 'delta', @@ -588,10 +607,10 @@ function trackProgress(logger: Logger) { let timestamp = Date.now(); let summary: string; - logger.info('Reading GitHub users and groups'); + logger.info('Reading GitHub users and teams'); - function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { - summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`; + function markReadComplete(read: { users: unknown[]; teams: unknown[] }) { + summary = `${read.users.length} GitHub users and ${read.teams.length} GitHub teams`; const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); timestamp = Date.now(); logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); From 3c44761b919109359942cdd6823c888aef49b278 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 20 Jul 2023 16:15:22 +1000 Subject: [PATCH 2/6] changeset Signed-off-by: Jack Grigg --- .changeset/young-ants-nail.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/young-ants-nail.md diff --git a/.changeset/young-ants-nail.md b/.changeset/young-ants-nail.md new file mode 100644 index 0000000000..68c0c3f860 --- /dev/null +++ b/.changeset/young-ants-nail.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-module-github': patch +'@backstage/catalog-model': patch +--- + +Allow github user and team transforms to return any Entity From e64c9470f0a588ebcc5072d4dbde6c44e87e6e81 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 10 Aug 2023 10:53:46 +1000 Subject: [PATCH 3/6] api reports Signed-off-by: Jack Grigg --- packages/catalog-model/api-report.md | 10 ++++++++++ plugins/catalog-backend-module-github/api-report.md | 6 ++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index b8eb20c590..5eeb6d2f31 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -53,6 +53,16 @@ export { ApiEntityV1alpha1 }; // @public export const apiEntityV1alpha1Validator: KindValidator; +// @public (undocumented) +export function areGroupEntities( + entities: Entity[], +): entities is GroupEntityV1alpha1[]; + +// @public (undocumented) +export function areUserEntities( + entities: Entity[], +): entities is UserEntityV1alpha1[]; + // @public export class CommonValidatorFunctions { static isJsonSafe(value: unknown): boolean; diff --git a/plugins/catalog-backend-module-github/api-report.md b/plugins/catalog-backend-module-github/api-report.md index 7b944b33a9..12ab6d35e7 100644 --- a/plugins/catalog-backend-module-github/api-report.md +++ b/plugins/catalog-backend-module-github/api-report.md @@ -16,7 +16,6 @@ import { EventSubscriber } from '@backstage/plugin-events-node'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GithubIntegrationConfig } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; -import { GroupEntity } from '@backstage/catalog-model'; import { LocationSpec } from '@backstage/plugin-catalog-node'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -25,7 +24,6 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend'; import { TaskRunner } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; -import { UserEntity } from '@backstage/catalog-model'; // @public export const defaultOrganizationTeamTransformer: TeamTransformer; @@ -303,7 +301,7 @@ export type GithubUser = { export type TeamTransformer = ( item: GithubTeam, ctx: TransformerContext, -) => Promise; +) => Promise; // @public export interface TransformerContext { @@ -319,5 +317,5 @@ export interface TransformerContext { export type UserTransformer = ( item: GithubUser, ctx: TransformerContext, -) => Promise; +) => Promise; ``` From 040eaa361b913c10b733412509d41abc1fac5e35 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 18 Aug 2023 21:19:01 +1000 Subject: [PATCH 4/6] fix tests Signed-off-by: Jack Grigg --- .../catalog-backend-module-github/src/lib/github.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index a0559ea070..d83e8031cc 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -229,7 +229,7 @@ describe('github', () => { it('reads teams', async () => { const output = { - groups: [ + teams: [ expect.objectContaining({ metadata: expect.objectContaining({ name: 'team', @@ -329,7 +329,7 @@ describe('github', () => { it('reads teams', async () => { const output = { - groups: [ + teams: [ expect.objectContaining({ metadata: expect.objectContaining({ name: 'Team-custom', @@ -409,7 +409,7 @@ describe('github', () => { }; const output = { - groups: [ + teams: [ expect.objectContaining({ metadata: expect.objectContaining({ name: 'Team-custom', From 40986999bed720d7e46ca2eedb8f41f55053f9af Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 24 Aug 2023 13:22:18 +1000 Subject: [PATCH 5/6] remove guards from api Signed-off-by: Jack Grigg --- packages/catalog-model/api-report.md | 10 ------ .../catalog-model/src/entity/conditions.ts | 14 -------- .../src/lib/guards.ts | 32 +++++++++++++++++++ .../GithubMultiOrgReaderProcessor.ts | 3 +- .../processors/GithubOrgReaderProcessor.ts | 2 +- .../providers/GithubMultiOrgEntityProvider.ts | 3 +- .../src/providers/GithubOrgEntityProvider.ts | 3 +- 7 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 plugins/catalog-backend-module-github/src/lib/guards.ts diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 5eeb6d2f31..b8eb20c590 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -53,16 +53,6 @@ export { ApiEntityV1alpha1 }; // @public export const apiEntityV1alpha1Validator: KindValidator; -// @public (undocumented) -export function areGroupEntities( - entities: Entity[], -): entities is GroupEntityV1alpha1[]; - -// @public (undocumented) -export function areUserEntities( - entities: Entity[], -): entities is UserEntityV1alpha1[]; - // @public export class CommonValidatorFunctions { static isJsonSafe(value: unknown): boolean; diff --git a/packages/catalog-model/src/entity/conditions.ts b/packages/catalog-model/src/entity/conditions.ts index 7e798ccb31..d761695b5e 100644 --- a/packages/catalog-model/src/entity/conditions.ts +++ b/packages/catalog-model/src/entity/conditions.ts @@ -50,14 +50,6 @@ export function isDomainEntity(entity: Entity): entity is DomainEntity { export function isGroupEntity(entity: Entity): entity is GroupEntity { return entity.kind.toLocaleUpperCase('en-US') === 'GROUP'; } -/** - * @public - */ -export function areGroupEntities( - entities: Entity[], -): entities is GroupEntity[] { - return entities.every(e => isGroupEntity(e)); -} /** * @public */ @@ -82,9 +74,3 @@ export function isSystemEntity(entity: Entity): entity is SystemEntity { export function isUserEntity(entity: Entity): entity is UserEntity { return entity.kind.toLocaleUpperCase('en-US') === 'USER'; } -/** - * @public - */ -export function areUserEntities(entities: Entity[]): entities is UserEntity[] { - return entities.every(e => isUserEntity(e)); -} diff --git a/plugins/catalog-backend-module-github/src/lib/guards.ts b/plugins/catalog-backend-module-github/src/lib/guards.ts new file mode 100644 index 0000000000..f092d2365a --- /dev/null +++ b/plugins/catalog-backend-module-github/src/lib/guards.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Entity, + GroupEntity, + UserEntity, + isGroupEntity, + isUserEntity, +} from '@backstage/catalog-model'; + +export function areGroupEntities( + entities: Entity[], +): entities is GroupEntity[] { + return entities.every(e => isGroupEntity(e)); +} + +export function areUserEntities(entities: Entity[]): entities is UserEntity[] { + return entities.every(e => isUserEntity(e)); +} diff --git a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts index 8d19f9fca6..8bfbd20e2e 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts @@ -17,8 +17,6 @@ import { DEFAULT_NAMESPACE, Entity, - areGroupEntities, - areUserEntities, isGroupEntity, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -51,6 +49,7 @@ import { TeamTransformer, UserTransformer, } from '../lib'; +import { areGroupEntities, areUserEntities } from '../lib/guards'; /** * Extracts teams and users out of a multiple GitHub orgs namespaced per org. diff --git a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts index 295cf5ecd5..63a273378b 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts @@ -37,7 +37,7 @@ import { getOrganizationUsers, parseGithubOrgUrl, } from '../lib'; -import { areGroupEntities, areUserEntities } from '@backstage/catalog-model'; +import { areGroupEntities, areUserEntities } from '../lib/guards'; type GraphQL = typeof graphql; diff --git a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts index 901e7f5185..65a622890d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts @@ -18,8 +18,6 @@ import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, - areGroupEntities, - areUserEntities, DEFAULT_NAMESPACE, Entity, isGroupEntity, @@ -80,6 +78,7 @@ import { getOrganizationTeamsFromUsers, } from '../lib/github'; import { splitTeamSlug } from '../lib/util'; +import { areGroupEntities, areUserEntities } from '../lib/guards'; /** * Options for {@link GithubMultiOrgEntityProvider}. diff --git a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts index 1381e7689d..22f1f12e8d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts @@ -19,8 +19,6 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - areGroupEntities, - areUserEntities, isGroupEntity, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -71,6 +69,7 @@ import { } from '../lib/github'; import { assignGroupsToUsers, buildOrgHierarchy } from '../lib/org'; import { parseGithubOrgUrl, splitTeamSlug } from '../lib/util'; +import { areGroupEntities, areUserEntities } from '../lib/guards'; /** * Options for {@link GithubOrgEntityProvider}. From 505d07cf18b0c19c822c6b8395b91f0c2520eacb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 25 Aug 2023 12:50:38 +1000 Subject: [PATCH 6/6] fix changeset Signed-off-by: Jack Grigg --- .changeset/young-ants-nail.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.changeset/young-ants-nail.md b/.changeset/young-ants-nail.md index 68c0c3f860..6648098687 100644 --- a/.changeset/young-ants-nail.md +++ b/.changeset/young-ants-nail.md @@ -1,6 +1,5 @@ --- '@backstage/plugin-catalog-backend-module-github': patch -'@backstage/catalog-model': patch --- Allow github user and team transforms to return any Entity