paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 47 Packages Projects Releases Wiki Activity

relax github transform types

Browse Source 
Signed-off-by: Jack Grigg <jrgrigg@gmail.com>
This commit is contained in:
Jack Grigg
2023-07-20 16:09:59 +10:00
parent 083a4022e5
commit b420b80124
8 changed files with 150 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/catalog-model/src/entity/conditions.ts
+14
View File
@@ -50,6 +50,14 @@ export function isDomainEntity(entity: Entity): entity is DomainEntity {
export function isGroupEntity(entity: Entity): entity is GroupEntity {
return entity.kind.toLocaleUpperCase('en-US') === 'GROUP';
}
/**
* @public
*/
export function areGroupEntities(
entities: Entity[],
): entities is GroupEntity[] {
return entities.every(e => isGroupEntity(e));
}
/**
* @public
*/
@@ -74,3 +82,9 @@ export function isSystemEntity(entity: Entity): entity is SystemEntity {
export function isUserEntity(entity: Entity): entity is UserEntity {
return entity.kind.toLocaleUpperCase('en-US') === 'USER';
}
/**
* @public
*/
export function areUserEntities(entities: Entity[]): entities is UserEntity[] {
return entities.every(e => isUserEntity(e));
}
plugins/catalog-backend-module-github/src/lib/defaultTransformers.ts
+5 -5
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model';
import { graphql } from '@octokit/graphql';
import {
ANNOTATION_GITHUB_TEAM_SLUG,
@@ -34,24 +34,24 @@ export interface TransformerContext {
}
/**
* Transformer for GitHub users to UserEntity
* Transformer for GitHub users to an Entity
*
* @public
*/
export type UserTransformer = (
item: GithubUser,
ctx: TransformerContext,
) => Promise<UserEntity | undefined>;
) => Promise<Entity | undefined>;
/**
* Transformer for GitHub Team to GroupEntity
* Transformer for GitHub Team to an Entity
*
* @public
*/
export type TeamTransformer = (
item: GithubTeam,
ctx: TransformerContext,
) => Promise<GroupEntity | undefined>;
) => Promise<Entity | undefined>;
/**
* Default transformer for GitHub users to UserEntity
plugins/catalog-backend-module-github/src/lib/github.test.ts
+1 -1
View File
@@ -444,7 +444,7 @@ describe('github', () => {
customTeamTransformer,
);
expect(teams.groups).toHaveLength(1);
expect(teams.teams).toHaveLength(1);
expect(teams).toEqual(output);
});
});
plugins/catalog-backend-module-github/src/lib/github.ts
+16 -16
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { Entity, GroupEntity, UserEntity } from '@backstage/catalog-model';
import { Entity } from '@backstage/catalog-model';
import { GithubCredentialType } from '@backstage/integration';
import { graphql } from '@octokit/graphql';
import {
@@ -139,7 +139,7 @@ export async function getOrganizationUsers(
org: string,
tokenType: GithubCredentialType,
userTransformer: UserTransformer = defaultUserTransformer,
): Promise<{ users: UserEntity[] }> {
): Promise<{ users: Entity[] }> {
const query = `
query users($org: String!, $email: Boolean!, $cursor: String) {
organization(login: $org) {
@@ -188,7 +188,7 @@ export async function getOrganizationTeams(
org: string,
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
): Promise<{
groups: GroupEntity[];
teams: Entity[];
}> {
const query = `
query teams($org: String!, $cursor: String) {
@@ -222,7 +222,7 @@ export async function getOrganizationTeams(
const materialisedTeams = async (
item: GithubTeamResponse,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> => {
): Promise<Entity | undefined> => {
const memberNames: GithubUser[] = [];
if (!item.members.pageInfo.hasNextPage) {
@@ -247,7 +247,7 @@ export async function getOrganizationTeams(
return await teamTransformer(team, ctx);
};
const groups = await queryWithPaging(
const teams = await queryWithPaging(
client,
query,
org,
@@ -256,7 +256,7 @@ export async function getOrganizationTeams(
{ org },
);
return { groups };
return { teams };
}
export async function getOrganizationTeamsFromUsers(
@@ -265,7 +265,7 @@ export async function getOrganizationTeamsFromUsers(
userLogins: string[],
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
): Promise<{
groups: GroupEntity[];
teams: Entity[];
}> {
const query = `
query teams($org: String!, $cursor: String, $userLogins: [String!] = "") {
@@ -306,7 +306,7 @@ export async function getOrganizationTeamsFromUsers(
const materialisedTeams = async (
item: GithubTeamResponse,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> => {
): Promise<Entity | undefined> => {
const memberNames: GithubUser[] = [];
if (!item.members.pageInfo.hasNextPage) {
@@ -331,7 +331,7 @@ export async function getOrganizationTeamsFromUsers(
return await teamTransformer(team, ctx);
};
const groups = await queryWithPaging(
const teams = await queryWithPaging(
client,
query,
org,
@@ -340,7 +340,7 @@ export async function getOrganizationTeamsFromUsers(
{ org, userLogins },
);
return { groups };
return { teams };
}
export async function getOrganizationsFromUser(
@@ -377,7 +377,7 @@ export async function getOrganizationTeam(
teamSlug: string,
teamTransformer: TeamTransformer = defaultOrganizationTeamTransformer,
): Promise<{
group: GroupEntity;
team: Entity;
}> {
const query = `
query teams($org: String!, $teamSlug: String!) {
@@ -401,7 +401,7 @@ export async function getOrganizationTeam(
const materialisedTeam = async (
item: GithubTeamResponse,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> => {
): Promise<Entity | undefined> => {
const memberNames: GithubUser[] = [];
if (!item.members.pageInfo.hasNextPage) {
@@ -432,17 +432,17 @@ export async function getOrganizationTeam(
});
if (!response.organization?.team)
throw new Error(`Found no match for group ${teamSlug}`);
throw new Error(`Found no match for team ${teamSlug}`);
const group = await materialisedTeam(response.organization?.team, {
const team = await materialisedTeam(response.organization?.team, {
query,
client,
org,
});
if (!group) throw new Error(`Can't transform for group ${teamSlug}`);
if (!team) throw new Error(`Can't transform for team ${teamSlug}`);
return { group };
return { team };
}
export async function getOrganizationRepositories(
plugins/catalog-backend-module-github/src/processors/GithubMultiOrgReaderProcessor.ts
+17 -11
View File
@@ -16,9 +16,11 @@
import {
DEFAULT_NAMESPACE,
GroupEntity,
Entity,
areGroupEntities,
areUserEntities,
isGroupEntity,
stringifyEntityRef,
UserEntity,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
@@ -147,7 +149,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
client,
orgConfig.name,
tokenType,
async (githubUser, ctx): Promise<UserEntity | undefined> => {
async (githubUser, ctx): Promise<Entity | undefined> => {
const result = this.options.userTransformer
? await this.options.userTransformer(githubUser, ctx)
: await defaultUserTransformer(githubUser, ctx);
@@ -160,15 +162,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
},
);
const { groups } = await getOrganizationTeams(
const { teams } = await getOrganizationTeams(
client,
orgConfig.name,
async (team, ctx): Promise<GroupEntity | undefined> => {
async (team, ctx): Promise<Entity | undefined> => {
const result = this.options.teamTransformer
? await this.options.teamTransformer(team, ctx)
: await defaultOrganizationTeamTransformer(team, ctx);
if (result) {
if (result && isGroupEntity(result)) {
result.metadata.namespace = orgConfig.groupNamespace;
// Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here
result.spec.members = team.members.map(
@@ -185,7 +187,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.debug(
`Read ${users.length} GitHub users and ${groups.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`,
`Read ${users.length} GitHub users and ${teams.length} GitHub teams from ${orgConfig.name} in ${duration} seconds`,
);
// Grab current users from `allUsersMap` if they already exist in our
@@ -199,11 +201,15 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor {
return allUsersMap.get(userRef);
});
assignGroupsToUsers(pendingUsers, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(pendingUsers)) {
assignGroupsToUsers(pendingUsers, teams);
}
}
for (const group of groups) {
emit(processingResult.entity(location, group));
for (const team of teams) {
emit(processingResult.entity(location, team));
}
} catch (e) {
this.logger.error(
plugins/catalog-backend-module-github/src/processors/GithubOrgReaderProcessor.ts
+11 -6
View File
@@ -37,6 +37,7 @@ import {
getOrganizationUsers,
parseGithubOrgUrl,
} from '../lib';
import { areGroupEntities, areUserEntities } from '@backstage/catalog-model';
type GraphQL = typeof graphql;
@@ -101,19 +102,23 @@ export class GithubOrgReaderProcessor implements CatalogProcessor {
this.logger.info('Reading GitHub users and groups');
const { users } = await getOrganizationUsers(client, org, tokenType);
const { groups } = await getOrganizationTeams(client, org);
const { teams } = await getOrganizationTeams(client, org);
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.debug(
`Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
`Read ${users.length} GitHub users and ${teams.length} GitHub teams in ${duration} seconds`,
);
assignGroupsToUsers(users, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(users)) {
assignGroupsToUsers(users, teams);
}
}
// Done!
for (const group of groups) {
emit(processingResult.entity(location, group));
for (const team of teams) {
emit(processingResult.entity(location, team));
}
for (const user of users) {
emit(processingResult.entity(location, user));
plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts
+45 -30
View File
@@ -18,9 +18,11 @@ import { TaskRunner } from '@backstage/backend-tasks';
import {
ANNOTATION_LOCATION,
ANNOTATION_ORIGIN_LOCATION,
areGroupEntities,
areUserEntities,
DEFAULT_NAMESPACE,
Entity,
GroupEntity,
isGroupEntity,
parseEntityRef,
stringifyEntityRef,
UserEntity,
@@ -240,7 +242,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
const { markReadComplete } = trackProgress(logger);
const allUsersMap = new Map();
const allGroups: Entity[] = [];
const allTeams: Entity[] = [];
const orgsToProcess = this.options.orgs?.length
? this.options.orgs
@@ -266,7 +268,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
this.options.userTransformer,
);
const { groups } = await getOrganizationTeams(
const { teams } = await getOrganizationTeams(
client,
org,
this.defaultMultiOrgTeamTransformer.bind(this),
@@ -283,10 +285,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
return allUsersMap.get(userRef);
});
assignGroupsToUsers(pendingUsers, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(pendingUsers)) {
assignGroupsToUsers(pendingUsers, teams);
}
}
allGroups.push(...groups);
allTeams.push(...teams);
} catch (e) {
logger.error(`Failed to read GitHub org data for ${org}: ${e}`);
}
@@ -294,11 +300,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
const allUsers = Array.from(allUsersMap.values());
const { markCommitComplete } = markReadComplete({ allUsers, allGroups });
const { markCommitComplete } = markReadComplete({ allUsers, allTeams });
await this.connection.applyMutation({
type: 'full',
entities: [...allUsers, ...allGroups].map(entity => ({
entities: [...allUsers, ...allTeams].map(entity => ({
locationKey: `github-multi-org-provider:${this.options.id}`,
entity: withLocations(
`https://${this.options.gitHubConfig.host}`,
@@ -422,7 +428,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
this.options.userTransformer,
);
const { groups } = await getOrganizationTeams(
const { teams } = await getOrganizationTeams(
client,
org,
this.defaultMultiOrgTeamTransformer.bind(this),
@@ -440,7 +446,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers: orgHeaders,
});
const { groups: userGroups } = await getOrganizationTeamsFromUsers(
const { teams: userTeams } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
users.map(
@@ -451,12 +457,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers(users, userGroups);
if (areGroupEntities(userTeams) && areUserEntities(users)) {
assignGroupsToUsers(users, userTeams);
}
}
const { added, removed } = this.createAddEntitiesOperation([
...users,
...groups,
...teams,
]);
await this.connection.applyMutation({
type: 'delta',
@@ -536,14 +544,16 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
const { teams } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
[login],
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers([user], groups);
if (areGroupEntities(teams)) {
assignGroupsToUsers([user], teams);
}
}
}
@@ -623,7 +633,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
});
const teamSlug = event.team.slug;
const { group } = await getOrganizationTeam(
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
@@ -637,10 +647,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
this.options.userTransformer,
);
const usersFromChangedGroup =
group.spec.members?.map(m =>
stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })),
) || [];
const usersFromChangedGroup = isGroupEntity(team)
? team.spec.members?.map(m =>
stringifyEntityRef(parseEntityRef(m, { defaultKind: 'user' })),
) || []
: [];
const usersToRebuild = users.filter(u =>
usersFromChangedGroup.includes(stringifyEntityRef(u)),
);
@@ -656,7 +667,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
const { teams } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
usersToRebuild.map(
@@ -667,7 +678,9 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers(usersToRebuild, groups);
if (areGroupEntities(teams) && areUserEntities(usersToRebuild)) {
assignGroupsToUsers(usersToRebuild, teams);
}
}
const oldName = event.changes.name?.from || '';
@@ -693,7 +706,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
const { removed } = this.createRemoveEntitiesOperation([oldGroup]);
const { added } = this.createAddEntitiesOperation([
...usersToRebuild,
group,
team,
]);
await this.connection.applyMutation({
type: 'delta',
@@ -729,7 +742,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
});
const teamSlug = event.team.slug;
const { group } = await getOrganizationTeam(
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
@@ -765,17 +778,19 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
headers: orgHeaders,
});
const { groups } = await getOrganizationTeamsFromUsers(
const { teams } = await getOrganizationTeamsFromUsers(
orgClient,
userOrg,
[login],
this.defaultMultiOrgTeamTransformer.bind(this),
);
assignGroupsToUsers([user], groups);
if (areGroupEntities(teams)) {
assignGroupsToUsers([user], teams);
}
}
const { added, removed } = this.createAddEntitiesOperation([user, group]);
const { added, removed } = this.createAddEntitiesOperation([user, team]);
await this.connection.applyMutation({
type: 'delta',
removed,
@@ -812,14 +827,14 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
private async defaultMultiOrgTeamTransformer(
team: GithubTeam,
ctx: TransformerContext,
): Promise<GroupEntity | undefined> {
): Promise<Entity | undefined> {
if (this.options.teamTransformer) {
return await this.options.teamTransformer(team, ctx);
}
const result = await defaultOrganizationTeamTransformer(team, ctx);
if (result) {
if (result && result.spec) {
result.metadata.namespace = ctx.org.toLocaleLowerCase('en-US');
// Group `spec.members` inherits the namespace of it's group so need to explicitly specify refs here
result.spec.members = team.members.map(
@@ -885,9 +900,9 @@ function trackProgress(logger: Logger) {
function markReadComplete(read: {
allUsers: unknown[];
allGroups: unknown[];
allTeams: unknown[];
}) {
summary = `${read.allUsers.length} GitHub users and ${read.allGroups.length} GitHub groups`;
summary = `${read.allUsers.length} GitHub users and ${read.allTeams.length} GitHub groups`;
const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
timestamp = Date.now();
logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts
+41 -22
View File
@@ -19,6 +19,9 @@ import {
ANNOTATION_LOCATION,
ANNOTATION_ORIGIN_LOCATION,
Entity,
areGroupEntities,
areUserEntities,
isGroupEntity,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
@@ -223,20 +226,24 @@ export class GithubOrgEntityProvider
tokenType,
this.options.userTransformer,
);
const { groups } = await getOrganizationTeams(
const { teams } = await getOrganizationTeams(
client,
org,
this.options.teamTransformer,
);
assignGroupsToUsers(users, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(users)) {
assignGroupsToUsers(users, teams);
}
}
const { markCommitComplete } = markReadComplete({ users, groups });
const { markCommitComplete } = markReadComplete({ users, teams });
await this.connection.applyMutation({
type: 'full',
entities: [...users, ...groups].map(entity => ({
entities: [...users, ...teams].map(entity => ({
locationKey: `github-org-provider:${this.options.id}`,
entity: withLocations(
`https://${this.options.gitHubConfig.host}`,
@@ -340,7 +347,7 @@ export class GithubOrgEntityProvider
});
const { org } = parseGithubOrgUrl(this.options.orgUrl);
const { group } = await getOrganizationTeam(
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
@@ -354,20 +361,28 @@ export class GithubOrgEntityProvider
this.options.userTransformer,
);
const usersFromChangedGroup = group.spec.members || [];
if (!isGroupEntity(team)) {
return;
}
const usersFromChangedGroup = team.spec.members || [];
const usersToRebuild = users.filter(u =>
usersFromChangedGroup.includes(u.metadata.name),
);
const { groups } = await getOrganizationTeamsFromUsers(
const { teams } = await getOrganizationTeamsFromUsers(
client,
org,
usersToRebuild.map(u => u.metadata.name),
this.options.teamTransformer,
);
assignGroupsToUsers(usersToRebuild, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(usersToRebuild)) {
assignGroupsToUsers(usersToRebuild, teams);
}
}
const oldName = event.changes.name?.from || event.team.name;
const oldSlug = oldName.toLowerCase().replaceAll(/\s/gi, '-');
@@ -380,14 +395,14 @@ export class GithubOrgEntityProvider
const { removed } = createDeltaOperation(org, [
{
...group,
...team,
metadata: {
name: oldSlug,
description: oldDescriptionSlug,
},
},
]);
const { added } = createDeltaOperation(org, [...usersToRebuild, ...groups]);
const { added } = createDeltaOperation(org, [...usersToRebuild, ...teams]);
await this.connection.applyMutation({
type: 'delta',
removed,
@@ -423,7 +438,7 @@ export class GithubOrgEntityProvider
});
const { org } = parseGithubOrgUrl(this.options.orgUrl);
const { group } = await getOrganizationTeam(
const { team } = await getOrganizationTeam(
client,
org,
teamSlug,
@@ -439,7 +454,7 @@ export class GithubOrgEntityProvider
const usersToRebuild = users.filter(u => u.metadata.name === userLogin);
const { groups } = await getOrganizationTeamsFromUsers(
const { teams } = await getOrganizationTeamsFromUsers(
client,
org,
[userLogin],
@@ -447,16 +462,20 @@ export class GithubOrgEntityProvider
);
// we include group because the removed event need to update the old group too
if (!groups.some(g => g.metadata.name === group.metadata.name)) {
groups.push(group);
if (!teams.some(t => t.metadata.name === team.metadata.name)) {
teams.push(team);
}
assignGroupsToUsers(usersToRebuild, groups);
buildOrgHierarchy(groups);
if (areGroupEntities(teams)) {
buildOrgHierarchy(teams);
if (areUserEntities(usersToRebuild)) {
assignGroupsToUsers(usersToRebuild, teams);
}
}
const { added, removed } = createDeltaOperation(org, [
...usersToRebuild,
...groups,
...teams,
]);
await this.connection.applyMutation({
type: 'delta',
@@ -588,10 +607,10 @@ function trackProgress(logger: Logger) {
let timestamp = Date.now();
let summary: string;
logger.info('Reading GitHub users and groups');
logger.info('Reading GitHub users and teams');
function markReadComplete(read: { users: unknown[]; groups: unknown[] }) {
summary = `${read.users.length} GitHub users and ${read.groups.length} GitHub groups`;
function markReadComplete(read: { users: unknown[]; teams: unknown[] }) {
summary = `${read.users.length} GitHub users and ${read.teams.length} GitHub teams`;
const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
timestamp = Date.now();
logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);