diff --git a/app-config.yaml b/app-config.yaml index d46c601e33..bd8b7febd0 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -69,7 +69,7 @@ kubernetes: catalog: rules: - - allow: [Component, API, Group, Template, Location] + - allow: [Component, API, Group, User, Template, Location] processors: github: providers: @@ -89,6 +89,18 @@ catalog: # token: # $secret: # env: GHE_PRIVATE_TOKEN + githubOrg: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using the API: + # - target: https://ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: @@ -109,19 +121,15 @@ catalog: # Backstage example components - type: github target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml - # Example component for github-actions - type: github target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml - # Example component for techdocs - type: github target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml - # Backstage example APIs - type: github target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml - # Backstage example templates - type: github target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index fdac6dd938..219b04a0f7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -45,7 +45,7 @@ export default async function createPlugin({ useHotCleanup( module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000), + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), ); return await createRouter({ diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index 3e2f51b40a..eec60f7f4f 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -40,7 +40,10 @@ class AllEntityPolicies implements EntityPolicy { async enforce(entity: Entity): Promise { let result = entity; for (const policy of this.policies) { - result = await policy.enforce(entity); + const output = await policy.enforce(entity); + if (output) { + result = output; + } } return result; } @@ -51,12 +54,11 @@ class AllEntityPolicies implements EntityPolicy { class AnyEntityPolicy implements EntityPolicy { constructor(private readonly policies: EntityPolicy[]) {} - async enforce(entity: Entity): Promise { + async enforce(entity: Entity): Promise { for (const policy of this.policies) { - try { - return await policy.enforce(entity); - } catch { - continue; + const output = await policy.enforce(entity); + if (output !== null) { + return output; } } throw new Error(`The entity did not match any known policy`); @@ -98,7 +100,7 @@ export class EntityPolicies implements EntityPolicy { this.policy = policy; } - enforce(entity: Entity): Promise { + enforce(entity: Entity): Promise { return this.policy.enforce(entity); } } diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts index 35a1c59ef8..51b87734f9 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts @@ -49,7 +49,7 @@ paths: '200': description: A paged array of pets content: - application/json: + application/json: schema: $ref: "#/components/schemas/Pets" components: @@ -86,14 +86,14 @@ components: await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts index 972f6df96d..2936b98531 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts @@ -50,7 +50,13 @@ export class ApiEntityV1alpha1Policy implements EntityPolicy { }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts index e784934d48..a5651b6501 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts @@ -50,14 +50,14 @@ describe('ComponentV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 38b9c27720..3bd6b8250b 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -50,7 +50,13 @@ export class ComponentEntityV1alpha1Policy implements EntityPolicy { }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts index 94dcc2f14f..dcf62e07f5 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts @@ -53,14 +53,14 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { @@ -98,6 +98,11 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); + it('accepts no ancestors', async () => { + (entity as any).spec.ancestors = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + it('rejects missing children', async () => { delete (entity as any).spec.children; await expect(policy.enforce(entity)).rejects.toThrow(/children/); @@ -108,6 +113,11 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); + it('accepts no children', async () => { + (entity as any).spec.children = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + it('rejects missing descendants', async () => { delete (entity as any).spec.descendants; await expect(policy.enforce(entity)).rejects.toThrow(/descendants/); @@ -117,4 +127,9 @@ describe('GroupV1alpha1Policy', () => { (entity as any).spec.descendants = ['']; await expect(policy.enforce(entity)).resolves.toBe(entity); }); + + it('accepts no descendants', async () => { + (entity as any).spec.descendants = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); }); diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts index c1d39d5b44..9733a81a66 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts @@ -44,15 +44,35 @@ export class GroupEntityV1alpha1Policy implements EntityPolicy { .object({ type: yup.string().required().min(1), parent: yup.string().notRequired().min(1), - ancestors: yup.array(yup.string()).required(), - children: yup.array(yup.string()).required(), - descendants: yup.array(yup.string()).required(), + // Use these manual tests because yup .required() requires at least + // one element and there is no simple workaround -_- + ancestors: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'ancestors must be defined', + test: v => Boolean(v), + }), + children: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'children must be defined', + test: v => Boolean(v), + }), + descendants: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'descendants must be defined', + test: v => Boolean(v), + }), }) .required(), }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts index 62837ed500..aed22aca05 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts @@ -47,14 +47,14 @@ describe('LocationV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index a69a3a574a..f53fa9624c 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -48,7 +48,13 @@ export class LocationEntityV1alpha1Policy implements EntityPolicy { }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts index ce9ed8bedd..cd3a4cfb71 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts @@ -64,14 +64,14 @@ describe('TemplateEntityV1alpah1', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts index 8aa79d57db..1986638051 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts @@ -50,7 +50,13 @@ export class TemplateEntityV1alpha1Policy implements EntityPolicy { }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts index b641c6cad3..62f529ff7c 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts @@ -54,14 +54,14 @@ describe('UserV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('spec accepts unknown additional fields', async () => { @@ -147,4 +147,14 @@ describe('UserV1alpha1Policy', () => { (entity as any).spec.memberOf[0] = 7; await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/); }); + + it('accepts empty memberOf', async () => { + (entity as any).spec.memberOf = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + + it('rejects null memberOf', async () => { + (entity as any).spec.memberOf = null; + await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/); + }); }); diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts index a6a304f509..bfab45f202 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts @@ -50,13 +50,25 @@ export class UserEntityV1alpha1Policy implements EntityPolicy { picture: yup.string().min(1).notRequired(), }) .notRequired(), - memberOf: yup.array(yup.string()).required(), + // Use this manual test because yup .required() requires at least one + // element and there is no simple workaround -_- + memberOf: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'memberOf must be defined', + test: v => Boolean(v), + }), }) .required(), }); } - async enforce(envelope: Entity): Promise { + async enforce(envelope: Entity): Promise { + if ( + KIND !== envelope.kind || + !API_VERSION.includes(envelope.apiVersion as any) + ) { + return undefined; + } return await this.schema.validate(envelope, { strict: true }); } } diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 3aafdc241a..aa96e46102 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -27,10 +27,11 @@ export type EntityPolicy = { * Applies validation or mutation on an entity. * * @param entity The entity, as validated/mutated so far in the policy tree - * @returns The incoming entity, or a mutated version of the same + * @returns The incoming entity, or a mutated version of the same, or + * undefined if this processor could not handle the entity * @throws An error if the entity should be rejected */ - enforce(entity: Entity): Promise; + enforce(entity: Entity): Promise; }; export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue }; diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 17e9fa0728..f1278bd3c3 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -66,7 +66,7 @@ scaffolder: catalog: rules: - - allow: [Component, API, Group, Template, Location] + - allow: [Component, API, Group, User, Template, Location] processors: github: providers: diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 6e2aa096d3..53796ab94e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -23,6 +23,7 @@ "@backstage/backend-common": "^0.1.1-alpha.23", "@backstage/catalog-model": "^0.1.1-alpha.23", "@backstage/config": "^0.1.1-alpha.23", + "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index dd3f9eacda..f612094a2d 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -19,6 +19,7 @@ import { Entity, EntityPolicies, EntityPolicy, + ENTITY_DEFAULT_NAMESPACE, LocationSpec, } from '@backstage/catalog-model'; import { Config, ConfigReader } from '@backstage/config'; @@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; +import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; @@ -85,6 +87,7 @@ export class LocationReaders implements LocationReader { new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), new AzureApiReaderProcessor(config), + GithubOrgReaderProcessor.fromConfig(config), new UrlReaderProcessor(), new YamlProcessor(), PlaceholderProcessor.default(), @@ -229,8 +232,15 @@ export class LocationReaders implements LocationReader { this.readLocation.bind(this), ); } catch (e) { - const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`; + const message = `Processor ${ + processor.constructor.name + } threw an error while processing entity ${current.kind}:${ + current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${current.metadata.name} at ${item.location.type} ${ + item.location.target + }, ${e}`; emit(result.generalError(item.location, message)); + this.logger.warn(message); } } } diff --git a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts index 7360a01a84..0a6f2a11c0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts @@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor { } async processEntity(entity: Entity): Promise { - return await this.policy.enforce(entity); + const output = await this.policy.enforce(entity); + if (!output) { + throw new Error(`No policy applied to entity`); + } + return output; } } diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts new file mode 100644 index 0000000000..324fc7f627 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -0,0 +1,170 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { graphql } from '@octokit/graphql'; +import * as results from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { getOrganizationTeams, getOrganizationUsers } from './util/github'; +import { buildOrgHierarchy } from './util/org'; + +/** + * Extracts teams and users out of a GitHub org. + */ +export class GithubOrgReaderProcessor implements LocationProcessor { + static fromConfig(config: Config) { + return new GithubOrgReaderProcessor(readConfig(config)); + } + + constructor(private readonly providers: ProviderConfig[]) {} + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github-org') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + ); + } + + const { org } = parseUrl(location.target); + const client = !provider.token + ? graphql + : graphql.defaults({ + headers: { + authorization: `token ${provider.token}`, + }, + }); + + const { users } = await getOrganizationUsers(client, org); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + org, + ); + buildOrgHierarchy(groups, users, groupMemberUsers); + + for (const group of groups) { + emit(results.entity(location, group)); + } + for (const user of users) { + emit(results.entity(location, user)); + } + + return true; + } +} + +/* + * Helpers + */ + +/** + * The configuration parameters for a single GitHub API provider. + */ +type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + */ + apiBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +// TODO(freben): Break out common code and config from here and GithubReaderProcessor +export function readConfig(config: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ?? + []; + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (!apiBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }); + } + + return providers; +} + +export function parseUrl(urlString: string): { org: string } { + const path = new URL(urlString).pathname.substr(1).split('/'); + + // /spotify + if (path.length === 1) { + return { org: path[0] }; + } + + // /orgs/spotify[/] + if ( + path.length >= 2 && + path.length <= 3 && + path[0] === 'orgs' && + [undefined, 'teams', 'people'].includes(path[2]) + ) { + return { org: path[1] }; + } + + throw new Error(`Expected a URL pointing to / or /orgs/`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/util/github.ts b/plugins/catalog-backend/src/ingestion/processors/util/github.ts new file mode 100644 index 0000000000..5cb963c797 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/github.ts @@ -0,0 +1,285 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { graphql } from '@octokit/graphql'; + +// Graphql types + +type QueryResponse = { + organization: Organization; +}; + +type Organization = { + membersWithRole: Connection; + team: Team; + teams: Connection; +}; + +type PageInfo = { + hasNextPage: boolean; + endCursor?: string; +}; + +type User = { + login: string; + bio?: string; + avatarUrl?: string; + email?: string; + name?: string; +}; + +type Team = { + slug: string; + combinedSlug: string; + description?: string; + parentTeam?: Team; + members: Connection; +}; + +type Connection = { + pageInfo: PageInfo; + nodes: T[]; +}; + +/** + * Gets all the users out of a GitHub organization. + * + * Note that the users will not have their memberships filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + */ +export async function getOrganizationUsers( + client: typeof graphql, + org: string, +): Promise<{ users: UserEntity[] }> { + const users: UserEntity[] = []; + + // There is no user -> teams edge, so we leave the memberships empty for + // now and let the team iteration handle it instead + let cursor: string | undefined = undefined; + const query = ` + query users($org: String!, $cursor: String) { + organization(login: $org) { + membersWithRole(first: 100, after: $cursor) { + pageInfo { hasNextPage, endCursor } + nodes { avatarUrl, bio, email, login, name } + } + } + }`; + + for (let i = 0; i < 100 /* just for sanity */; ++i) { + const response: QueryResponse = await client(query, { + org, + cursor, + }); + + const connection = response.organization?.membersWithRole; + if (!connection) { + throw new Error(`Found no organization named ${org}`); + } + + for (const user of connection.nodes) { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: user.login, + annotations: { + 'github.com/user-login': user.login, + }, + }, + spec: { + profile: {}, + memberOf: [], + }, + }; + + if (user.bio) entity.metadata.description = user.bio; + if (user.name) entity.spec.profile!.displayName = user.name; + if (user.email) entity.spec.profile!.email = user.email; + if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl; + + users.push(entity); + } + + const { hasNextPage, endCursor } = connection.pageInfo; + if (!hasNextPage) { + break; + } else { + cursor = endCursor; + } + } + + return { users }; +} + +/** + * Gets all the teams out of a GitHub organization. + * + * Note that the teams will not have any relations apart from parent filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + */ +export async function getOrganizationTeams( + client: typeof graphql, + org: string, +): Promise<{ + groups: GroupEntity[]; + groupMemberUsers: Map; +}> { + const groups: GroupEntity[] = []; + const groupMemberUsers = new Map(); + + let cursor: string | undefined = undefined; + const query = ` + query teams($org: String!, $cursor: String) { + organization(login: $org) { + teams(first: 100, after: $cursor) { + pageInfo { hasNextPage, endCursor } + nodes { + slug + combinedSlug + parentTeam { slug } + members(first: 100, membership: IMMEDIATE) { + pageInfo { hasNextPage, endCursor } + nodes { login } + } + } + } + } + }`; + + for (let i = 0; i < 100 /* just for sanity */; ++i) { + const response: QueryResponse = await client(query, { + org, + cursor, + }); + + const connection = response.organization?.teams; + if (!connection) { + throw new Error(`Found no organization named ${org}`); + } + + for (const team of connection.nodes) { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: team.slug, + annotations: { + 'github.com/team-slug': team.combinedSlug, + }, + }, + spec: { + type: 'team', + ancestors: [], + children: [], + descendants: [], + }, + }; + + if (team.description) entity.metadata.description = team.description; + if (team.parentTeam) entity.spec.parent = team.parentTeam.slug; + + groups.push(entity); + + const memberNames: string[] = []; + groupMemberUsers.set(team.slug, memberNames); + + if (!team.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of team.members.nodes) { + memberNames.push(user.login); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(client, org, team.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + } + + const { hasNextPage, endCursor } = connection.pageInfo; + if (!hasNextPage) { + break; + } else { + cursor = endCursor; + } + } + + return { groups, groupMemberUsers }; +} + +/** + * Gets all the users out of a GitHub organization. + * + * Note that the users will not have their memberships filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + * @param teamSlug The slug of the team to read + */ +export async function getTeamMembers( + client: typeof graphql, + org: string, + teamSlug: string, +): Promise<{ members: string[] }> { + const members: string[] = []; + + let cursor: string | undefined = undefined; + const query = ` + query members($org: String!, $teamSlug: String!, $cursor: String) { + organization(login: $org) { + team(slug: $teamSlug) { + members(first: 100, after: $cursor, membership: IMMEDIATE) { + pageInfo { hasNextPage, endCursor } + nodes { login } + } + } + } + }`; + + for (let j = 0; j < 100 /* just for sanity */; ++j) { + const response: QueryResponse = await client(query, { + org, + teamSlug, + cursor, + }); + + const connection = response.organization?.team?.members; + if (!connection) { + throw new Error(`Found no team named ${teamSlug} in named ${org}`); + } + + for (const user of connection.nodes) { + members.push(user.login); + } + + const { hasNextPage, endCursor } = connection.pageInfo; + if (!hasNextPage) { + break; + } else { + cursor = endCursor; + } + } + + return { members }; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.ts new file mode 100644 index 0000000000..bcc4c7b5e6 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; + +export function buildOrgHierarchy( + groups: GroupEntity[], + users: UserEntity[], + groupMemberUsers: Map, +) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + + // + // Make sure that u.memberOf contain all g + // + + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + + // + // Make sure that g.parent.children contain g + // + + for (const group of groups) { + const selfName = group.metadata.name; + const parentName = group.spec.parent; + if (parentName) { + const parent = groupsByName.get(parentName); + if (parent && !parent.spec.children.includes(selfName)) { + parent.spec.children.push(selfName); + } + } + } + + // + // Make sure that g.descendants is complete + // + + function visitDescendants(current: GroupEntity): string[] { + if (current.spec.descendants.length) { + return current.spec.descendants; + } + + const accumulator = new Set(); + for (const childName of current.spec.children) { + accumulator.add(childName); + const child = groupsByName.get(childName); + if (child) { + for (const d of visitDescendants(child)) { + accumulator.add(d); + } + } + } + + const descendants = Array.from(accumulator); + current.spec.descendants = descendants; + return descendants; + } + + for (const group of groups) { + visitDescendants(group); + } + + // + // Make sure that g.ancestors is complete + // + + function visitAncestors(current: GroupEntity): string[] { + if (current.spec.ancestors.length) { + return current.spec.ancestors; + } + + let ancestors: string[]; + const parentName = current.spec.parent; + if (!parentName) { + ancestors = []; + } else { + const parent = groupsByName.get(parentName); + if (parent) { + ancestors = [parentName, ...visitAncestors(parent)]; + } else { + ancestors = [parentName]; + } + } + + current.spec.ancestors = ancestors; + return ancestors; + } + + for (const group of groups) { + visitAncestors(group); + } +} diff --git a/yarn.lock b/yarn.lock index de83b20273..138fa44963 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3393,6 +3393,15 @@ "@octokit/types" "^5.0.0" universal-user-agent "^5.0.0" +"@octokit/graphql@^4.5.6": + version "4.5.6" + resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.6.tgz#708143ba15cf7c1879ed6188266e7f270be805d4" + integrity sha512-Rry+unqKTa3svswT2ZAuqenpLrzJd+JTv89LTeVa5UM/5OX8o4KTkPL7/1ABq4f/ZkELb0XEK/2IEoYwykcLXg== + dependencies: + "@octokit/request" "^5.3.0" + "@octokit/types" "^5.0.0" + universal-user-agent "^6.0.0" + "@octokit/plugin-enterprise-rest@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" @@ -22558,6 +22567,11 @@ universal-user-agent@^5.0.0: dependencies: os-name "^3.1.0" +universal-user-agent@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" + integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== + universalify@^0.1.0: version "0.1.2" resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"