diff --git a/app-config.yaml b/app-config.yaml index e080b0bdb9..cdf01d67f6 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -69,7 +69,7 @@ kubernetes: catalog: rules: - - allow: [Component, API, Group, Template, Location] + - allow: [Component, API, Group, User, Template, Location] processors: github: providers: @@ -89,6 +89,18 @@ catalog: # token: # $secret: # env: GHE_PRIVATE_TOKEN + githubOrg: + providers: + - target: https://github.com + token: + $secret: + env: GITHUB_PRIVATE_TOKEN + #### Example for how to add your GitHub Enterprise instance using the API: + # - target: https://ghe.example.net + # apiBaseUrl: https://ghe.example.net/api/v3 + # token: + # $secret: + # env: GHE_PRIVATE_TOKEN bitbucketApi: username: $secret: @@ -109,19 +121,15 @@ catalog: # Backstage example components - type: github target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml - # Example component for github-actions - type: github target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml - # Example component for techdocs - type: github target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml - # Backstage example APIs - type: github target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml - # Backstage example templates - type: github target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index fdac6dd938..219b04a0f7 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -45,7 +45,7 @@ export default async function createPlugin({ useHotCleanup( module, - runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000), + runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), ); return await createRouter({ diff --git a/packages/catalog-model/src/EntityPolicies.ts b/packages/catalog-model/src/EntityPolicies.ts index 3e2f51b40a..ca59338c16 100644 --- a/packages/catalog-model/src/EntityPolicies.ts +++ b/packages/catalog-model/src/EntityPolicies.ts @@ -23,12 +23,12 @@ import { SchemaValidEntityPolicy, } from './entity'; import { - ApiEntityV1alpha1Policy, - ComponentEntityV1alpha1Policy, - GroupEntityV1alpha1Policy, - LocationEntityV1alpha1Policy, - TemplateEntityV1alpha1Policy, - UserEntityV1alpha1Policy, + apiEntityV1alpha1Policy, + componentEntityV1alpha1Policy, + groupEntityV1alpha1Policy, + locationEntityV1alpha1Policy, + templateEntityV1alpha1Policy, + userEntityV1alpha1Policy, } from './kinds'; import { EntityPolicy } from './types'; @@ -40,7 +40,10 @@ class AllEntityPolicies implements EntityPolicy { async enforce(entity: Entity): Promise { let result = entity; for (const policy of this.policies) { - result = await policy.enforce(entity); + const output = await policy.enforce(entity); + if (output) { + result = output; + } } return result; } @@ -51,12 +54,11 @@ class AllEntityPolicies implements EntityPolicy { class AnyEntityPolicy implements EntityPolicy { constructor(private readonly policies: EntityPolicy[]) {} - async enforce(entity: Entity): Promise { + async enforce(entity: Entity): Promise { for (const policy of this.policies) { - try { - return await policy.enforce(entity); - } catch { - continue; + const output = await policy.enforce(entity); + if (output !== null) { + return output; } } throw new Error(`The entity did not match any known policy`); @@ -76,12 +78,12 @@ export class EntityPolicies implements EntityPolicy { new ReservedFieldsEntityPolicy(), ]), EntityPolicies.anyOf([ - new ComponentEntityV1alpha1Policy(), - new GroupEntityV1alpha1Policy(), - new UserEntityV1alpha1Policy(), - new LocationEntityV1alpha1Policy(), - new TemplateEntityV1alpha1Policy(), - new ApiEntityV1alpha1Policy(), + componentEntityV1alpha1Policy, + groupEntityV1alpha1Policy, + userEntityV1alpha1Policy, + locationEntityV1alpha1Policy, + templateEntityV1alpha1Policy, + apiEntityV1alpha1Policy, ]), ]); } @@ -98,7 +100,7 @@ export class EntityPolicies implements EntityPolicy { this.policy = policy; } - enforce(entity: Entity): Promise { + enforce(entity: Entity): Promise { return this.policy.enforce(entity); } } diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts index 35a1c59ef8..142a8d8160 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { ApiEntityV1alpha1, - ApiEntityV1alpha1Policy, + apiEntityV1alpha1Policy as policy, } from './ApiEntityV1alpha1'; describe('ApiV1alpha1Policy', () => { let entity: ApiEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -49,7 +47,7 @@ paths: '200': description: A paged array of pets content: - application/json: + application/json: schema: $ref: "#/components/schemas/Pets" components: @@ -74,7 +72,6 @@ components: `, }, }; - policy = new ApiEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -86,14 +83,14 @@ components: await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts index 972f6df96d..052172d0a1 100644 --- a/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts @@ -16,11 +16,24 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'API' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + type: yup.string().required().min(1), + lifecycle: yup.string().required().min(1), + owner: yup.string().required().min(1), + definition: yup.string().required().min(1), + }) + .required(), +}); + export interface ApiEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -32,25 +45,4 @@ export interface ApiEntityV1alpha1 extends Entity { }; } -export class ApiEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - type: yup.string().required().min(1), - lifecycle: yup.string().required().min(1), - owner: yup.string().required().min(1), - definition: yup.string().required().min(1), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const apiEntityV1alpha1Policy = schemaPolicy(KIND, API_VERSION, schema); diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts index e784934d48..57d6d47388 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { ComponentEntityV1alpha1, - ComponentEntityV1alpha1Policy, + componentEntityV1alpha1Policy as policy, } from './ComponentEntityV1alpha1'; describe('ComponentV1alpha1Policy', () => { let entity: ComponentEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -38,7 +36,6 @@ describe('ComponentV1alpha1Policy', () => { implementsApis: ['api-0'], }, }; - policy = new ComponentEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -50,14 +47,14 @@ describe('ComponentV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 38b9c27720..9efc491ca9 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -16,11 +16,24 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'Component' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + type: yup.string().required().min(1), + lifecycle: yup.string().required().min(1), + owner: yup.string().required().min(1), + implementsApis: yup.array(yup.string()).notRequired(), + }) + .required(), +}); + export interface ComponentEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -32,25 +45,8 @@ export interface ComponentEntityV1alpha1 extends Entity { }; } -export class ComponentEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - type: yup.string().required().min(1), - lifecycle: yup.string().required().min(1), - owner: yup.string().required().min(1), - implementsApis: yup.array(yup.string()).notRequired(), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const componentEntityV1alpha1Policy = schemaPolicy( + KIND, + API_VERSION, + schema, +); diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts index 94dcc2f14f..dc0ecd3fa3 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { GroupEntityV1alpha1, - GroupEntityV1alpha1Policy, + groupEntityV1alpha1Policy as policy, } from './GroupEntityV1alpha1'; describe('GroupV1alpha1Policy', () => { let entity: GroupEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -41,7 +39,6 @@ describe('GroupV1alpha1Policy', () => { descendants: ['desc-a', 'desc-b'], }, }; - policy = new GroupEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -53,14 +50,14 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { @@ -98,6 +95,11 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); + it('accepts no ancestors', async () => { + (entity as any).spec.ancestors = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + it('rejects missing children', async () => { delete (entity as any).spec.children; await expect(policy.enforce(entity)).rejects.toThrow(/children/); @@ -108,6 +110,11 @@ describe('GroupV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); + it('accepts no children', async () => { + (entity as any).spec.children = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + it('rejects missing descendants', async () => { delete (entity as any).spec.descendants; await expect(policy.enforce(entity)).rejects.toThrow(/descendants/); @@ -117,4 +124,9 @@ describe('GroupV1alpha1Policy', () => { (entity as any).spec.descendants = ['']; await expect(policy.enforce(entity)).resolves.toBe(entity); }); + + it('accepts no descendants', async () => { + (entity as any).spec.descendants = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); }); diff --git a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts index c1d39d5b44..073332d707 100644 --- a/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts @@ -16,11 +16,39 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'Group' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + type: yup.string().required().min(1), + parent: yup.string().notRequired().min(1), + // Use these manual tests because yup .required() requires at least + // one element and there is no simple workaround -_- + ancestors: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'ancestors must be defined', + test: v => Boolean(v), + }), + children: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'children must be defined', + test: v => Boolean(v), + }), + descendants: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'descendants must be defined', + test: v => Boolean(v), + }), + }) + .required(), +}); + export interface GroupEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -33,26 +61,8 @@ export interface GroupEntityV1alpha1 extends Entity { }; } -export class GroupEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - type: yup.string().required().min(1), - parent: yup.string().notRequired().min(1), - ancestors: yup.array(yup.string()).required(), - children: yup.array(yup.string()).required(), - descendants: yup.array(yup.string()).required(), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const groupEntityV1alpha1Policy = schemaPolicy( + KIND, + API_VERSION, + schema, +); diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts index 62837ed500..a259eaeca8 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { LocationEntityV1alpha1, - LocationEntityV1alpha1Policy, + locationEntityV1alpha1Policy as policy, } from './LocationEntityV1alpha1'; describe('LocationV1alpha1Policy', () => { let entity: LocationEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -35,7 +33,6 @@ describe('LocationV1alpha1Policy', () => { type: 'github', }, }; - policy = new LocationEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -47,14 +44,14 @@ describe('LocationV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts index a69a3a574a..6d1f1f5515 100644 --- a/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts @@ -16,11 +16,23 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'Location' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + type: yup.string().required().min(1), + target: yup.string().notRequired().min(1), + targets: yup.array(yup.string()).notRequired(), + }) + .required(), +}); + export interface LocationEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -31,24 +43,8 @@ export interface LocationEntityV1alpha1 extends Entity { }; } -export class LocationEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - type: yup.string().required().min(1), - target: yup.string().notRequired().min(1), - targets: yup.array(yup.string()).notRequired(), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const locationEntityV1alpha1Policy = schemaPolicy( + KIND, + API_VERSION, + schema, +); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts index ce9ed8bedd..86351b41aa 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { TemplateEntityV1alpha1, - TemplateEntityV1alpha1Policy, + templateEntityV1alpha1Policy as policy, } from './TemplateEntityV1alpha1'; -describe('TemplateEntityV1alpah1', () => { +describe('templateEntityV1alpha1', () => { let entity: TemplateEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -52,7 +50,6 @@ describe('TemplateEntityV1alpah1', () => { }, }, }; - policy = new TemplateEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -64,14 +61,14 @@ describe('TemplateEntityV1alpah1', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('rejects missing type', async () => { diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts index 8aa79d57db..a26540fda2 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts @@ -16,11 +16,25 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy, JSONSchema } from '../types'; +import type { JSONSchema } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'Template' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + type: yup.string().required().min(1), + path: yup.string(), + schema: yup.object().required(), + templater: yup.string().required(), + }) + .required(), +}); + export interface TemplateEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -32,25 +46,8 @@ export interface TemplateEntityV1alpha1 extends Entity { }; } -export class TemplateEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - type: yup.string().required().min(1), - path: yup.string(), - schema: yup.object().required(), - templater: yup.string().required(), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const templateEntityV1alpha1Policy = schemaPolicy( + KIND, + API_VERSION, + schema, +); diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts index b641c6cad3..97fe0888f1 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { EntityPolicy } from '../types'; import { UserEntityV1alpha1, - UserEntityV1alpha1Policy, + userEntityV1alpha1Policy as policy, } from './UserEntityV1alpha1'; -describe('UserV1alpha1Policy', () => { +describe('userEntityV1alpha1Policy', () => { let entity: UserEntityV1alpha1; - let policy: EntityPolicy; beforeEach(() => { entity = { @@ -40,7 +38,6 @@ describe('UserV1alpha1Policy', () => { memberOf: ['team-a', 'developers'], }, }; - policy = new UserEntityV1alpha1Policy(); }); it('happy path: accepts valid data', async () => { @@ -54,14 +51,14 @@ describe('UserV1alpha1Policy', () => { await expect(policy.enforce(entity)).resolves.toBe(entity); }); - it('rejects unknown apiVersion', async () => { + it('ignores unknown apiVersion', async () => { (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); - it('rejects unknown kind', async () => { + it('ignores unknown kind', async () => { (entity as any).kind = 'Wizard'; - await expect(policy.enforce(entity)).rejects.toThrow(/kind/); + await expect(policy.enforce(entity)).resolves.toBeUndefined(); }); it('spec accepts unknown additional fields', async () => { @@ -147,4 +144,14 @@ describe('UserV1alpha1Policy', () => { (entity as any).spec.memberOf[0] = 7; await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/); }); + + it('accepts empty memberOf', async () => { + (entity as any).spec.memberOf = []; + await expect(policy.enforce(entity)).resolves.toBe(entity); + }); + + it('rejects null memberOf', async () => { + (entity as any).spec.memberOf = null; + await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/); + }); }); diff --git a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts index a6a304f509..ff4edcc8a2 100644 --- a/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/UserEntityV1alpha1.ts @@ -16,11 +16,34 @@ import * as yup from 'yup'; import type { Entity } from '../entity/Entity'; -import type { EntityPolicy } from '../types'; +import { schemaPolicy } from './util'; const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const; const KIND = 'User' as const; +const schema = yup.object>({ + apiVersion: yup.string().required().oneOf(API_VERSION), + kind: yup.string().required().equals([KIND]), + spec: yup + .object({ + profile: yup + .object({ + displayName: yup.string().min(1).notRequired(), + email: yup.string().min(1).notRequired(), + picture: yup.string().min(1).notRequired(), + }) + .notRequired(), + // Use this manual test because yup .required() requires at least one + // element and there is no simple workaround -_- + memberOf: yup.array(yup.string()).test({ + name: 'isDefined', + message: 'memberOf must be defined', + test: v => Boolean(v), + }), + }) + .required(), +}); + export interface UserEntityV1alpha1 extends Entity { apiVersion: typeof API_VERSION[number]; kind: typeof KIND; @@ -34,29 +57,4 @@ export interface UserEntityV1alpha1 extends Entity { }; } -export class UserEntityV1alpha1Policy implements EntityPolicy { - private schema: yup.Schema; - - constructor() { - this.schema = yup.object>({ - apiVersion: yup.string().required().oneOf(API_VERSION), - kind: yup.string().required().equals([KIND]), - spec: yup - .object({ - profile: yup - .object({ - displayName: yup.string().min(1).notRequired(), - email: yup.string().min(1).notRequired(), - picture: yup.string().min(1).notRequired(), - }) - .notRequired(), - memberOf: yup.array(yup.string()).required(), - }) - .required(), - }); - } - - async enforce(envelope: Entity): Promise { - return await this.schema.validate(envelope, { strict: true }); - } -} +export const userEntityV1alpha1Policy = schemaPolicy(KIND, API_VERSION, schema); diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts index 1ac93b426d..6df4f1212a 100644 --- a/packages/catalog-model/src/kinds/index.ts +++ b/packages/catalog-model/src/kinds/index.ts @@ -14,32 +14,32 @@ * limitations under the License. */ -export { ApiEntityV1alpha1Policy } from './ApiEntityV1alpha1'; +export { apiEntityV1alpha1Policy } from './ApiEntityV1alpha1'; export type { ApiEntityV1alpha1 as ApiEntity, ApiEntityV1alpha1, } from './ApiEntityV1alpha1'; -export { ComponentEntityV1alpha1Policy } from './ComponentEntityV1alpha1'; +export { componentEntityV1alpha1Policy } from './ComponentEntityV1alpha1'; export type { ComponentEntityV1alpha1 as ComponentEntity, ComponentEntityV1alpha1, } from './ComponentEntityV1alpha1'; -export { GroupEntityV1alpha1Policy } from './GroupEntityV1alpha1'; +export { groupEntityV1alpha1Policy } from './GroupEntityV1alpha1'; export type { GroupEntityV1alpha1 as GroupEntity, GroupEntityV1alpha1, } from './GroupEntityV1alpha1'; -export { LocationEntityV1alpha1Policy } from './LocationEntityV1alpha1'; +export { locationEntityV1alpha1Policy } from './LocationEntityV1alpha1'; export type { LocationEntityV1alpha1 as LocationEntity, LocationEntityV1alpha1, } from './LocationEntityV1alpha1'; -export { TemplateEntityV1alpha1Policy } from './TemplateEntityV1alpha1'; +export { templateEntityV1alpha1Policy } from './TemplateEntityV1alpha1'; export type { TemplateEntityV1alpha1 as TemplateEntity, TemplateEntityV1alpha1, } from './TemplateEntityV1alpha1'; -export { UserEntityV1alpha1Policy } from './UserEntityV1alpha1'; +export { userEntityV1alpha1Policy } from './UserEntityV1alpha1'; export type { UserEntityV1alpha1 as UserEntity, UserEntityV1alpha1, diff --git a/packages/catalog-model/src/kinds/util.ts b/packages/catalog-model/src/kinds/util.ts new file mode 100644 index 0000000000..491b2c5e2f --- /dev/null +++ b/packages/catalog-model/src/kinds/util.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as yup from 'yup'; +import { Entity } from '../entity'; +import { EntityPolicy } from '../types'; + +export function schemaPolicy( + kind: string, + apiVersion: readonly string[], + schema: yup.Schema, +): EntityPolicy { + return { + async enforce(envelope: Entity): Promise { + if ( + kind !== envelope.kind || + !apiVersion.includes(envelope.apiVersion as any) + ) { + return undefined; + } + return await schema.validate(envelope, { strict: true }); + }, + }; +} diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 3aafdc241a..aa96e46102 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -27,10 +27,11 @@ export type EntityPolicy = { * Applies validation or mutation on an entity. * * @param entity The entity, as validated/mutated so far in the policy tree - * @returns The incoming entity, or a mutated version of the same + * @returns The incoming entity, or a mutated version of the same, or + * undefined if this processor could not handle the entity * @throws An error if the entity should be rejected */ - enforce(entity: Entity): Promise; + enforce(entity: Entity): Promise; }; export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue }; diff --git a/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.test.tsx.hbs b/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.test.tsx.hbs index 1069790218..fdb39444d8 100644 --- a/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.test.tsx.hbs +++ b/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.test.tsx.hbs @@ -10,7 +10,7 @@ import { setupServer } from 'msw/node'; describe('ExampleComponent', () => { const server = setupServer(); // Enable API mocking before tests. - beforeAll(() => server.listen()) + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) // Reset any runtime request handlers we may add during the tests. afterEach(() => server.resetHandlers()) @@ -32,4 +32,3 @@ describe('ExampleComponent', () => { expect(rendered.getByText('Welcome to {{ id }}!')).toBeInTheDocument(); }); }); - \ No newline at end of file diff --git a/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx.hbs b/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx.hbs index c584289077..ca1990b4bc 100644 --- a/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx.hbs +++ b/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx.hbs @@ -7,7 +7,7 @@ import { setupServer } from 'msw/node'; describe('ExampleFetchComponent', () => { const server = setupServer(); // Enable API mocking before tests. - beforeAll(() => server.listen()) + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) // Reset any runtime request handlers we may add during the tests. afterEach(() => server.resetHandlers()) diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 17e9fa0728..f1278bd3c3 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -66,7 +66,7 @@ scaffolder: catalog: rules: - - allow: [Component, API, Group, Template, Location] + - allow: [Component, API, Group, User, Template, Location] processors: github: providers: diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 13c300d4b9..b95a78aa3a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -23,6 +23,7 @@ "@backstage/backend-common": "^0.1.1-alpha.23", "@backstage/catalog-model": "^0.1.1-alpha.23", "@backstage/config": "^0.1.1-alpha.23", + "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 6200bd51c6..00ee95f3a5 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -19,6 +19,7 @@ import { Entity, EntityPolicies, EntityPolicy, + ENTITY_DEFAULT_NAMESPACE, LocationSpec, } from '@backstage/catalog-model'; import { Config, ConfigReader } from '@backstage/config'; @@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; +import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; @@ -86,6 +88,7 @@ export class LocationReaders implements LocationReader { new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(config), new AzureApiReaderProcessor(config), + GithubOrgReaderProcessor.fromConfig(config), new UrlReaderProcessor(), new YamlProcessor(), PlaceholderProcessor.default(), @@ -178,12 +181,14 @@ export class LocationReaders implements LocationReader { } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); + this.logger.warn(message); } } } const message = `No processor was able to read location ${item.location.type} ${item.location.target}`; emit(result.inputError(item.location, message)); + this.logger.warn(message); } private async handleData( @@ -203,6 +208,7 @@ export class LocationReaders implements LocationReader { } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); + this.logger.warn(message); } } } @@ -231,8 +237,15 @@ export class LocationReaders implements LocationReader { this.readLocation.bind(this), ); } catch (e) { - const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`; + // Construct the name carefully, if we got validation errors we do + // not want to crash here due to missing metadata or so + const namespace = !current.metadata + ? '' + : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE; + const name = !current.metadata ? '' : current.metadata.name; + const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); + this.logger.warn(message); } } } @@ -255,6 +268,7 @@ export class LocationReaders implements LocationReader { } catch (e) { const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`; emit(result.generalError(item.location, message)); + this.logger.warn(message); } } } diff --git a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts index 7360a01a84..67da9a6705 100644 --- a/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts @@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor { } async processEntity(entity: Entity): Promise { - return await this.policy.enforce(entity); + const output = await this.policy.enforce(entity); + if (!output) { + throw new Error(`Entity did not match any known schema`); + } + return output; } } diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts new file mode 100644 index 0000000000..5084d4c16f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { graphql } from '@octokit/graphql'; +import * as results from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; +import { getOrganizationTeams, getOrganizationUsers } from './util/github'; +import { buildOrgHierarchy } from './util/org'; + +/** + * Extracts teams and users out of a GitHub org. + */ +export class GithubOrgReaderProcessor implements LocationProcessor { + static fromConfig(config: Config) { + return new GithubOrgReaderProcessor(readConfig(config)); + } + + constructor(private readonly providers: ProviderConfig[]) {} + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github-org') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(`${p.target}/`), + ); + if (!provider) { + throw new Error( + `There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`, + ); + } + + const { org } = parseUrl(location.target); + const client = !provider.token + ? graphql + : graphql.defaults({ + headers: { + authorization: `token ${provider.token}`, + }, + }); + + const { users } = await getOrganizationUsers(client, org); + const { groups, groupMemberUsers } = await getOrganizationTeams( + client, + org, + ); + buildOrgHierarchy(groups, users, groupMemberUsers); + + for (const group of groups) { + emit(results.entity(location, group)); + } + for (const user of users) { + emit(results.entity(location, user)); + } + + return true; + } +} + +/* + * Helpers + */ + +/** + * The configuration parameters for a single GitHub API provider. + */ +type ProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. "https://github.com", + * with no trailing slash. + */ + target: string; + + /** + * The base URL of the API of this provider, e.g. "https://api.github.com", + * with no trailing slash. + * + * May be omitted specifically for GitHub; then it will be deduced. + */ + apiBaseUrl?: string; + + /** + * The authorization token to use for requests to this provider. + * + * If no token is specified, anonymous access is used. + */ + token?: string; +}; + +// TODO(freben): Break out common code and config from here and GithubReaderProcessor +export function readConfig(config: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; + + const providerConfigs = + config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ?? + []; + + // First read all the explicit providers + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + const token = providerConfig.getOptionalString('token'); + + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (target === 'https://github.com') { + apiBaseUrl = 'https://api.github.com'; + } + + if (!apiBaseUrl) { + throw new Error( + `Provider at ${target} must configure an explicit apiBaseUrl`, + ); + } + + providers.push({ target, apiBaseUrl, token }); + } + + // If no explicit github.com provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.target === 'https://github.com')) { + providers.push({ + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }); + } + + return providers; +} + +export function parseUrl(urlString: string): { org: string } { + const path = new URL(urlString).pathname.substr(1).split('/'); + + // /spotify + if (path.length === 1 && path[0].length) { + return { org: decodeURIComponent(path[0]) }; + } + + throw new Error(`Expected a URL pointing to /`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index e30b320bf4..1ab4cb09b8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -27,7 +27,7 @@ describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost:23000'; const server = setupServer(); - beforeAll(() => server.listen()); + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); afterEach(() => server.resetHandlers()); afterAll(() => server.close()); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/util/github.test.ts new file mode 100644 index 0000000000..25a37e0546 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/github.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { graphql } from '@octokit/graphql'; +import { graphql as graphqlMsw } from 'msw'; +import { setupServer } from 'msw/node'; +import { + getOrganizationTeams, + getOrganizationUsers, + getTeamMembers, + QueryResponse, +} from './github'; + +describe('github', () => { + const server = setupServer(); + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + describe('getOrganizationUsers', () => { + it('reads members', async () => { + const input: QueryResponse = { + organization: { + membersWithRole: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + login: 'a', + name: 'b', + bio: 'c', + email: 'd', + avatarUrl: 'e', + }, + ], + }, + }, + }; + + const output = { + users: [ + expect.objectContaining({ + metadata: expect.objectContaining({ name: 'a', description: 'c' }), + spec: { + profile: { displayName: 'b', email: 'd', picture: 'e' }, + memberOf: [], + }, + }), + ], + }; + + server.use( + graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getOrganizationUsers(graphql, 'a')).resolves.toEqual(output); + }); + }); + + describe('getOrganizationTeams', () => { + it('reads teams', async () => { + const input: QueryResponse = { + organization: { + teams: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + slug: 'team', + combinedSlug: 'blah/team', + parentTeam: { + slug: 'parent', + combinedSlug: '', + members: { pageInfo: { hasNextPage: false }, nodes: [] }, + }, + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'user' }], + }, + }, + ], + }, + }, + }; + + const output = { + groups: [ + expect.objectContaining({ + metadata: expect.objectContaining({ name: 'team' }), + spec: { + type: 'team', + parent: 'parent', + ancestors: [], + children: [], + descendants: [], + }, + }), + ], + groupMemberUsers: new Map([['team', ['user']]]), + }; + + server.use( + graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output); + }); + }); + + describe('getTeamMembers', () => { + it('reads team members', async () => { + const input: QueryResponse = { + organization: { + team: { + slug: '', + combinedSlug: '', + members: { + pageInfo: { hasNextPage: false }, + nodes: [{ login: 'user' }], + }, + }, + }, + }; + + const output = { + members: ['user'], + }; + + server.use( + graphqlMsw.query('members', (_req, res, ctx) => res(ctx.data(input))), + ); + + await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/github.ts b/plugins/catalog-backend/src/ingestion/processors/util/github.ts new file mode 100644 index 0000000000..96cf4a7668 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/github.ts @@ -0,0 +1,297 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { graphql } from '@octokit/graphql'; + +// Graphql types + +export type QueryResponse = { + organization: Organization; +}; + +export type Organization = { + membersWithRole?: Connection; + team?: Team; + teams?: Connection; +}; + +export type PageInfo = { + hasNextPage: boolean; + endCursor?: string; +}; + +export type User = { + login: string; + bio?: string; + avatarUrl?: string; + email?: string; + name?: string; +}; + +export type Team = { + slug: string; + combinedSlug: string; + description?: string; + parentTeam?: Team; + members: Connection; +}; + +export type Connection = { + pageInfo: PageInfo; + nodes: T[]; +}; + +/** + * Gets all the users out of a GitHub organization. + * + * Note that the users will not have their memberships filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + */ +export async function getOrganizationUsers( + client: typeof graphql, + org: string, +): Promise<{ users: UserEntity[] }> { + const query = ` + query users($org: String!, $cursor: String) { + organization(login: $org) { + membersWithRole(first: 100, after: $cursor) { + pageInfo { hasNextPage, endCursor } + nodes { avatarUrl, bio, email, login, name } + } + } + }`; + + // There is no user -> teams edge, so we leave the memberships empty for + // now and let the team iteration handle it instead + const mapper = (user: User) => { + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: user.login, + annotations: { + 'github.com/user-login': user.login, + }, + }, + spec: { + profile: {}, + memberOf: [], + }, + }; + + if (user.bio) entity.metadata.description = user.bio; + if (user.name) entity.spec.profile!.displayName = user.name; + if (user.email) entity.spec.profile!.email = user.email; + if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl; + + return entity; + }; + + const users = await queryWithPaging( + client, + query, + r => r.organization?.membersWithRole, + mapper, + { org }, + ); + + return { users }; +} + +/** + * Gets all the teams out of a GitHub organization. + * + * Note that the teams will not have any relations apart from parent filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + */ +export async function getOrganizationTeams( + client: typeof graphql, + org: string, +): Promise<{ + groups: GroupEntity[]; + groupMemberUsers: Map; +}> { + const query = ` + query teams($org: String!, $cursor: String) { + organization(login: $org) { + teams(first: 100, after: $cursor) { + pageInfo { hasNextPage, endCursor } + nodes { + slug + combinedSlug + parentTeam { slug } + members(first: 100, membership: IMMEDIATE) { + pageInfo { hasNextPage } + nodes { login } + } + } + } + } + }`; + + // Gets populated inside the mapper below + const groupMemberUsers = new Map(); + + const mapper = async (team: Team) => { + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: team.slug, + annotations: { + 'github.com/team-slug': team.combinedSlug, + }, + }, + spec: { + type: 'team', + ancestors: [], + children: [], + descendants: [], + }, + }; + + if (team.description) entity.metadata.description = team.description; + if (team.parentTeam) entity.spec.parent = team.parentTeam.slug; + + const memberNames: string[] = []; + groupMemberUsers.set(team.slug, memberNames); + + if (!team.members.pageInfo.hasNextPage) { + // We got all the members in one go, run the fast path + for (const user of team.members.nodes) { + memberNames.push(user.login); + } + } else { + // There were more than a hundred immediate members - run the slow + // path of fetching them explicitly + const { members } = await getTeamMembers(client, org, team.slug); + for (const userLogin of members) { + memberNames.push(userLogin); + } + } + + return entity; + }; + + const groups = await queryWithPaging( + client, + query, + r => r.organization?.teams, + mapper, + { org }, + ); + + return { groups, groupMemberUsers }; +} + +/** + * Gets all the users out of a GitHub organization. + * + * Note that the users will not have their memberships filled in. + * + * @param client An octokit graphql client + * @param org The slug of the org to read + * @param teamSlug The slug of the team to read + */ +export async function getTeamMembers( + client: typeof graphql, + org: string, + teamSlug: string, +): Promise<{ members: string[] }> { + const query = ` + query members($org: String!, $teamSlug: String!, $cursor: String) { + organization(login: $org) { + team(slug: $teamSlug) { + members(first: 100, after: $cursor, membership: IMMEDIATE) { + pageInfo { hasNextPage, endCursor } + nodes { login } + } + } + } + }`; + + const members = await queryWithPaging( + client, + query, + r => r.organization?.team?.members, + user => user.login, + { org, teamSlug }, + ); + + return { members }; +} + +// +// Helpers +// + +/** + * Assists in repeatedly executing a query with a paged response. + * + * Requires that the query accepts a $cursor variable. + * + * @param client The octokit client + * @param query The query to execute + * @param connection A function that, given the response, picks out the actual + * Connection object that's being iterated + * @param mapper A function that, given one of the nodes in the Connection, + * returns the model mapped form of it + * @param variables The variable values that the query needs, minus the cursor + */ +export async function queryWithPaging< + GraphqlType, + OutputType, + Variables extends {}, + Response = QueryResponse +>( + client: typeof graphql, + query: string, + connection: (response: Response) => Connection | undefined, + mapper: (item: GraphqlType) => Promise | OutputType, + variables: Variables, +): Promise { + const result: OutputType[] = []; + + let cursor: string | undefined = undefined; + for (let j = 0; j < 1000 /* just for sanity */; ++j) { + const response: Response = await client(query, { + ...variables, + cursor, + }); + + const conn = connection(response); + if (!conn) { + throw new Error(`Found no match for ${JSON.stringify(variables)}`); + } + + for (const node of conn.nodes) { + result.push(await mapper(node)); + } + + if (!conn.pageInfo.hasNextPage) { + break; + } else { + cursor = conn.pageInfo.endCursor; + } + } + + return result; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts new file mode 100644 index 0000000000..e0120735fe --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { buildOrgHierarchy } from './org'; + +function u(name: string): UserEntity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name }, + spec: { memberOf: [] }, + }; +} + +function g( + name: string, + parent: string | undefined, + children: string[], +): GroupEntity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { name }, + spec: { type: 'team', parent, children, ancestors: [], descendants: [] }, + }; +} + +describe('buildOrgHierarchy', () => { + it('puts users in the respective groups', () => { + const a = g('a', undefined, []); + const b = g('b', undefined, []); + const x = u('x'); + const y = u('y'); + const groupMemberUsers: Map = new Map([ + ['a', ['x', 'y']], + ['b', ['y']], + ]); + buildOrgHierarchy([a, b], [x, y], groupMemberUsers); + expect(x.spec.memberOf).toEqual(['a']); + expect(y.spec.memberOf).toEqual(['a', 'b']); + }); + + it('adds groups to their parent.children', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const d = g('d', 'a', []); + buildOrgHierarchy([a, b, c, d], [], new Map()); + expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd'])); + expect(b.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(c.spec.children).toEqual([]); + expect(d.spec.children).toEqual([]); + }); + + it('fills out descendants', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const d = g('d', 'a', []); + buildOrgHierarchy([a, b, c, d], [], new Map()); + expect(a.spec.descendants).toEqual(expect.arrayContaining(['b', 'c', 'd'])); + expect(b.spec.descendants).toEqual(expect.arrayContaining(['c'])); + expect(c.spec.descendants).toEqual([]); + expect(d.spec.descendants).toEqual([]); + }); + + it('fills out ancestors', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const d = g('d', 'a', []); + buildOrgHierarchy([a, b, c, d], [], new Map()); + expect(a.spec.ancestors).toEqual([]); + expect(b.spec.ancestors).toEqual(expect.arrayContaining(['a'])); + expect(c.spec.ancestors).toEqual(expect.arrayContaining(['a', 'b'])); + expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a'])); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.ts new file mode 100644 index 0000000000..bcc4c7b5e6 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; + +export function buildOrgHierarchy( + groups: GroupEntity[], + users: UserEntity[], + groupMemberUsers: Map, +) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + + // + // Make sure that u.memberOf contain all g + // + + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + + // + // Make sure that g.parent.children contain g + // + + for (const group of groups) { + const selfName = group.metadata.name; + const parentName = group.spec.parent; + if (parentName) { + const parent = groupsByName.get(parentName); + if (parent && !parent.spec.children.includes(selfName)) { + parent.spec.children.push(selfName); + } + } + } + + // + // Make sure that g.descendants is complete + // + + function visitDescendants(current: GroupEntity): string[] { + if (current.spec.descendants.length) { + return current.spec.descendants; + } + + const accumulator = new Set(); + for (const childName of current.spec.children) { + accumulator.add(childName); + const child = groupsByName.get(childName); + if (child) { + for (const d of visitDescendants(child)) { + accumulator.add(d); + } + } + } + + const descendants = Array.from(accumulator); + current.spec.descendants = descendants; + return descendants; + } + + for (const group of groups) { + visitDescendants(group); + } + + // + // Make sure that g.ancestors is complete + // + + function visitAncestors(current: GroupEntity): string[] { + if (current.spec.ancestors.length) { + return current.spec.ancestors; + } + + let ancestors: string[]; + const parentName = current.spec.parent; + if (!parentName) { + ancestors = []; + } else { + const parent = groupsByName.get(parentName); + if (parent) { + ancestors = [parentName, ...visitAncestors(parent)]; + } else { + ancestors = [parentName]; + } + } + + current.spec.ancestors = ancestors; + return ancestors; + } + + for (const group of groups) { + visitAncestors(group); + } +} diff --git a/plugins/catalog-graphql/src/graphql/module.test.ts b/plugins/catalog-graphql/src/graphql/module.test.ts index 09708f68f4..e3e03a9b4a 100644 --- a/plugins/catalog-graphql/src/graphql/module.test.ts +++ b/plugins/catalog-graphql/src/graphql/module.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { createModule } from './module'; import { execute } from 'graphql'; import { rest } from 'msw'; @@ -36,9 +37,8 @@ describe('Catalog Module', () => { }, ]); - beforeAll(() => worker.listen()); + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); afterAll(() => worker.close()); - afterEach(() => worker.resetHandlers()); describe('Default Entity', () => { diff --git a/plugins/catalog-graphql/src/service/client.test.ts b/plugins/catalog-graphql/src/service/client.test.ts index 876059802b..d8cc59d1a8 100644 --- a/plugins/catalog-graphql/src/service/client.test.ts +++ b/plugins/catalog-graphql/src/service/client.test.ts @@ -20,9 +20,8 @@ import { setupServer } from 'msw/node'; describe('Catalog GraphQL Module', () => { const worker = setupServer(); - beforeAll(() => worker.listen()); + beforeAll(() => worker.listen({ onUnhandledRequest: 'error' })); afterAll(() => worker.close()); - afterEach(() => worker.resetHandlers()); const baseUrl = 'http://localhost:1234'; diff --git a/plugins/catalog/src/api/CatalogClient.test.ts b/plugins/catalog/src/api/CatalogClient.test.ts index 18f062db77..832a114276 100644 --- a/plugins/catalog/src/api/CatalogClient.test.ts +++ b/plugins/catalog/src/api/CatalogClient.test.ts @@ -25,7 +25,7 @@ const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base'; const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); describe('CatalogClient', () => { - beforeAll(() => server.listen()); + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); afterEach(() => server.resetHandlers()); afterAll(() => server.close()); diff --git a/yarn.lock b/yarn.lock index 9940180cd6..a8628780c6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3393,6 +3393,15 @@ "@octokit/types" "^5.0.0" universal-user-agent "^5.0.0" +"@octokit/graphql@^4.5.6": + version "4.5.6" + resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.6.tgz#708143ba15cf7c1879ed6188266e7f270be805d4" + integrity sha512-Rry+unqKTa3svswT2ZAuqenpLrzJd+JTv89LTeVa5UM/5OX8o4KTkPL7/1ABq4f/ZkELb0XEK/2IEoYwykcLXg== + dependencies: + "@octokit/request" "^5.3.0" + "@octokit/types" "^5.0.0" + universal-user-agent "^6.0.0" + "@octokit/plugin-enterprise-rest@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" @@ -22781,6 +22790,11 @@ universal-user-agent@^5.0.0: dependencies: os-name "^3.1.0" +universal-user-agent@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" + integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== + universalify@^0.1.0: version "0.1.2" resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"