paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 48 Packages Projects Releases Wiki Activity

Merge pull request #2669 from spotify/freben/github-org

Browse Source 
feat(catalog-backend): implement github org entity ingestion
This commit is contained in:
Fredrik Adelöw
2020-09-30 15:08:24 +02:00
committed by GitHub
parent b011b68a1a 7ec6dc7f30
commit dc6b1e7c79
34 changed files with 1122 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+13 -5
View File
@@ -69,7 +69,7 @@ kubernetes:
catalog:
rules:
- allow: [Component, API, Group, Template, Location]
- allow: [Component, API, Group, User, Template, Location]
processors:
github:
providers:
@@ -89,6 +89,18 @@ catalog:
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
githubOrg:
providers:
- target: https://github.com
token:
$secret:
env: GITHUB_PRIVATE_TOKEN
#### Example for how to add your GitHub Enterprise instance using the API:
# - target: https://ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $secret:
# env: GHE_PRIVATE_TOKEN
bitbucketApi:
username:
$secret:
@@ -109,19 +121,15 @@ catalog:
# Backstage example components
- type: github
target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml
# Example component for github-actions
- type: github
target: https://github.com/spotify/backstage/blob/master/plugins/github-actions/examples/sample.yaml
# Example component for techdocs
- type: github
target: https://github.com/spotify/backstage/blob/master/plugins/techdocs-backend/examples/documented-component/documented-component.yaml
# Backstage example APIs
- type: github
target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml
# Backstage example templates
- type: github
target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/all-templates.yaml
packages/backend/src/plugins/catalog.ts
+1 -1
View File
@@ -45,7 +45,7 @@ export default async function createPlugin({
useHotCleanup(
module,
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 10000),
runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000),
);
return await createRouter({
packages/catalog-model/src/EntityPolicies.ts
+21 -19
View File
@@ -23,12 +23,12 @@ import {
SchemaValidEntityPolicy,
} from './entity';
import {
ApiEntityV1alpha1Policy,
ComponentEntityV1alpha1Policy,
GroupEntityV1alpha1Policy,
LocationEntityV1alpha1Policy,
TemplateEntityV1alpha1Policy,
UserEntityV1alpha1Policy,
apiEntityV1alpha1Policy,
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
userEntityV1alpha1Policy,
} from './kinds';
import { EntityPolicy } from './types';
@@ -40,7 +40,10 @@ class AllEntityPolicies implements EntityPolicy {
async enforce(entity: Entity): Promise<Entity> {
let result = entity;
for (const policy of this.policies) {
result = await policy.enforce(entity);
const output = await policy.enforce(entity);
if (output) {
result = output;
}
}
return result;
}
@@ -51,12 +54,11 @@ class AllEntityPolicies implements EntityPolicy {
class AnyEntityPolicy implements EntityPolicy {
constructor(private readonly policies: EntityPolicy[]) {}
async enforce(entity: Entity): Promise<Entity> {
async enforce(entity: Entity): Promise<Entity | undefined> {
for (const policy of this.policies) {
try {
return await policy.enforce(entity);
} catch {
continue;
const output = await policy.enforce(entity);
if (output !== null) {
return output;
}
}
throw new Error(`The entity did not match any known policy`);
@@ -76,12 +78,12 @@ export class EntityPolicies implements EntityPolicy {
new ReservedFieldsEntityPolicy(),
]),
EntityPolicies.anyOf([
new ComponentEntityV1alpha1Policy(),
new GroupEntityV1alpha1Policy(),
new UserEntityV1alpha1Policy(),
new LocationEntityV1alpha1Policy(),
new TemplateEntityV1alpha1Policy(),
new ApiEntityV1alpha1Policy(),
componentEntityV1alpha1Policy,
groupEntityV1alpha1Policy,
userEntityV1alpha1Policy,
locationEntityV1alpha1Policy,
templateEntityV1alpha1Policy,
apiEntityV1alpha1Policy,
]),
]);
}
@@ -98,7 +100,7 @@ export class EntityPolicies implements EntityPolicy {
this.policy = policy;
}
enforce(entity: Entity): Promise<Entity> {
enforce(entity: Entity): Promise<Entity | undefined> {
return this.policy.enforce(entity);
}
}
packages/catalog-model/src/kinds/ApiEntityV1alpha1.test.ts
+6 -9
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
ApiEntityV1alpha1,
ApiEntityV1alpha1Policy,
apiEntityV1alpha1Policy as policy,
} from './ApiEntityV1alpha1';
describe('ApiV1alpha1Policy', () => {
let entity: ApiEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -49,7 +47,7 @@ paths:
'200':
description: A paged array of pets
content:
application/json:
application/json:
schema:
$ref: "#/components/schemas/Pets"
components:
@@ -74,7 +72,6 @@ components:
`,
},
};
policy = new ApiEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -86,14 +83,14 @@ components:
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects missing type', async () => {
packages/catalog-model/src/kinds/ApiEntityV1alpha1.ts
+15 -23
View File
@@ -16,11 +16,24 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'API' as const;
const schema = yup.object<Partial<ApiEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
lifecycle: yup.string().required().min(1),
owner: yup.string().required().min(1),
definition: yup.string().required().min(1),
})
.required(),
});
export interface ApiEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -32,25 +45,4 @@ export interface ApiEntityV1alpha1 extends Entity {
};
}
export class ApiEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<ApiEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
lifecycle: yup.string().required().min(1),
owner: yup.string().required().min(1),
definition: yup.string().required().min(1),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const apiEntityV1alpha1Policy = schemaPolicy(KIND, API_VERSION, schema);
packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts
+5 -8
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
ComponentEntityV1alpha1,
ComponentEntityV1alpha1Policy,
componentEntityV1alpha1Policy as policy,
} from './ComponentEntityV1alpha1';
describe('ComponentV1alpha1Policy', () => {
let entity: ComponentEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -38,7 +36,6 @@ describe('ComponentV1alpha1Policy', () => {
implementsApis: ['api-0'],
},
};
policy = new ComponentEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -50,14 +47,14 @@ describe('ComponentV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects missing type', async () => {
packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts
+19 -23
View File
@@ -16,11 +16,24 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'Component' as const;
const schema = yup.object<Partial<ComponentEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
lifecycle: yup.string().required().min(1),
owner: yup.string().required().min(1),
implementsApis: yup.array(yup.string()).notRequired(),
})
.required(),
});
export interface ComponentEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -32,25 +45,8 @@ export interface ComponentEntityV1alpha1 extends Entity {
};
}
export class ComponentEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<ComponentEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
lifecycle: yup.string().required().min(1),
owner: yup.string().required().min(1),
implementsApis: yup.array(yup.string()).notRequired(),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const componentEntityV1alpha1Policy = schemaPolicy(
KIND,
API_VERSION,
schema,
);
packages/catalog-model/src/kinds/GroupEntityV1alpha1.test.ts
+20 -8
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
GroupEntityV1alpha1,
GroupEntityV1alpha1Policy,
groupEntityV1alpha1Policy as policy,
} from './GroupEntityV1alpha1';
describe('GroupV1alpha1Policy', () => {
let entity: GroupEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -41,7 +39,6 @@ describe('GroupV1alpha1Policy', () => {
descendants: ['desc-a', 'desc-b'],
},
};
policy = new GroupEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -53,14 +50,14 @@ describe('GroupV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects missing type', async () => {
@@ -98,6 +95,11 @@ describe('GroupV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('accepts no ancestors', async () => {
(entity as any).spec.ancestors = [];
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects missing children', async () => {
delete (entity as any).spec.children;
await expect(policy.enforce(entity)).rejects.toThrow(/children/);
@@ -108,6 +110,11 @@ describe('GroupV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('accepts no children', async () => {
(entity as any).spec.children = [];
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects missing descendants', async () => {
delete (entity as any).spec.descendants;
await expect(policy.enforce(entity)).rejects.toThrow(/descendants/);
@@ -117,4 +124,9 @@ describe('GroupV1alpha1Policy', () => {
(entity as any).spec.descendants = [''];
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('accepts no descendants', async () => {
(entity as any).spec.descendants = [];
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
});
packages/catalog-model/src/kinds/GroupEntityV1alpha1.ts
+34 -24
View File
@@ -16,11 +16,39 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'Group' as const;
const schema = yup.object<Partial<GroupEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
parent: yup.string().notRequired().min(1),
// Use these manual tests because yup .required() requires at least
// one element and there is no simple workaround -_-
ancestors: yup.array(yup.string()).test({
name: 'isDefined',
message: 'ancestors must be defined',
test: v => Boolean(v),
}),
children: yup.array(yup.string()).test({
name: 'isDefined',
message: 'children must be defined',
test: v => Boolean(v),
}),
descendants: yup.array(yup.string()).test({
name: 'isDefined',
message: 'descendants must be defined',
test: v => Boolean(v),
}),
})
.required(),
});
export interface GroupEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -33,26 +61,8 @@ export interface GroupEntityV1alpha1 extends Entity {
};
}
export class GroupEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<GroupEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
parent: yup.string().notRequired().min(1),
ancestors: yup.array(yup.string()).required(),
children: yup.array(yup.string()).required(),
descendants: yup.array(yup.string()).required(),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const groupEntityV1alpha1Policy = schemaPolicy(
KIND,
API_VERSION,
schema,
);
packages/catalog-model/src/kinds/LocationEntityV1alpha1.test.ts
+5 -8
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
LocationEntityV1alpha1,
LocationEntityV1alpha1Policy,
locationEntityV1alpha1Policy as policy,
} from './LocationEntityV1alpha1';
describe('LocationV1alpha1Policy', () => {
let entity: LocationEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -35,7 +33,6 @@ describe('LocationV1alpha1Policy', () => {
type: 'github',
},
};
policy = new LocationEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -47,14 +44,14 @@ describe('LocationV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects missing type', async () => {
packages/catalog-model/src/kinds/LocationEntityV1alpha1.ts
+18 -22
View File
@@ -16,11 +16,23 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'Location' as const;
const schema = yup.object<Partial<LocationEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
target: yup.string().notRequired().min(1),
targets: yup.array(yup.string()).notRequired(),
})
.required(),
});
export interface LocationEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -31,24 +43,8 @@ export interface LocationEntityV1alpha1 extends Entity {
};
}
export class LocationEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<LocationEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
target: yup.string().notRequired().min(1),
targets: yup.array(yup.string()).notRequired(),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const locationEntityV1alpha1Policy = schemaPolicy(
KIND,
API_VERSION,
schema,
);
packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts
+6 -9
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
TemplateEntityV1alpha1,
TemplateEntityV1alpha1Policy,
templateEntityV1alpha1Policy as policy,
} from './TemplateEntityV1alpha1';
describe('TemplateEntityV1alpah1', () => {
describe('templateEntityV1alpha1', () => {
let entity: TemplateEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -52,7 +50,6 @@ describe('TemplateEntityV1alpah1', () => {
},
},
};
policy = new TemplateEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -64,14 +61,14 @@ describe('TemplateEntityV1alpah1', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects missing type', async () => {
packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts
+20 -23
View File
@@ -16,11 +16,25 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy, JSONSchema } from '../types';
import type { JSONSchema } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'Template' as const;
const schema = yup.object<Partial<TemplateEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
path: yup.string(),
schema: yup.object().required(),
templater: yup.string().required(),
})
.required(),
});
export interface TemplateEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -32,25 +46,8 @@ export interface TemplateEntityV1alpha1 extends Entity {
};
}
export class TemplateEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<TemplateEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
type: yup.string().required().min(1),
path: yup.string(),
schema: yup.object().required(),
templater: yup.string().required(),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const templateEntityV1alpha1Policy = schemaPolicy(
KIND,
API_VERSION,
schema,
);
packages/catalog-model/src/kinds/UserEntityV1alpha1.test.ts
+16 -9
View File
@@ -14,15 +14,13 @@
* limitations under the License.
*/
import { EntityPolicy } from '../types';
import {
UserEntityV1alpha1,
UserEntityV1alpha1Policy,
userEntityV1alpha1Policy as policy,
} from './UserEntityV1alpha1';
describe('UserV1alpha1Policy', () => {
describe('userEntityV1alpha1Policy', () => {
let entity: UserEntityV1alpha1;
let policy: EntityPolicy;
beforeEach(() => {
entity = {
@@ -40,7 +38,6 @@ describe('UserV1alpha1Policy', () => {
memberOf: ['team-a', 'developers'],
},
};
policy = new UserEntityV1alpha1Policy();
});
it('happy path: accepts valid data', async () => {
@@ -54,14 +51,14 @@ describe('UserV1alpha1Policy', () => {
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects unknown apiVersion', async () => {
it('ignores unknown apiVersion', async () => {
(entity as any).apiVersion = 'backstage.io/v1beta0';
await expect(policy.enforce(entity)).rejects.toThrow(/apiVersion/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('rejects unknown kind', async () => {
it('ignores unknown kind', async () => {
(entity as any).kind = 'Wizard';
await expect(policy.enforce(entity)).rejects.toThrow(/kind/);
await expect(policy.enforce(entity)).resolves.toBeUndefined();
});
it('spec accepts unknown additional fields', async () => {
@@ -147,4 +144,14 @@ describe('UserV1alpha1Policy', () => {
(entity as any).spec.memberOf[0] = 7;
await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
});
it('accepts empty memberOf', async () => {
(entity as any).spec.memberOf = [];
await expect(policy.enforce(entity)).resolves.toBe(entity);
});
it('rejects null memberOf', async () => {
(entity as any).spec.memberOf = null;
await expect(policy.enforce(entity)).rejects.toThrow(/memberOf/);
});
});
packages/catalog-model/src/kinds/UserEntityV1alpha1.ts
+25 -27
View File
@@ -16,11 +16,34 @@
import * as yup from 'yup';
import type { Entity } from '../entity/Entity';
import type { EntityPolicy } from '../types';
import { schemaPolicy } from './util';
const API_VERSION = ['backstage.io/v1alpha1', 'backstage.io/v1beta1'] as const;
const KIND = 'User' as const;
const schema = yup.object<Partial<UserEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
profile: yup
.object({
displayName: yup.string().min(1).notRequired(),
email: yup.string().min(1).notRequired(),
picture: yup.string().min(1).notRequired(),
})
.notRequired(),
// Use this manual test because yup .required() requires at least one
// element and there is no simple workaround -_-
memberOf: yup.array(yup.string()).test({
name: 'isDefined',
message: 'memberOf must be defined',
test: v => Boolean(v),
}),
})
.required(),
});
export interface UserEntityV1alpha1 extends Entity {
apiVersion: typeof API_VERSION[number];
kind: typeof KIND;
@@ -34,29 +57,4 @@ export interface UserEntityV1alpha1 extends Entity {
};
}
export class UserEntityV1alpha1Policy implements EntityPolicy {
private schema: yup.Schema<any>;
constructor() {
this.schema = yup.object<Partial<UserEntityV1alpha1>>({
apiVersion: yup.string().required().oneOf(API_VERSION),
kind: yup.string().required().equals([KIND]),
spec: yup
.object({
profile: yup
.object({
displayName: yup.string().min(1).notRequired(),
email: yup.string().min(1).notRequired(),
picture: yup.string().min(1).notRequired(),
})
.notRequired(),
memberOf: yup.array(yup.string()).required(),
})
.required(),
});
}
async enforce(envelope: Entity): Promise<Entity> {
return await this.schema.validate(envelope, { strict: true });
}
}
export const userEntityV1alpha1Policy = schemaPolicy(KIND, API_VERSION, schema);
packages/catalog-model/src/kinds/index.ts
+6 -6
View File
@@ -14,32 +14,32 @@
* limitations under the License.
*/
export { ApiEntityV1alpha1Policy } from './ApiEntityV1alpha1';
export { apiEntityV1alpha1Policy } from './ApiEntityV1alpha1';
export type {
ApiEntityV1alpha1 as ApiEntity,
ApiEntityV1alpha1,
} from './ApiEntityV1alpha1';
export { ComponentEntityV1alpha1Policy } from './ComponentEntityV1alpha1';
export { componentEntityV1alpha1Policy } from './ComponentEntityV1alpha1';
export type {
ComponentEntityV1alpha1 as ComponentEntity,
ComponentEntityV1alpha1,
} from './ComponentEntityV1alpha1';
export { GroupEntityV1alpha1Policy } from './GroupEntityV1alpha1';
export { groupEntityV1alpha1Policy } from './GroupEntityV1alpha1';
export type {
GroupEntityV1alpha1 as GroupEntity,
GroupEntityV1alpha1,
} from './GroupEntityV1alpha1';
export { LocationEntityV1alpha1Policy } from './LocationEntityV1alpha1';
export { locationEntityV1alpha1Policy } from './LocationEntityV1alpha1';
export type {
LocationEntityV1alpha1 as LocationEntity,
LocationEntityV1alpha1,
} from './LocationEntityV1alpha1';
export { TemplateEntityV1alpha1Policy } from './TemplateEntityV1alpha1';
export { templateEntityV1alpha1Policy } from './TemplateEntityV1alpha1';
export type {
TemplateEntityV1alpha1 as TemplateEntity,
TemplateEntityV1alpha1,
} from './TemplateEntityV1alpha1';
export { UserEntityV1alpha1Policy } from './UserEntityV1alpha1';
export { userEntityV1alpha1Policy } from './UserEntityV1alpha1';
export type {
UserEntityV1alpha1 as UserEntity,
UserEntityV1alpha1,
packages/catalog-model/src/kinds/util.ts
+37
View File
@@ -0,0 +1,37 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as yup from 'yup';
import { Entity } from '../entity';
import { EntityPolicy } from '../types';
export function schemaPolicy(
kind: string,
apiVersion: readonly string[],
schema: yup.Schema<any>,
): EntityPolicy {
return {
async enforce(envelope: Entity): Promise<Entity | undefined> {
if (
kind !== envelope.kind ||
!apiVersion.includes(envelope.apiVersion as any)
) {
return undefined;
}
return await schema.validate(envelope, { strict: true });
},
};
}
packages/catalog-model/src/types.ts
+3 -2
View File
@@ -27,10 +27,11 @@ export type EntityPolicy = {
* Applies validation or mutation on an entity.
*
* @param entity The entity, as validated/mutated so far in the policy tree
* @returns The incoming entity, or a mutated version of the same
* @returns The incoming entity, or a mutated version of the same, or
* undefined if this processor could not handle the entity
* @throws An error if the entity should be rejected
*/
enforce(entity: Entity): Promise<Entity>;
enforce(entity: Entity): Promise<Entity | undefined>;
};
export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue };
packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.test.tsx.hbs
+1 -2
View File
@@ -10,7 +10,7 @@ import { setupServer } from 'msw/node';
describe('ExampleComponent', () => {
const server = setupServer();
// Enable API mocking before tests.
beforeAll(() => server.listen())
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }))
// Reset any runtime request handlers we may add during the tests.
afterEach(() => server.resetHandlers())
@@ -32,4 +32,3 @@ describe('ExampleComponent', () => {
expect(rendered.getByText('Welcome to {{ id }}!')).toBeInTheDocument();
});
});
packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx.hbs
+1 -1
View File
@@ -7,7 +7,7 @@ import { setupServer } from 'msw/node';
describe('ExampleFetchComponent', () => {
const server = setupServer();
// Enable API mocking before tests.
beforeAll(() => server.listen())
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }))
// Reset any runtime request handlers we may add during the tests.
afterEach(() => server.resetHandlers())
packages/create-app/templates/default-app/app-config.yaml.hbs
+1 -1
View File
@@ -66,7 +66,7 @@ scaffolder:
catalog:
rules:
- allow: [Component, API, Group, Template, Location]
- allow: [Component, API, Group, User, Template, Location]
processors:
github:
providers:
plugins/catalog-backend/package.json
+1
View File
@@ -23,6 +23,7 @@
"@backstage/backend-common": "^0.1.1-alpha.23",
"@backstage/catalog-model": "^0.1.1-alpha.23",
"@backstage/config": "^0.1.1-alpha.23",
"@octokit/graphql": "^4.5.6",
"@types/express": "^4.17.6",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+15 -1
View File
@@ -19,6 +19,7 @@ import {
Entity,
EntityPolicies,
EntityPolicy,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
} from '@backstage/catalog-model';
import { Config, ConfigReader } from '@backstage/config';
@@ -30,6 +31,7 @@ import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
import { FileReaderProcessor } from './processors/FileReaderProcessor';
import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
@@ -86,6 +88,7 @@ export class LocationReaders implements LocationReader {
new GitlabReaderProcessor(),
new BitbucketApiReaderProcessor(config),
new AzureApiReaderProcessor(config),
GithubOrgReaderProcessor.fromConfig(config),
new UrlReaderProcessor(),
new YamlProcessor(),
PlaceholderProcessor.default(),
@@ -178,12 +181,14 @@ export class LocationReaders implements LocationReader {
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
}
}
}
const message = `No processor was able to read location ${item.location.type} ${item.location.target}`;
emit(result.inputError(item.location, message));
this.logger.warn(message);
}
private async handleData(
@@ -203,6 +208,7 @@ export class LocationReaders implements LocationReader {
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
}
}
}
@@ -231,8 +237,15 @@ export class LocationReaders implements LocationReader {
this.readLocation.bind(this),
);
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while processing entity at ${item.location.type} ${item.location.target}, ${e}`;
// Construct the name carefully, if we got validation errors we do
// not want to crash here due to missing metadata or so
const namespace = !current.metadata
? ''
: current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
const name = !current.metadata ? '' : current.metadata.name;
const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
}
}
}
@@ -255,6 +268,7 @@ export class LocationReaders implements LocationReader {
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
this.logger.warn(message);
}
}
}
plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts
+5 -1
View File
@@ -25,6 +25,10 @@ export class EntityPolicyProcessor implements LocationProcessor {
}
async processEntity(entity: Entity): Promise<Entity> {
return await this.policy.enforce(entity);
const output = await this.policy.enforce(entity);
if (!output) {
throw new Error(`Entity did not match any known schema`);
}
return output;
}
}
plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts
+160
View File
@@ -0,0 +1,160 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { graphql } from '@octokit/graphql';
import * as results from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { getOrganizationTeams, getOrganizationUsers } from './util/github';
import { buildOrgHierarchy } from './util/org';
/**
* Extracts teams and users out of a GitHub org.
*/
export class GithubOrgReaderProcessor implements LocationProcessor {
static fromConfig(config: Config) {
return new GithubOrgReaderProcessor(readConfig(config));
}
constructor(private readonly providers: ProviderConfig[]) {}
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: LocationProcessorEmit,
): Promise<boolean> {
if (location.type !== 'github-org') {
return false;
}
const provider = this.providers.find(p =>
location.target.startsWith(`${p.target}/`),
);
if (!provider) {
throw new Error(
`There is no GitHub Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.githubOrg.providers.`,
);
}
const { org } = parseUrl(location.target);
const client = !provider.token
? graphql
: graphql.defaults({
headers: {
authorization: `token ${provider.token}`,
},
});
const { users } = await getOrganizationUsers(client, org);
const { groups, groupMemberUsers } = await getOrganizationTeams(
client,
org,
);
buildOrgHierarchy(groups, users, groupMemberUsers);
for (const group of groups) {
emit(results.entity(location, group));
}
for (const user of users) {
emit(results.entity(location, user));
}
return true;
}
}
/*
* Helpers
*/
/**
* The configuration parameters for a single GitHub API provider.
*/
type ProviderConfig = {
/**
* The prefix of the target that this matches on, e.g. "https://github.com",
* with no trailing slash.
*/
target: string;
/**
* The base URL of the API of this provider, e.g. "https://api.github.com",
* with no trailing slash.
*
* May be omitted specifically for GitHub; then it will be deduced.
*/
apiBaseUrl?: string;
/**
* The authorization token to use for requests to this provider.
*
* If no token is specified, anonymous access is used.
*/
token?: string;
};
// TODO(freben): Break out common code and config from here and GithubReaderProcessor
export function readConfig(config: Config): ProviderConfig[] {
const providers: ProviderConfig[] = [];
const providerConfigs =
config.getOptionalConfigArray('catalog.processors.githubOrg.providers') ??
[];
// First read all the explicit providers
for (const providerConfig of providerConfigs) {
const target = providerConfig.getString('target').replace(/\/+$/, '');
let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
const token = providerConfig.getOptionalString('token');
if (apiBaseUrl) {
apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
} else if (target === 'https://github.com') {
apiBaseUrl = 'https://api.github.com';
}
if (!apiBaseUrl) {
throw new Error(
`Provider at ${target} must configure an explicit apiBaseUrl`,
);
}
providers.push({ target, apiBaseUrl, token });
}
// If no explicit github.com provider was added, put one in the list as
// a convenience
if (!providers.some(p => p.target === 'https://github.com')) {
providers.push({
target: 'https://github.com',
apiBaseUrl: 'https://api.github.com',
});
}
return providers;
}
export function parseUrl(urlString: string): { org: string } {
const path = new URL(urlString).pathname.substr(1).split('/');
// /spotify
if (path.length === 1 && path[0].length) {
return { org: decodeURIComponent(path[0]) };
}
throw new Error(`Expected a URL pointing to /<org>`);
}
plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts
+1 -1
View File
@@ -27,7 +27,7 @@ describe('UrlReaderProcessor', () => {
const mockApiOrigin = 'http://localhost:23000';
const server = setupServer();
beforeAll(() => server.listen());
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
afterEach(() => server.resetHandlers());
afterAll(() => server.close());
plugins/catalog-backend/src/ingestion/processors/util/github.test.ts
+147
View File
@@ -0,0 +1,147 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { graphql } from '@octokit/graphql';
import { graphql as graphqlMsw } from 'msw';
import { setupServer } from 'msw/node';
import {
getOrganizationTeams,
getOrganizationUsers,
getTeamMembers,
QueryResponse,
} from './github';
describe('github', () => {
const server = setupServer();
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
afterEach(() => server.resetHandlers());
afterAll(() => server.close());
describe('getOrganizationUsers', () => {
it('reads members', async () => {
const input: QueryResponse = {
organization: {
membersWithRole: {
pageInfo: { hasNextPage: false },
nodes: [
{
login: 'a',
name: 'b',
bio: 'c',
email: 'd',
avatarUrl: 'e',
},
],
},
},
};
const output = {
users: [
expect.objectContaining({
metadata: expect.objectContaining({ name: 'a', description: 'c' }),
spec: {
profile: { displayName: 'b', email: 'd', picture: 'e' },
memberOf: [],
},
}),
],
};
server.use(
graphqlMsw.query('users', (_req, res, ctx) => res(ctx.data(input))),
);
await expect(getOrganizationUsers(graphql, 'a')).resolves.toEqual(output);
});
});
describe('getOrganizationTeams', () => {
it('reads teams', async () => {
const input: QueryResponse = {
organization: {
teams: {
pageInfo: { hasNextPage: false },
nodes: [
{
slug: 'team',
combinedSlug: 'blah/team',
parentTeam: {
slug: 'parent',
combinedSlug: '',
members: { pageInfo: { hasNextPage: false }, nodes: [] },
},
members: {
pageInfo: { hasNextPage: false },
nodes: [{ login: 'user' }],
},
},
],
},
},
};
const output = {
groups: [
expect.objectContaining({
metadata: expect.objectContaining({ name: 'team' }),
spec: {
type: 'team',
parent: 'parent',
ancestors: [],
children: [],
descendants: [],
},
}),
],
groupMemberUsers: new Map([['team', ['user']]]),
};
server.use(
graphqlMsw.query('teams', (_req, res, ctx) => res(ctx.data(input))),
);
await expect(getOrganizationTeams(graphql, 'a')).resolves.toEqual(output);
});
});
describe('getTeamMembers', () => {
it('reads team members', async () => {
const input: QueryResponse = {
organization: {
team: {
slug: '',
combinedSlug: '',
members: {
pageInfo: { hasNextPage: false },
nodes: [{ login: 'user' }],
},
},
},
};
const output = {
members: ['user'],
};
server.use(
graphqlMsw.query('members', (_req, res, ctx) => res(ctx.data(input))),
);
await expect(getTeamMembers(graphql, 'a', 'b')).resolves.toEqual(output);
});
});
});
plugins/catalog-backend/src/ingestion/processors/util/github.ts
+297
View File
@@ -0,0 +1,297 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { graphql } from '@octokit/graphql';
// Graphql types
export type QueryResponse = {
organization: Organization;
};
export type Organization = {
membersWithRole?: Connection<User>;
team?: Team;
teams?: Connection<Team>;
};
export type PageInfo = {
hasNextPage: boolean;
endCursor?: string;
};
export type User = {
login: string;
bio?: string;
avatarUrl?: string;
email?: string;
name?: string;
};
export type Team = {
slug: string;
combinedSlug: string;
description?: string;
parentTeam?: Team;
members: Connection<User>;
};
export type Connection<T> = {
pageInfo: PageInfo;
nodes: T[];
};
/**
* Gets all the users out of a GitHub organization.
*
* Note that the users will not have their memberships filled in.
*
* @param client An octokit graphql client
* @param org The slug of the org to read
*/
export async function getOrganizationUsers(
client: typeof graphql,
org: string,
): Promise<{ users: UserEntity[] }> {
const query = `
query users($org: String!, $cursor: String) {
organization(login: $org) {
membersWithRole(first: 100, after: $cursor) {
pageInfo { hasNextPage, endCursor }
nodes { avatarUrl, bio, email, login, name }
}
}
}`;
// There is no user -> teams edge, so we leave the memberships empty for
// now and let the team iteration handle it instead
const mapper = (user: User) => {
const entity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: user.login,
annotations: {
'github.com/user-login': user.login,
},
},
spec: {
profile: {},
memberOf: [],
},
};
if (user.bio) entity.metadata.description = user.bio;
if (user.name) entity.spec.profile!.displayName = user.name;
if (user.email) entity.spec.profile!.email = user.email;
if (user.avatarUrl) entity.spec.profile!.picture = user.avatarUrl;
return entity;
};
const users = await queryWithPaging(
client,
query,
r => r.organization?.membersWithRole,
mapper,
{ org },
);
return { users };
}
/**
* Gets all the teams out of a GitHub organization.
*
* Note that the teams will not have any relations apart from parent filled in.
*
* @param client An octokit graphql client
* @param org The slug of the org to read
*/
export async function getOrganizationTeams(
client: typeof graphql,
org: string,
): Promise<{
groups: GroupEntity[];
groupMemberUsers: Map<string, string[]>;
}> {
const query = `
query teams($org: String!, $cursor: String) {
organization(login: $org) {
teams(first: 100, after: $cursor) {
pageInfo { hasNextPage, endCursor }
nodes {
slug
combinedSlug
parentTeam { slug }
members(first: 100, membership: IMMEDIATE) {
pageInfo { hasNextPage }
nodes { login }
}
}
}
}
}`;
// Gets populated inside the mapper below
const groupMemberUsers = new Map<string, string[]>();
const mapper = async (team: Team) => {
const entity: GroupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: team.slug,
annotations: {
'github.com/team-slug': team.combinedSlug,
},
},
spec: {
type: 'team',
ancestors: [],
children: [],
descendants: [],
},
};
if (team.description) entity.metadata.description = team.description;
if (team.parentTeam) entity.spec.parent = team.parentTeam.slug;
const memberNames: string[] = [];
groupMemberUsers.set(team.slug, memberNames);
if (!team.members.pageInfo.hasNextPage) {
// We got all the members in one go, run the fast path
for (const user of team.members.nodes) {
memberNames.push(user.login);
}
} else {
// There were more than a hundred immediate members - run the slow
// path of fetching them explicitly
const { members } = await getTeamMembers(client, org, team.slug);
for (const userLogin of members) {
memberNames.push(userLogin);
}
}
return entity;
};
const groups = await queryWithPaging(
client,
query,
r => r.organization?.teams,
mapper,
{ org },
);
return { groups, groupMemberUsers };
}
/**
* Gets all the users out of a GitHub organization.
*
* Note that the users will not have their memberships filled in.
*
* @param client An octokit graphql client
* @param org The slug of the org to read
* @param teamSlug The slug of the team to read
*/
export async function getTeamMembers(
client: typeof graphql,
org: string,
teamSlug: string,
): Promise<{ members: string[] }> {
const query = `
query members($org: String!, $teamSlug: String!, $cursor: String) {
organization(login: $org) {
team(slug: $teamSlug) {
members(first: 100, after: $cursor, membership: IMMEDIATE) {
pageInfo { hasNextPage, endCursor }
nodes { login }
}
}
}
}`;
const members = await queryWithPaging(
client,
query,
r => r.organization?.team?.members,
user => user.login,
{ org, teamSlug },
);
return { members };
}
//
// Helpers
//
/**
* Assists in repeatedly executing a query with a paged response.
*
* Requires that the query accepts a $cursor variable.
*
* @param client The octokit client
* @param query The query to execute
* @param connection A function that, given the response, picks out the actual
* Connection object that's being iterated
* @param mapper A function that, given one of the nodes in the Connection,
* returns the model mapped form of it
* @param variables The variable values that the query needs, minus the cursor
*/
export async function queryWithPaging<
GraphqlType,
OutputType,
Variables extends {},
Response = QueryResponse
>(
client: typeof graphql,
query: string,
connection: (response: Response) => Connection<GraphqlType> | undefined,
mapper: (item: GraphqlType) => Promise<OutputType> | OutputType,
variables: Variables,
): Promise<OutputType[]> {
const result: OutputType[] = [];
let cursor: string | undefined = undefined;
for (let j = 0; j < 1000 /* just for sanity */; ++j) {
const response: Response = await client(query, {
...variables,
cursor,
});
const conn = connection(response);
if (!conn) {
throw new Error(`Found no match for ${JSON.stringify(variables)}`);
}
for (const node of conn.nodes) {
result.push(await mapper(node));
}
if (!conn.pageInfo.hasNextPage) {
break;
} else {
cursor = conn.pageInfo.endCursor;
}
}
return result;
}
plugins/catalog-backend/src/ingestion/processors/util/org.test.ts
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { buildOrgHierarchy } from './org';
function u(name: string): UserEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name },
spec: { memberOf: [] },
};
}
function g(
name: string,
parent: string | undefined,
children: string[],
): GroupEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: { name },
spec: { type: 'team', parent, children, ancestors: [], descendants: [] },
};
}
describe('buildOrgHierarchy', () => {
it('puts users in the respective groups', () => {
const a = g('a', undefined, []);
const b = g('b', undefined, []);
const x = u('x');
const y = u('y');
const groupMemberUsers: Map<string, string[]> = new Map([
['a', ['x', 'y']],
['b', ['y']],
]);
buildOrgHierarchy([a, b], [x, y], groupMemberUsers);
expect(x.spec.memberOf).toEqual(['a']);
expect(y.spec.memberOf).toEqual(['a', 'b']);
});
it('adds groups to their parent.children', () => {
const a = g('a', undefined, []);
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd']));
expect(b.spec.children).toEqual(expect.arrayContaining(['c']));
expect(c.spec.children).toEqual([]);
expect(d.spec.children).toEqual([]);
});
it('fills out descendants', () => {
const a = g('a', undefined, []);
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
expect(a.spec.descendants).toEqual(expect.arrayContaining(['b', 'c', 'd']));
expect(b.spec.descendants).toEqual(expect.arrayContaining(['c']));
expect(c.spec.descendants).toEqual([]);
expect(d.spec.descendants).toEqual([]);
});
it('fills out ancestors', () => {
const a = g('a', undefined, []);
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
expect(a.spec.ancestors).toEqual([]);
expect(b.spec.ancestors).toEqual(expect.arrayContaining(['a']));
expect(c.spec.ancestors).toEqual(expect.arrayContaining(['a', 'b']));
expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a']));
});
});
plugins/catalog-backend/src/ingestion/processors/util/org.ts
+113
View File
@@ -0,0 +1,113 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
export function buildOrgHierarchy(
groups: GroupEntity[],
users: UserEntity[],
groupMemberUsers: Map<string, string[]>,
) {
const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
const usersByName = new Map(users.map(u => [u.metadata.name, u]));
//
// Make sure that u.memberOf contain all g
//
for (const [groupName, userNames] of groupMemberUsers.entries()) {
for (const userName of userNames) {
const user = usersByName.get(userName);
if (user && !user.spec.memberOf.includes(groupName)) {
user.spec.memberOf.push(groupName);
}
}
}
//
// Make sure that g.parent.children contain g
//
for (const group of groups) {
const selfName = group.metadata.name;
const parentName = group.spec.parent;
if (parentName) {
const parent = groupsByName.get(parentName);
if (parent && !parent.spec.children.includes(selfName)) {
parent.spec.children.push(selfName);
}
}
}
//
// Make sure that g.descendants is complete
//
function visitDescendants(current: GroupEntity): string[] {
if (current.spec.descendants.length) {
return current.spec.descendants;
}
const accumulator = new Set<string>();
for (const childName of current.spec.children) {
accumulator.add(childName);
const child = groupsByName.get(childName);
if (child) {
for (const d of visitDescendants(child)) {
accumulator.add(d);
}
}
}
const descendants = Array.from(accumulator);
current.spec.descendants = descendants;
return descendants;
}
for (const group of groups) {
visitDescendants(group);
}
//
// Make sure that g.ancestors is complete
//
function visitAncestors(current: GroupEntity): string[] {
if (current.spec.ancestors.length) {
return current.spec.ancestors;
}
let ancestors: string[];
const parentName = current.spec.parent;
if (!parentName) {
ancestors = [];
} else {
const parent = groupsByName.get(parentName);
if (parent) {
ancestors = [parentName, ...visitAncestors(parent)];
} else {
ancestors = [parentName];
}
}
current.spec.ancestors = ancestors;
return ancestors;
}
for (const group of groups) {
visitAncestors(group);
}
}
plugins/catalog-graphql/src/graphql/module.test.ts
+2 -2
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { createModule } from './module';
import { execute } from 'graphql';
import { rest } from 'msw';
@@ -36,9 +37,8 @@ describe('Catalog Module', () => {
},
]);
beforeAll(() => worker.listen());
beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
afterAll(() => worker.close());
afterEach(() => worker.resetHandlers());
describe('Default Entity', () => {
plugins/catalog-graphql/src/service/client.test.ts
+1 -2
View File
@@ -20,9 +20,8 @@ import { setupServer } from 'msw/node';
describe('Catalog GraphQL Module', () => {
const worker = setupServer();
beforeAll(() => worker.listen());
beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
afterAll(() => worker.close());
afterEach(() => worker.resetHandlers());
const baseUrl = 'http://localhost:1234';
plugins/catalog/src/api/CatalogClient.test.ts
+1 -1
View File
@@ -25,7 +25,7 @@ const mockBaseUrl = 'http://backstage:9191/i-am-a-mock-base';
const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
describe('CatalogClient', () => {
beforeAll(() => server.listen());
beforeAll(() => server.listen({ onUnhandledRequest: 'error' }));
afterEach(() => server.resetHandlers());
afterAll(() => server.close());
yarn.lock
+14
View File
@@ -3393,6 +3393,15 @@
"@octokit/types" "^5.0.0"
universal-user-agent "^5.0.0"
"@octokit/graphql@^4.5.6":
version "4.5.6"
resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.6.tgz#708143ba15cf7c1879ed6188266e7f270be805d4"
integrity sha512-Rry+unqKTa3svswT2ZAuqenpLrzJd+JTv89LTeVa5UM/5OX8o4KTkPL7/1ABq4f/ZkELb0XEK/2IEoYwykcLXg==
dependencies:
"@octokit/request" "^5.3.0"
"@octokit/types" "^5.0.0"
universal-user-agent "^6.0.0"
"@octokit/plugin-enterprise-rest@^6.0.1":
version "6.0.1"
resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437"
@@ -22781,6 +22790,11 @@ universal-user-agent@^5.0.0:
dependencies:
os-name "^3.1.0"
universal-user-agent@^6.0.0:
version "6.0.0"
resolved "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee"
integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==
universalify@^0.1.0:
version "0.1.2"
resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"