From 90bd5ab9e7fa086b14a6f5b736fdaea0c5275e55 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 17 Jun 2021 19:19:42 +0200 Subject: [PATCH 01/17] chore: updating the api-docs as some PR's were merged after the api-docs PR merge that were not rebuilt Signed-off-by: blam --- plugins/auth-backend/api-report.md | 21 +++++++++++++++++++++ plugins/explore/api-report.md | 29 ++++++++++++++++++++++++++++- plugins/techdocs/api-report.md | 7 +++++-- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/plugins/auth-backend/api-report.md b/plugins/auth-backend/api-report.md index dea5500d53..4788b9c102 100644 --- a/plugins/auth-backend/api-report.md +++ b/plugins/auth-backend/api-report.md @@ -6,12 +6,14 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JSONWebKey } from 'jose'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Profile } from 'passport'; +import { UserEntity } from '@backstage/catalog-model'; // @public (undocumented) export type AuthProviderFactory = (options: AuthProviderFactoryOptions) => AuthProviderRouteHandlers; @@ -47,8 +49,13 @@ export type AuthResponse = { export type BackstageIdentity = { id: string; idToken?: string; + token?: string; + entity?: Entity; }; +// @public (undocumented) +export const createGoogleProvider: (options?: GoogleProviderOptions | undefined) => AuthProviderFactory; + // @public (undocumented) export function createRouter({ logger, config, discovery, database, providerFactories, }: RouterOptions): Promise; @@ -63,6 +70,20 @@ export const encodeState: (state: OAuthState) => string; // @public (undocumented) export const ensuresXRequestedWith: (req: express.Request) => boolean; +// @public (undocumented) +export const googleDefaultSignInResolver: SignInResolver; + +// @public (undocumented) +export const googleEmailSignInResolver: SignInResolver; + +// @public (undocumented) +export type GoogleProviderOptions = { + authHandler?: AuthHandler; + signIn?: { + resolver?: SignInResolver; + }; +}; + // @public export class IdentityClient { constructor(options: { diff --git a/plugins/explore/api-report.md b/plugins/explore/api-report.md index 590af53de8..4116247061 100644 --- a/plugins/explore/api-report.md +++ b/plugins/explore/api-report.md @@ -5,8 +5,10 @@ ```ts import { BackstagePlugin } from '@backstage/core'; +import { default } from 'react'; import { ExternalRouteRef } from '@backstage/core'; import { RouteRef } from '@backstage/core'; +import { TabProps } from '@material-ui/core'; // @public (undocumented) export const catalogEntityRouteRef: ExternalRouteRef<{ @@ -15,11 +17,22 @@ export const catalogEntityRouteRef: ExternalRouteRef<{ namespace: string; }, false>; +// @public (undocumented) +export const DomainExplorerContent: ({ title, }: { + title?: string | undefined; +}) => JSX.Element; + +// @public +export const ExploreLayout: { + ({ title, subtitle, children, }: ExploreLayoutProps): JSX.Element; + Route: (props: SubRoute) => null; +}; + // @public (undocumented) export const ExplorePage: () => JSX.Element; // @public (undocumented) -export const explorePlugin: BackstagePlugin<{ +const explorePlugin: BackstagePlugin<{ explore: RouteRef; }, { catalogEntity: ExternalRouteRef<{ @@ -29,9 +42,23 @@ export const explorePlugin: BackstagePlugin<{ }, false>; }>; +export { explorePlugin } + +export { explorePlugin as plugin } + // @public (undocumented) export const exploreRouteRef: RouteRef; +// @public (undocumented) +export const GroupsExplorerContent: ({ title, }: { + title?: string | undefined; +}) => JSX.Element; + +// @public (undocumented) +export const ToolExplorerContent: ({ title }: { + title?: string | undefined; +}) => JSX.Element; + // (No @packageDocumentation comment for this package) diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 801eabf368..69a62b4362 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -43,6 +43,9 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element; // @public (undocumented) export const Router: () => JSX.Element; +// @public (undocumented) +export type SyncResult = 'cached' | 'updated' | 'timeout'; + // @public (undocumented) export interface TechDocsApi { // (undocumented) @@ -109,7 +112,7 @@ export interface TechDocsStorageApi { // (undocumented) getStorageUrl(): Promise; // (undocumented) - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName): Promise; } // @public (undocumented) @@ -137,7 +140,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName): Promise; } From 2d881016cc8f4262243851ced4646fb0dc3800aa Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 18 Jun 2021 10:03:01 +0200 Subject: [PATCH 02/17] chore: dont export default auth provider Signed-off-by: blam Signed-off-by: blam --- plugins/auth-backend/src/providers/google/index.ts | 6 +----- plugins/auth-backend/src/providers/google/provider.ts | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/auth-backend/src/providers/google/index.ts b/plugins/auth-backend/src/providers/google/index.ts index 8bff8b250b..61805b2dcd 100644 --- a/plugins/auth-backend/src/providers/google/index.ts +++ b/plugins/auth-backend/src/providers/google/index.ts @@ -14,9 +14,5 @@ * limitations under the License. */ -export { - createGoogleProvider, - googleDefaultSignInResolver, - googleEmailSignInResolver, -} from './provider'; +export { createGoogleProvider, googleEmailSignInResolver } from './provider'; export type { GoogleProviderOptions } from './provider'; diff --git a/plugins/auth-backend/src/providers/google/provider.ts b/plugins/auth-backend/src/providers/google/provider.ts index ace0c8e132..b47399e1c6 100644 --- a/plugins/auth-backend/src/providers/google/provider.ts +++ b/plugins/auth-backend/src/providers/google/provider.ts @@ -198,7 +198,7 @@ export const googleEmailSignInResolver: SignInResolver = async ( return { id: entity.metadata.name, entity, token }; }; -export const googleDefaultSignInResolver: SignInResolver = async ( +const googleDefaultSignInResolver: SignInResolver = async ( info, ctx, ) => { From 12942d16c48469a1511f6c0596c9f4e228949a74 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 18 Jun 2021 10:06:27 +0200 Subject: [PATCH 03/17] chore: updating api-report Signed-off-by: blam --- plugins/auth-backend/api-report.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/auth-backend/api-report.md b/plugins/auth-backend/api-report.md index 4788b9c102..6ae878accf 100644 --- a/plugins/auth-backend/api-report.md +++ b/plugins/auth-backend/api-report.md @@ -70,9 +70,6 @@ export const encodeState: (state: OAuthState) => string; // @public (undocumented) export const ensuresXRequestedWith: (req: express.Request) => boolean; -// @public (undocumented) -export const googleDefaultSignInResolver: SignInResolver; - // @public (undocumented) export const googleEmailSignInResolver: SignInResolver; From 36e9a40849457a012067ab17f8da7f8bbda83d91 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 18 Jun 2021 10:09:35 +0200 Subject: [PATCH 04/17] chore: removing export Signed-off-by: blam --- .changeset/perfect-gifts-compete.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/perfect-gifts-compete.md diff --git a/.changeset/perfect-gifts-compete.md b/.changeset/perfect-gifts-compete.md new file mode 100644 index 0000000000..7c03798888 --- /dev/null +++ b/.changeset/perfect-gifts-compete.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Don't export the `defaultGoogleAuthProvider` From 127048f92be4611747a6ac83a46dc7bcd8af0d2e Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 28 May 2021 18:38:45 +0200 Subject: [PATCH 05/17] Split `MicrosoftGraphOrgReaderProcessor` into a separate package and make it customizable Signed-off-by: Oliver Sand --- .changeset/metal-badgers-carry.md | 17 + .changeset/silent-ways-laugh.md | 6 + packages/backend/package.json | 2 + .../.eslintrc.js | 3 + .../README.md | 86 ++++ .../config.d.ts | 80 ++++ .../package.json | 53 +++ .../src/index.ts | 18 + .../src/microsoftGraph/client.test.ts | 363 +++++++++++++++ .../src/microsoftGraph/client.ts | 235 ++++++++++ .../src/microsoftGraph/config.test.ts | 75 ++++ .../src/microsoftGraph/config.ts | 91 ++++ .../src/microsoftGraph/constants.ts | 32 ++ .../src/microsoftGraph/helper.test.ts | 29 ++ .../src/microsoftGraph/helper.ts | 22 + .../src/microsoftGraph/index.ts | 35 ++ .../src/microsoftGraph/org.test.ts | 94 ++++ .../src/microsoftGraph/org.ts | 87 ++++ .../src/microsoftGraph/read.test.ts | 354 +++++++++++++++ .../src/microsoftGraph/read.ts | 419 ++++++++++++++++++ .../src/microsoftGraph/types.ts | 32 ++ .../MicrosoftGraphOrgReaderProcessor.ts | 111 +++++ .../src/processors/index.ts | 17 + .../src/setupTests.ts | 17 + .../MicrosoftGraphOrgReaderProcessor.ts | 12 +- yarn.lock | 29 +- 26 files changed, 2312 insertions(+), 7 deletions(-) create mode 100644 .changeset/metal-badgers-carry.md create mode 100644 .changeset/silent-ways-laugh.md create mode 100644 plugins/catalog-backend-extension-msgraph/.eslintrc.js create mode 100644 plugins/catalog-backend-extension-msgraph/README.md create mode 100644 plugins/catalog-backend-extension-msgraph/config.d.ts create mode 100644 plugins/catalog-backend-extension-msgraph/package.json create mode 100644 plugins/catalog-backend-extension-msgraph/src/index.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/processors/index.ts create mode 100644 plugins/catalog-backend-extension-msgraph/src/setupTests.ts diff --git a/.changeset/metal-badgers-carry.md b/.changeset/metal-badgers-carry.md new file mode 100644 index 0000000000..17592e062f --- /dev/null +++ b/.changeset/metal-badgers-carry.md @@ -0,0 +1,17 @@ +--- +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-backend-extension-msgraph': patch +--- + +Move `MicrosoftGraphOrgReaderProcessor` from `@backstage/plugin-catalog-backend` +to `@backstage/plugin-catalog-backend-extension-msgraph`. + +For now `MicrosoftGraphOrgReaderProcessor` is only deprecated in +`@backstage/plugin-catalog-backend`, but will be removed in the future. While it +is now registered by default, it has to be registered manually in the future. + +TODO: Do we really want to deprecate the transformer before removing it? +It is actually pretty hard to switch to the new transformer as one has to call +`builder.replaceProcessors()` to replace ALL transformers. +As an alternative we can do a breaking change directly with the migration steps +(adding the dependency, adding an import and calling `builder.addProcessor()`). diff --git a/.changeset/silent-ways-laugh.md b/.changeset/silent-ways-laugh.md new file mode 100644 index 0000000000..cdb28a520c --- /dev/null +++ b/.changeset/silent-ways-laugh.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-extension-msgraph': patch +--- + +Allow customizations of `MicrosoftGraphOrgReaderProcessor` by passing an +optional `groupTransformer`, `userTransformer`, and `organizationTransformer`. diff --git a/packages/backend/package.json b/packages/backend/package.json index f4f304d714..b0fb1ec9d4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,10 +31,12 @@ "@backstage/catalog-client": "^0.3.13", "@backstage/catalog-model": "^0.8.2", "@backstage/config": "^0.1.5", + "@backstage/integration": "^0.5.6", "@backstage/plugin-app-backend": "^0.3.13", "@backstage/plugin-auth-backend": "^0.3.12", "@backstage/plugin-badges-backend": "^0.1.6", "@backstage/plugin-catalog-backend": "^0.10.2", + "@backstage/plugin-catalog-backend-extension-msgraph": "^0.1.0", "@backstage/plugin-code-coverage-backend": "^0.1.6", "@backstage/plugin-graphql-backend": "^0.1.8", "@backstage/plugin-kubernetes-backend": "^0.3.8", diff --git a/plugins/catalog-backend-extension-msgraph/.eslintrc.js b/plugins/catalog-backend-extension-msgraph/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/catalog-backend-extension-msgraph/README.md b/plugins/catalog-backend-extension-msgraph/README.md new file mode 100644 index 0000000000..8296d277df --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/README.md @@ -0,0 +1,86 @@ +# Catalog Backend Extension for Microsoft Graph + +This is an extension to the `plugin-catalog-backend` plugin, providing a +`MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data +from the Microsoft Graph API. This processor is useful, if you want to import +users and groups from Office 365. + +## Getting Started + +1. The processor is not installed by default, therefore you have to add a + dependency to `@backstage/plugin-catalog-backend-extension-msgraph` to your + backend package. + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-catalog-backend-extension-msgraph +``` + +2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you have to register it in the catalog plugin: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + }), +); +``` + +3. Configure the processor: + +```yaml +# app-config.yaml +catalog: + processors: + microsoftGraphOrg: + providers: + - target: https://graph.microsoft.com/v1.0 + authority: https://login.microsoftonline.com + tenantId: ${MICROSOFT_GRAPH_TENANT_ID} + clientId: ${MICROSOFT_GRAPH_CLIENT_ID} + clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional filter for user, see Microsoft Graph API for the syntax + userFilter: accountEnabled eq true and userType eq 'member' + # Optional filter for group, see Microsoft Graph API for the syntax + groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') +``` + +## Customize the Processor + +In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` allows to pass transformers for users, groups and the organization. + +1. Create a transformer: + +```ts +export async function myGroupTransformer( + group: MicrosoftGraph.Group, + groupPhoto?: string, +): Promise { + if ( + ((group as unknown) as { + creationOptions: string[]; + }).creationOptions.includes('ProvisionGroupHomepage') + ) { + return undefined; + } + + // Transformations may change namespace, change entity naming pattern, fill + // profile with more or other details... + + // Create the group entity on your own, or wrap the default transformer + return await defaultGroupTransformer(group, groupPhoto); +} +``` + +2. Configure the processor with the transformer: + +```ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + groupTransformer: myGroupTransformer, + }), +); +``` diff --git a/plugins/catalog-backend-extension-msgraph/config.d.ts b/plugins/catalog-backend-extension-msgraph/config.d.ts new file mode 100644 index 0000000000..d1b9630477 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/config.d.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** + * Configuration options for the catalog plugin. + */ + catalog?: { + /** + * List of processor-specific options and attributes + */ + processors?: { + /** + * MicrosoftGraphOrgReaderProcessor configuration + */ + microsoftGraphOrg?: { + /** + * The configuration parameters for each single Microsoft Graph provider. + */ + providers: Array<{ + /** + * The prefix of the target that this matches on, e.g. + * "https://graph.microsoft.com/v1.0", with no trailing slash. + */ + target: string; + /** + * The auth authority used. + * + * Default value "https://login.microsoftonline.com" + */ + authority?: string; + /** + * The tenant whose org data we are interested in. + */ + tenantId: string; + /** + * The OAuth client ID to use for authenticating requests. + */ + clientId: string; + /** + * The OAuth client secret to use for authenticating requests. + * + * @visibility secret + */ + clientSecret: string; + + // TODO: Consider not making these config options and pass them in the + // constructor instead. They are probably not environment specifc, so + // they could also be configured "in code". + + /** + * The filter to apply to extract users. + * + * E.g. "accountEnabled eq true and userType eq 'member'" + */ + userFilter?: string; + /** + * The filter to apply to extract groups. + * + * E.g. "securityEnabled eq false and mailEnabled eq true" + */ + groupFilter?: string; + }>; + }; + }; + }; +} diff --git a/plugins/catalog-backend-extension-msgraph/package.json b/plugins/catalog-backend-extension-msgraph/package.json new file mode 100644 index 0000000000..9feb8e973f --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/package.json @@ -0,0 +1,53 @@ +{ + "name": "@backstage/plugin-catalog-backend-extension-msgraph", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-extension-msgraph" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@azure/msal-node": "^1.1.0", + "@backstage/catalog-model": "^0.8.2", + "@backstage/config": "^0.1.5", + "@backstage/plugin-catalog-backend": "^0.10.2", + "@microsoft/microsoft-graph-types": "^1.25.0", + "cross-fetch": "^3.0.6", + "lodash": "^4.17.15", + "p-limit": "^3.0.2", + "winston": "^3.2.1", + "qs": "^6.9.4" + }, + "devDependencies": { + "@backstage/cli": "^0.7.0", + "@backstage/test-utils": "^0.1.13", + "@types/lodash": "^4.14.151", + "msw": "^0.21.2" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-extension-msgraph/src/index.ts b/plugins/catalog-backend-extension-msgraph/src/index.ts new file mode 100644 index 0000000000..83c8c14e37 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './processors'; +export * from './microsoftGraph'; diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts new file mode 100644 index 0000000000..5ef82432bb --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts @@ -0,0 +1,363 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as msal from '@azure/msal-node'; +import { msw } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { MicrosoftGraphClient } from './client'; + +describe('MicrosoftGraphClient', () => { + const confidentialClientApplication: jest.Mocked = { + acquireTokenByClientCredential: jest.fn(), + } as any; + let client: MicrosoftGraphClient; + const worker = setupServer(); + + msw.setupDefaultHandlers(worker); + + beforeEach(() => { + confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue( + { token: 'ACCESS_TOKEN' } as any, + ); + client = new MicrosoftGraphClient( + 'https://example.com', + confidentialClientApplication, + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should perform raw request', async () => { + worker.use( + rest.get('https://other.example.com/', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: 'example' })), + ), + ); + + const response = await client.requestRaw('https://other.example.com/'); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ value: 'example' }); + expect( + confidentialClientApplication.acquireTokenByClientCredential, + ).toBeCalledTimes(1); + expect( + confidentialClientApplication.acquireTokenByClientCredential, + ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] }); + }); + + it('should perform simple api request', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: 'example' })), + ), + ); + + const response = await client.requestApi('users'); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ value: 'example' }); + }); + + it('should perform api request with filter, select and expand', async () => { + worker.use( + rest.get('https://example.com/users', (req, res, ctx) => + res(ctx.status(200), ctx.json({ queryString: req.url.search })), + ), + ); + + const response = await client.requestApi('users', { + filter: 'test eq true', + expand: ['children'], + select: ['id', 'children'], + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + queryString: + '?$filter=test%20eq%20true&$select=id,children&$expand=children', + }); + }); + + it('should perform collection request for a single page', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: ['first'], + }), + ), + ), + ); + + const values = await collectAsyncIterable( + client.requestCollection('users'), + ); + + expect(values).toEqual(['first']); + }); + + it('should perform collection request for multiple pages', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: ['first'], + '@odata.nextLink': 'https://example.com/users2', + }), + ), + ), + ); + worker.use( + rest.get('https://example.com/users2', (_, res, ctx) => + res(ctx.status(200), ctx.json({ value: ['second'] })), + ), + ); + + const values = await collectAsyncIterable( + client.requestCollection('users'), + ); + + expect(values).toEqual(['first', 'second']); + }); + + it('should load user profile', async () => { + worker.use( + rest.get('https://example.com/users/user-id', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + surname: 'Example', + }), + ), + ), + ); + + const userProfile = await client.getUserProfile('user-id'); + + expect(userProfile).toEqual({ surname: 'Example' }); + }); + + it('should throw expection if load user profile fails', async () => { + worker.use( + rest.get('https://example.com/users/user-id', (_, res, ctx) => + res(ctx.status(404)), + ), + ); + + await expect(() => client.getUserProfile('user-id')).rejects.toThrowError(); + }); + + it('should load user profile photo with max size of 120', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [ + { + height: 120, + id: 120, + }, + { + height: 500, + id: 500, + }, + ], + }), + ), + ), + ); + worker.use( + rest.get( + 'https://example.com/users/user-id/photos/120/*', + (_, res, ctx) => res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should not fail if user has no profile photo', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => + res(ctx.status(404)), + ), + ); + + const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); + + expect(photo).toBeFalsy(); + }); + + it('should load user profile photo', async () => { + worker.use( + rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) => + res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhoto('user-id'); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load user profile photo for size 120', async () => { + worker.use( + rest.get( + 'https://example.com/users/user-id/photos/120/*', + (_, res, ctx) => res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getUserPhoto('user-id', '120'); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load users', async () => { + worker.use( + rest.get('https://example.com/users', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [{ surname: 'Example' }], + }), + ), + ), + ); + + const values = await collectAsyncIterable(client.getUsers()); + + expect(values).toEqual([{ surname: 'Example' }]); + }); + + it('should load group profile photo with max size of 120', async () => { + worker.use( + rest.get('https://example.com/groups/group-id/photos', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [ + { + height: 120, + id: 120, + }, + ], + }), + ), + ), + ); + worker.use( + rest.get( + 'https://example.com/groups/group-id/photos/120/*', + (_, res, ctx) => res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getGroupPhotoWithSizeLimit('group-id', 120); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load group profile photo', async () => { + worker.use( + rest.get('https://example.com/groups/group-id/photo/*', (_, res, ctx) => + res(ctx.status(200), ctx.text('911')), + ), + ); + + const photo = await client.getGroupPhoto('group-id'); + + expect(photo).toEqual('data:image/jpeg;base64,OTEx'); + }); + + it('should load groups', async () => { + worker.use( + rest.get('https://example.com/groups', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [{ displayName: 'Example' }], + }), + ), + ), + ); + + const values = await collectAsyncIterable(client.getGroups()); + + expect(values).toEqual([{ displayName: 'Example' }]); + }); + + it('should load group members', async () => { + worker.use( + rest.get('https://example.com/groups/group-id/members', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + value: [ + { '@odata.type': '#microsoft.graph.user' }, + { '@odata.type': '#microsoft.graph.group' }, + ], + }), + ), + ), + ); + + const values = await collectAsyncIterable( + client.getGroupMembers('group-id'), + ); + + expect(values).toEqual([ + { '@odata.type': '#microsoft.graph.user' }, + { '@odata.type': '#microsoft.graph.group' }, + ]); + }); + + it('should load organization', async () => { + worker.use( + rest.get('https://example.com/organization/tentant-id', (_, res, ctx) => + res( + ctx.status(200), + ctx.json({ + displayName: 'Example', + }), + ), + ), + ); + + const organization = await client.getOrganization('tentant-id'); + + expect(organization).toEqual({ displayName: 'Example' }); + }); +}); + +async function collectAsyncIterable( + iterable: AsyncIterable, +): Promise { + const values = []; + for await (const value of iterable) { + values.push(value); + } + return values; +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts new file mode 100644 index 0000000000..3dfc58e773 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts @@ -0,0 +1,235 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as msal from '@azure/msal-node'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import fetch from 'cross-fetch'; +import qs from 'qs'; +import { MicrosoftGraphProviderConfig } from './config'; + +export type ODataQuery = { + filter?: string; + expand?: string[]; + select?: string[]; +}; + +export type GroupMember = + | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' }) + | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' }); + +export class MicrosoftGraphClient { + static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient { + const clientConfig: msal.Configuration = { + auth: { + clientId: config.clientId, + clientSecret: config.clientSecret, + authority: `${config.authority}/${config.tenantId}`, + }, + }; + const pca = new msal.ConfidentialClientApplication(clientConfig); + return new MicrosoftGraphClient(config.target, pca); + } + + constructor( + private readonly baseUrl: string, + private readonly pca: msal.ConfidentialClientApplication, + ) {} + + async *requestCollection( + path: string, + query?: ODataQuery, + ): AsyncIterable { + let response = await this.requestApi(path, query); + + for (;;) { + if (response.status !== 200) { + await this.handleError(path, response); + } + + const result = await response.json(); + const elements: T[] = result.value; + + yield* elements; + + // Follow cursor to the next page if one is available + if (!result['@odata.nextLink']) { + return; + } + + response = await this.requestRaw(result['@odata.nextLink']); + } + } + + async requestApi(path: string, query?: ODataQuery): Promise { + const queryString = qs.stringify( + { + $filter: query?.filter, + $select: query?.select?.join(','), + $expand: query?.expand?.join(','), + }, + { + addQueryPrefix: true, + // Microsoft Graph doesn't like an encoded query string + encode: false, + }, + ); + + return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`); + } + + async requestRaw(url: string): Promise { + // Make sure that we always have a valid access token (might be cached) + const token = await this.pca.acquireTokenByClientCredential({ + scopes: ['https://graph.microsoft.com/.default'], + }); + + if (!token) { + throw new Error('Error while requesting token for Microsoft Graph'); + } + + return await fetch(url, { + headers: { + Authorization: `Bearer ${token.accessToken}`, + }, + }); + } + + async getUserProfile(userId: string): Promise { + const response = await this.requestApi(`users/${userId}`); + + if (response.status !== 200) { + await this.handleError('user profile', response); + } + + return await response.json(); + } + + async getUserPhotoWithSizeLimit( + userId: string, + maxSize: number, + ): Promise { + return await this.getPhotoWithSizeLimit('users', userId, maxSize); + } + + async getUserPhoto( + userId: string, + sizeId?: string, + ): Promise { + return await this.getPhoto('users', userId, sizeId); + } + + async *getUsers(query?: ODataQuery): AsyncIterable { + yield* this.requestCollection(`users`, query); + } + + async getGroupPhotoWithSizeLimit( + groupId: string, + maxSize: number, + ): Promise { + return await this.getPhotoWithSizeLimit('groups', groupId, maxSize); + } + + async getGroupPhoto( + groupId: string, + sizeId?: string, + ): Promise { + return await this.getPhoto('groups', groupId, sizeId); + } + + async *getGroups(query?: ODataQuery): AsyncIterable { + yield* this.requestCollection(`groups`, query); + } + + async *getGroupMembers(groupId: string): AsyncIterable { + yield* this.requestCollection(`groups/${groupId}/members`); + } + + async getOrganization( + tenantId: string, + ): Promise { + const response = await this.requestApi(`organization/${tenantId}`); + + if (response.status !== 200) { + await this.handleError(`organization/${tenantId}`, response); + } + + return await response.json(); + } + + private async getPhotoWithSizeLimit( + entityName: string, + id: string, + maxSize: number, + ): Promise { + const response = await this.requestApi(`${entityName}/${id}/photos`); + + if (response.status === 404) { + return undefined; + } else if (response.status !== 200) { + await this.handleError(`${entityName} photos`, response); + } + + const result = await response.json(); + const photos = result.value as MicrosoftGraph.ProfilePhoto[]; + let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined; + + // Find the biggest picture that is smaller than the max size + for (const p of photos) { + if ( + !selectedPhoto || + (p.height! >= selectedPhoto.height! && p.height! <= maxSize) + ) { + selectedPhoto = p; + } + } + + if (!selectedPhoto) { + return undefined; + } + + return await this.getPhoto(entityName, id, selectedPhoto.id!); + } + + private async getPhoto( + entityName: string, + id: string, + sizeId?: string, + ): Promise { + const path = sizeId + ? `${entityName}/${id}/photos/${sizeId}/$value` + : `${entityName}/${id}/photo/$value`; + const response = await this.requestApi(path); + + if (response.status === 404) { + return undefined; + } else if (response.status !== 200) { + await this.handleError('photo', response); + } + + return `data:image/jpeg;base64,${Buffer.from( + await response.arrayBuffer(), + ).toString('base64')}`; + } + + private async handleError(path: string, response: Response): Promise { + const result = await response.json(); + const error = result.error as MicrosoftGraph.PublicError; + + throw new Error( + `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`, + ); + } +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts new file mode 100644 index 0000000000..4671fd23ae --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readMicrosoftGraphConfig } from './config'; + +describe('readMicrosoftGraphConfig', () => { + it('applies all of the defaults', () => { + const config = { + providers: [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + }, + ], + }; + const actual = readMicrosoftGraphConfig(new ConfigReader(config)); + const expected = [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.microsoftonline.com', + userFilter: undefined, + groupFilter: undefined, + }, + ]; + expect(actual).toEqual(expected); + }); + + it('reads all the values', () => { + const config = { + providers: [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com/', + userFilter: 'accountEnabled eq true', + groupFilter: 'securityEnabled eq false', + }, + ], + }; + const actual = readMicrosoftGraphConfig(new ConfigReader(config)); + const expected = [ + { + target: 'target', + tenantId: 'tenantId', + clientId: 'clientId', + clientSecret: 'clientSecret', + authority: 'https://login.example.com', + userFilter: 'accountEnabled eq true', + groupFilter: 'securityEnabled eq false', + }, + ]; + expect(actual).toEqual(expected); + }); +}); diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts new file mode 100644 index 0000000000..72416a63ee --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +/** + * The configuration parameters for a single Microsoft Graph provider. + */ +export type MicrosoftGraphProviderConfig = { + /** + * The prefix of the target that this matches on, e.g. + * "https://graph.microsoft.com/v1.0", with no trailing slash. + */ + target: string; + /** + * The auth authority used. + * + * E.g. "https://login.microsoftonline.com" + */ + authority?: string; + /** + * The tenant whose org data we are interested in. + */ + tenantId: string; + /** + * The OAuth client ID to use for authenticating requests. + */ + clientId: string; + /** + * The OAuth client secret to use for authenticating requests. + * + * @visibility secret + */ + clientSecret: string; + /** + * The filter to apply to extract users. + * + * E.g. "accountEnabled eq true and userType eq 'member'" + */ + userFilter?: string; + /** + * The filter to apply to extract groups. + * + * E.g. "securityEnabled eq false and mailEnabled eq true" + */ + groupFilter?: string; +}; + +export function readMicrosoftGraphConfig( + config: Config, +): MicrosoftGraphProviderConfig[] { + const providers: MicrosoftGraphProviderConfig[] = []; + const providerConfigs = config.getOptionalConfigArray('providers') ?? []; + + for (const providerConfig of providerConfigs) { + const target = providerConfig.getString('target').replace(/\/+$/, ''); + const authority = + providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') || + 'https://login.microsoftonline.com'; + const tenantId = providerConfig.getString('tenantId'); + const clientId = providerConfig.getString('clientId'); + const clientSecret = providerConfig.getString('clientSecret'); + const userFilter = providerConfig.getOptionalString('userFilter'); + const groupFilter = providerConfig.getOptionalString('groupFilter'); + + providers.push({ + target, + authority, + tenantId, + clientId, + clientSecret, + userFilter, + groupFilter, + }); + } + + return providers; +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts new file mode 100644 index 0000000000..6d34d0c159 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The tenant id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION = + 'graph.microsoft.com/tenant-id'; + +/** + * The group id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION = + 'graph.microsoft.com/group-id'; + +/** + * The user id used by the Microsoft Graph API + */ +export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id'; diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts new file mode 100644 index 0000000000..48a07bbce8 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { normalizeEntityName } from './helper'; + +describe('normalizeEntityName', () => { + it('should normalize name to valid entity name', () => { + expect(normalizeEntityName('User Name')).toBe('user_name'); + }); + + it('should normalize e-mail to valid entity name', () => { + expect(normalizeEntityName('user.name@example.com')).toBe( + 'user.name_example.com', + ); + }); +}); diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts new file mode 100644 index 0000000000..e7a804cbff --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function normalizeEntityName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9_\-\.]/g, '_'); +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts new file mode 100644 index 0000000000..89b1a35a79 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { MicrosoftGraphClient } from './client'; +export { readMicrosoftGraphConfig } from './config'; +export type { MicrosoftGraphProviderConfig } from './config'; +export { + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, +} from './constants'; +export { normalizeEntityName } from './helper'; +export { + defaultGroupTransformer, + defaultOrganizationTransformer, + defaultUserTransformer, + readMicrosoftGraphOrg, +} from './read'; +export type { + GroupTransformer, + OrganizationTransformer, + UserTransformer, +} from './types'; diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts new file mode 100644 index 0000000000..e264847478 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { buildMemberOf, buildOrgHierarchy } from './org'; + +function g( + name: string, + parent: string | undefined, + children: string[], +): GroupEntity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { name }, + spec: { type: 'team', parent, children }, + }; +} + +describe('buildOrgHierarchy', () => { + it('adds groups to their parent.children', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const d = g('d', 'a', []); + buildOrgHierarchy([a, b, c, d]); + expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd'])); + expect(b.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(c.spec.children).toEqual([]); + expect(d.spec.children).toEqual([]); + }); + + it('sets parent of groups children', () => { + const a = g('a', undefined, ['b', 'd']); + const b = g('b', undefined, ['c']); + const c = g('c', undefined, []); + const d = g('d', undefined, []); + buildOrgHierarchy([a, b, c, d]); + expect(a.spec.parent).toBeUndefined(); + expect(b.spec.parent).toBe('a'); + expect(c.spec.parent).toBe('b'); + expect(d.spec.parent).toBe('a'); + }); +}); + +describe('buildMemberOf', () => { + it('fills indirect member of groups', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const u: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'n' }, + spec: { profile: {}, memberOf: ['c'] }, + }; + + const groups = [a, b, c]; + buildOrgHierarchy(groups); + buildMemberOf(groups, [u]); + expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + }); + + it('takes group spec.members into account', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + c.spec.members = ['n']; + const u: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'n' }, + spec: { profile: {}, memberOf: [] }, + }; + + const groups = [a, b, c]; + buildOrgHierarchy(groups); + buildMemberOf(groups, [u]); + expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + }); +}); diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts new file mode 100644 index 0000000000..69ccc9fb77 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; + +// TODO: Copied from plugin-catalog-backend, but we could also export them from +// there. Or move them to catalog-model. + +export function buildOrgHierarchy(groups: GroupEntity[]) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + + // + // Make sure that g.parent.children contain g + // + + for (const group of groups) { + const selfName = group.metadata.name; + const parentName = group.spec.parent; + if (parentName) { + const parent = groupsByName.get(parentName); + if (parent && !parent.spec.children.includes(selfName)) { + parent.spec.children.push(selfName); + } + } + } + + // + // Make sure that g.children.parent is g + // + + for (const group of groups) { + const selfName = group.metadata.name; + for (const childName of group.spec.children) { + const child = groupsByName.get(childName); + if (child && !child.spec.parent) { + child.spec.parent = selfName; + } + } + } +} + +// Ensure that users have their transitive group memberships. Requires that +// the groups were previously processed with buildOrgHierarchy() +export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + + users.forEach(user => { + const transitiveMemberOf = new Set(); + + const todo = [ + ...user.spec.memberOf, + ...groups + .filter(g => g.spec.members?.includes(user.metadata.name)) + .map(g => g.metadata.name), + ]; + + for (;;) { + const current = todo.pop(); + if (!current) { + break; + } + + if (!transitiveMemberOf.has(current)) { + transitiveMemberOf.add(current); + const group = groupsByName.get(current); + if (group?.spec.parent) { + todo.push(group.spec.parent); + } + } + } + + user.spec.memberOf = [...transitiveMemberOf]; + }); +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts new file mode 100644 index 0000000000..fe63b838cf --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts @@ -0,0 +1,354 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import merge from 'lodash/merge'; +import { GroupMember, MicrosoftGraphClient } from './client'; +import { + readMicrosoftGraphGroups, + readMicrosoftGraphOrganization, + readMicrosoftGraphUsers, + resolveRelations, +} from './read'; + +function user(data: Partial): UserEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'name' }, + spec: { profile: {}, memberOf: [] }, + } as UserEntity, + data, + ); +} + +function group(data: Partial): GroupEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'name', + }, + spec: { + children: [], + type: 'team', + }, + } as GroupEntity, + data, + ); +} + +describe('read microsoft graph', () => { + const client: jest.Mocked = { + getUsers: jest.fn(), + getGroups: jest.fn(), + getGroupMembers: jest.fn(), + getUserPhotoWithSizeLimit: jest.fn(), + getGroupPhotoWithSizeLimit: jest.fn(), + getOrganization: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + describe('readMicrosoftGraphUsers', () => { + it('should read users', async () => { + async function* getExampleUsers() { + yield { + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + + client.getUsers.mockImplementation(getExampleUsers); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsers(client, { + userFilter: 'accountEnabled eq true', + }); + + expect(users).toEqual([ + user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'userid', + }, + name: 'user.name_example.com', + }, + spec: { + profile: { + displayName: 'User Name', + email: 'user.name@example.com', + picture: 'data:image/jpeg;base64,...', + }, + memberOf: [], + }, + }), + ]); + + expect(client.getUsers).toBeCalledTimes(1); + expect(client.getUsers).toBeCalledWith({ + filter: 'accountEnabled eq true', + }); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); + }); + + describe('readMicrosoftGraphOrganization', () => { + it('should read organization', async () => { + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + + const { rootGroup } = await readMicrosoftGraphOrganization( + client, + 'tenantid', + ); + + expect(rootGroup).toEqual( + group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }), + ); + + expect(client.getOrganization).toBeCalledTimes(1); + expect(client.getOrganization).toBeCalledWith('tenantid'); + }); + + it('should read organization with custom transformer', async () => { + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + + const { rootGroup } = await readMicrosoftGraphOrganization( + client, + 'tenantid', + { transformer: async _ => undefined }, + ); + + expect(rootGroup).toEqual(undefined); + + expect(client.getOrganization).toBeCalledTimes(1); + expect(client.getOrganization).toBeCalledWith('tenantid'); + }); + }); + + describe('readMicrosoftGraphGroups', () => { + it('should read groups', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { + groups, + groupMember, + groupMemberOf, + rootGroup, + } = await readMicrosoftGraphGroups(client, 'tenantid', { + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + filter: 'securityEnabled eq false', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); + }); + + describe('resolveRelations', () => { + it('should resolve relations', async () => { + const rootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenant-id-root', + }, + name: 'root', + }, + spec: { + type: 'root', + children: [], + }, + }); + const groupA = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-a', + }, + name: 'a', + }, + }); + const groupB = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-b', + }, + name: 'b', + }, + }); + const groupC = group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'group-id-c', + }, + name: 'c', + }, + }); + const user1 = user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'user-id-1', + }, + name: 'user1', + }, + }); + const user2 = user({ + metadata: { + annotations: { + 'graph.microsoft.com/user-id': 'user-id-2', + }, + name: 'user2', + }, + }); + const groups = [rootGroup, groupA, groupB, groupC]; + const users = [user1, user2]; + const groupMember = new Map>(); + groupMember.set('group-id-b', new Set(['group-id-c'])); + const groupMemberOf = new Map>(); + groupMemberOf.set('user-id-1', new Set(['group-id-a'])); + groupMemberOf.set('user-id-2', new Set(['group-id-c'])); + + // We have a root groups + // We have three groups: a, b, c. c is child of b + // we have two users: u1, u2. u1 is member of a, u2 is member of c + resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); + + expect(rootGroup.spec.parent).toBeUndefined(); + expect(rootGroup.spec.children).toEqual( + expect.arrayContaining(['a', 'b']), + ); + + expect(groupA.spec.parent).toEqual('root'); + expect(groupA.spec.children).toEqual(expect.arrayContaining([])); + + expect(groupB.spec.parent).toEqual('root'); + expect(groupB.spec.children).toEqual(expect.arrayContaining(['c'])); + + expect(groupC.spec.parent).toEqual('b'); + expect(groupC.spec.children).toEqual(expect.arrayContaining([])); + + expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a'])); + expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c'])); + }); + }); +}); diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts new file mode 100644 index 0000000000..308fbf34fd --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts @@ -0,0 +1,419 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import limiterFactory from 'p-limit'; +import { MicrosoftGraphClient } from './client'; +import { + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, +} from './constants'; +import { normalizeEntityName } from './helper'; +import { buildMemberOf, buildOrgHierarchy } from './org'; +import { + GroupTransformer, + OrganizationTransformer, + UserTransformer, +} from './types'; + +export async function defaultUserTransformer( + user: MicrosoftGraph.User, + userPhoto?: string, +): Promise { + if (!user.id || !user.displayName || !user.mail) { + return undefined; + } + + const name = normalizeEntityName(user.mail); + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name, + annotations: { + [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!, + }, + }, + spec: { + profile: { + displayName: user.displayName!, + email: user.mail!, + + // TODO: Additional fields? + // jobTitle: user.jobTitle || undefined, + // officeLocation: user.officeLocation || undefined, + // mobilePhone: user.mobilePhone || undefined, + }, + memberOf: [], + }, + }; + + if (userPhoto) { + entity.spec.profile!.picture = userPhoto; + } + + return entity; +} + +export async function readMicrosoftGraphUsers( + client: MicrosoftGraphClient, + options?: { userFilter?: string; transformer?: UserTransformer }, +): Promise<{ + users: UserEntity[]; // With all relations empty +}> { + const users: UserEntity[] = []; + const limiter = limiterFactory(10); + + const transformer = options?.transformer ?? defaultUserTransformer; + const promises: Promise[] = []; + + for await (const user of client.getUsers({ + filter: options?.userFilter, + })) { + // Process all users in parallel, otherwise it can take quite some time + promises.push( + limiter(async () => { + const userPhoto = await client.getUserPhotoWithSizeLimit( + user.id!, + // We are limiting the photo size, as users with full resolution photos + // can make the Backstage API slow + 120, + ); + + const entity = await transformer(user, userPhoto); + + if (!entity) { + return; + } + + users.push(entity); + }), + ); + } + + // Wait for all users and photos to be downloaded + await Promise.all(promises); + + return { users }; +} + +export async function defaultOrganizationTransformer( + organization: MicrosoftGraph.Organization, +): Promise { + if (!organization.id || !organization.displayName) { + return undefined; + } + + const name = normalizeEntityName(organization.displayName!); + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: name, + description: organization.displayName!, + annotations: { + [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!, + }, + }, + spec: { + type: 'root', + profile: { + displayName: organization.displayName!, + }, + children: [], + }, + }; +} + +export async function readMicrosoftGraphOrganization( + client: MicrosoftGraphClient, + tenantId: string, + options?: { transformer?: OrganizationTransformer }, +): Promise<{ + rootGroup?: GroupEntity; // With all relations empty +}> { + // For now we expect a single root organization + const organization = await client.getOrganization(tenantId); + const transformer = options?.transformer ?? defaultOrganizationTransformer; + const rootGroup = await transformer(organization); + + return { rootGroup }; +} + +export async function defaultGroupTransformer( + group: MicrosoftGraph.Group, + groupPhoto?: string, +): Promise { + if (!group.id || !group.displayName) { + return undefined; + } + + const name = normalizeEntityName(group.mailNickname || group.displayName); + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: name, + annotations: { + [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id, + }, + }, + spec: { + type: 'team', + profile: {}, + children: [], + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + if (group.displayName) { + entity.spec.profile!.displayName = group.displayName; + } + if (group.mail) { + entity.spec.profile!.email = group.mail; + } + if (groupPhoto) { + entity.spec.profile!.picture = groupPhoto; + } + + return entity; +} + +export async function readMicrosoftGraphGroups( + client: MicrosoftGraphClient, + tenantId: string, + options?: { groupFilter?: string; transformer?: GroupTransformer }, +): Promise<{ + groups: GroupEntity[]; // With all relations empty + rootGroup: GroupEntity | undefined; // With all relations empty + groupMember: Map>; + groupMemberOf: Map>; +}> { + const groups: GroupEntity[] = []; + const groupMember: Map> = new Map(); + const groupMemberOf: Map> = new Map(); + const limiter = limiterFactory(10); + + const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId); + if (rootGroup) { + groupMember.set(rootGroup.metadata.name, new Set()); + groups.push(rootGroup); + } + + const transformer = options?.transformer ?? defaultGroupTransformer; + const promises: Promise[] = []; + + for await (const group of client.getGroups({ + filter: options?.groupFilter, + })) { + // Process all groups in parallel, otherwise it can take quite some time + promises.push( + limiter(async () => { + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo + /* const groupPhoto = await client.getGroupPhotoWithSizeLimit( + group.id!, + // We are limiting the photo size, as groups with full resolution photos + // can make the Backstage API slow + 120, + );*/ + + const entity = await transformer(group /* , groupPhoto*/); + + if (!entity) { + return; + } + + for await (const member of client.getGroupMembers(group.id!)) { + if (!member.id) { + continue; + } + + if (member['@odata.type'] === '#microsoft.graph.user') { + ensureItem(groupMemberOf, member.id, group.id!); + } + + if (member['@odata.type'] === '#microsoft.graph.group') { + ensureItem(groupMember, group.id!, member.id); + } + } + + groups.push(entity); + }), + ); + } + + // Wait for all group members and photos to be loaded + await Promise.all(promises); + + return { + groups, + rootGroup, + groupMember, + groupMemberOf, + }; +} + +export function resolveRelations( + rootGroup: GroupEntity | undefined, + groups: GroupEntity[], + users: UserEntity[], + groupMember: Map>, + groupMemberOf: Map>, +) { + // Build reference lookup tables, we reference them by the id the the graph + const groupMap: Map = new Map(); // by group-id or tenant-id + + for (const group of groups) { + if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) { + groupMap.set( + group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION], + group, + ); + } + if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) { + groupMap.set( + group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION], + group, + ); + } + } + + // Resolve all member relationships into the reverse direction + const parentGroups = new Map>(); + + groupMember.forEach((members, groupId) => + members.forEach(m => ensureItem(parentGroups, m, groupId)), + ); + + // Make sure every group (except root) has at least one parent. If the parent is missing, add the root. + if (rootGroup) { + const tenantId = rootGroup.metadata.annotations![ + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION + ]; + + groups.forEach(group => { + const groupId = group.metadata.annotations![ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ]; + + if (!groupId) { + return; + } + + if (retrieveItems(parentGroups, groupId).size === 0) { + ensureItem(parentGroups, groupId, tenantId); + ensureItem(groupMember, tenantId, groupId); + } + }); + } + + groups.forEach(group => { + const id = + group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ?? + group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]; + + retrieveItems(groupMember, id).forEach(m => { + const childGroup = groupMap.get(m); + if (childGroup) { + // TODO: This break when groups are transformed into different namespaces, use full entity refs instead + + group.spec.children.push(childGroup.metadata.name); + } + }); + + retrieveItems(parentGroups, id).forEach(p => { + const parentGroup = groupMap.get(p); + if (parentGroup) { + // TODO: Only having a single parent group might not match every companies model, but fine for now. + + // TODO: use full entity refs + group.spec.parent = parentGroup.metadata.name; + } + }); + }); + + // Make sure that all groups have proper parents and children + buildOrgHierarchy(groups); + + // Set relations for all users + users.forEach(user => { + const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION]; + + retrieveItems(groupMemberOf, id).forEach(p => { + const parentGroup = groupMap.get(p); + if (parentGroup) { + // TODO: use full entity refs + user.spec.memberOf.push(parentGroup.metadata.name); + } + }); + }); + + // Make sure all transitive memberships are available + buildMemberOf(groups, users); +} + +export async function readMicrosoftGraphOrg( + client: MicrosoftGraphClient, + tenantId: string, + options?: { + userFilter?: string; + groupFilter?: string; + groupTransformer?: GroupTransformer; + }, +): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { + const { users } = await readMicrosoftGraphUsers(client, { + userFilter: options?.userFilter, + }); + const { + groups, + rootGroup, + groupMember, + groupMemberOf, + } = await readMicrosoftGraphGroups(client, tenantId, { + groupFilter: options?.groupFilter, + transformer: options?.groupTransformer, + }); + + resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); + users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + + return { users, groups }; +} + +function ensureItem( + target: Map>, + key: string, + value: string, +) { + let set = target.get(key); + if (!set) { + set = new Set(); + target.set(key, set); + } + set!.add(value); +} + +function retrieveItems( + target: Map>, + key: string, +): Set { + return target.get(key) ?? new Set(); +} diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts new file mode 100644 index 0000000000..55c28b8d7a --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; + +export type UserTransformer = ( + user: MicrosoftGraph.User, + userPhoto?: string, +) => Promise; + +export type OrganizationTransformer = ( + organization: MicrosoftGraph.Organization, +) => Promise; + +export type GroupTransformer = ( + group: MicrosoftGraph.Group, + groupPhoto?: string, +) => Promise; diff --git a/plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts new file mode 100644 index 0000000000..5d8b071663 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + CatalogProcessor, + CatalogProcessorEmit, + results, +} from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { + GroupTransformer, + MicrosoftGraphClient, + MicrosoftGraphProviderConfig, + readMicrosoftGraphConfig, + readMicrosoftGraphOrg, +} from '../microsoftGraph'; + +/** + * Extracts teams and users out of a the Microsoft Graph API. + */ +export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { + private readonly providers: MicrosoftGraphProviderConfig[]; + private readonly logger: Logger; + private readonly groupTransformer?: GroupTransformer; + + static fromConfig( + config: Config, + options: { logger: Logger; groupTransformer?: GroupTransformer }, + ) { + const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); + return new MicrosoftGraphOrgReaderProcessor({ + ...options, + providers: c ? readMicrosoftGraphConfig(c) : [], + }); + } + + constructor(options: { + providers: MicrosoftGraphProviderConfig[]; + logger: Logger; + groupTransformer?: GroupTransformer; + }) { + this.providers = options.providers; + this.logger = options.logger; + this.groupTransformer = options.groupTransformer; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'microsoft-graph-org') { + return false; + } + + const provider = this.providers.find(p => + location.target.startsWith(p.target), + ); + if (!provider) { + throw new Error( + `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`, + ); + } + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading Microsoft Graph users and groups'); + + // We create a client each time as we need one that matches the specific provider + const client = MicrosoftGraphClient.create(provider); + const { users, groups } = await readMicrosoftGraphOrg( + client, + provider.tenantId, + { + userFilter: provider.userFilter, + groupFilter: provider.groupFilter, + groupTransformer: this.groupTransformer, + }, + ); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`, + ); + + // Done! + for (const group of groups) { + emit(results.entity(location, group)); + } + for (const user of users) { + emit(results.entity(location, user)); + } + + return true; + } +} diff --git a/plugins/catalog-backend-extension-msgraph/src/processors/index.ts b/plugins/catalog-backend-extension-msgraph/src/processors/index.ts new file mode 100644 index 0000000000..46a0cce6f5 --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/processors/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; diff --git a/plugins/catalog-backend-extension-msgraph/src/setupTests.ts b/plugins/catalog-backend-extension-msgraph/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/plugins/catalog-backend-extension-msgraph/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts index 3456f2cafa..a25b3bc506 100644 --- a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -26,8 +26,14 @@ import { import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +// TODO: Remove this deprecated processor, the related code in +// ./microsoftGraph/, and the config section in the future. + /** - * Extracts teams and users out of an LDAP server. + * Extracts teams and users out of a the Microsoft Graph API. + * + * @deprecated Use the MicrosoftGraphOrgReaderProcessor from package + * @backstage/plugin-catalog-backend-extension-msgraph instead. */ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly providers: MicrosoftGraphProviderConfig[]; @@ -58,6 +64,10 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { return false; } + this.logger.warn( + 'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-extension-msgraph instead.', + ); + const provider = this.providers.find(p => location.target.startsWith(p.target), ); diff --git a/yarn.lock b/yarn.lock index 208ad9e952..ab538fb58a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -218,6 +218,13 @@ dependencies: debug "^4.1.1" +"@azure/msal-common@^4.3.0": + version "4.3.0" + resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-4.3.0.tgz#b540e92748656724088bf77192e59943a93135bc" + integrity sha512-jFqUWe83wVb6O8cNGGBFg2QlKvqM1ezUgJTEV7kIsAPX0RXhGFE4B1DLNt6hCnkTXDbw+KGW0zgxOEr4MJQwLw== + dependencies: + debug "^4.1.1" + "@azure/msal-node@1.0.0-beta.3", "@azure/msal-node@^1.0.0-beta.3": version "1.0.0-beta.3" resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-beta.3.tgz#c84c7948028b39e48b901f5fac35bdedcbc8772e" @@ -228,6 +235,16 @@ jsonwebtoken "^8.5.1" uuid "^8.3.0" +"@azure/msal-node@^1.1.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.1.0.tgz#e472cfadead169f8832066ae6c2d6b8eef4e89e4" + integrity sha512-gMO9aZdWOzufp1PcdD5ID25DdS9eInxgeCqx4Tk8PVU6Z7RxJQhoMzS64cJhGdpYgeIQwKljtF0CLCcPFxew/w== + dependencies: + "@azure/msal-common" "^4.3.0" + axios "^0.21.1" + jsonwebtoken "^8.5.1" + uuid "^8.3.0" + "@azure/storage-blob@^12.4.0": version "12.4.0" resolved "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.4.0.tgz#7127ddd9f413105e2c3688691bc4c6245d0806b3" @@ -1347,7 +1364,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.7.4": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1360,7 +1377,7 @@ yup "^0.29.3" "@backstage/catalog-model@^0.7.9": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1389,16 +1406,16 @@ react-use "^17.2.4" "@backstage/plugin-catalog@^0.5.1": - version "0.6.2" + version "0.6.3" dependencies: "@backstage/catalog-client" "^0.3.13" - "@backstage/catalog-model" "^0.8.2" - "@backstage/core" "^0.7.12" + "@backstage/catalog-model" "^0.8.3" + "@backstage/core" "^0.7.13" "@backstage/core-plugin-api" "^0.1.2" "@backstage/errors" "^0.1.1" "@backstage/integration" "^0.5.6" "@backstage/integration-react" "^0.1.3" - "@backstage/plugin-catalog-react" "^0.2.2" + "@backstage/plugin-catalog-react" "^0.2.3" "@backstage/theme" "^0.2.8" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" From 65f5f00c62d786f89e6495c9d662b4513c2370fb Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Mon, 31 May 2021 09:41:17 +0200 Subject: [PATCH 06/17] Resolve todos Signed-off-by: Oliver Sand --- .../src/microsoftGraph/read.test.ts | 20 ++++++++++++------- .../src/microsoftGraph/read.ts | 17 ++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts index fe63b838cf..8a5716f277 100644 --- a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts @@ -335,20 +335,26 @@ describe('read microsoft graph', () => { expect(rootGroup.spec.parent).toBeUndefined(); expect(rootGroup.spec.children).toEqual( - expect.arrayContaining(['a', 'b']), + expect.arrayContaining(['group:default/a', 'group:default/b']), ); - expect(groupA.spec.parent).toEqual('root'); + expect(groupA.spec.parent).toEqual('group:default/root'); expect(groupA.spec.children).toEqual(expect.arrayContaining([])); - expect(groupB.spec.parent).toEqual('root'); - expect(groupB.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(groupB.spec.parent).toEqual('group:default/root'); + expect(groupB.spec.children).toEqual( + expect.arrayContaining(['group:default/c']), + ); - expect(groupC.spec.parent).toEqual('b'); + expect(groupC.spec.parent).toEqual('group:default/b'); expect(groupC.spec.children).toEqual(expect.arrayContaining([])); - expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a'])); - expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c'])); + expect(user1.spec.memberOf).toEqual( + expect.arrayContaining(['group:default/a']), + ); + expect(user2.spec.memberOf).toEqual( + expect.arrayContaining(['group:default/c']), + ); }); }); }); diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts index 308fbf34fd..b047a57836 100644 --- a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GroupEntity, + stringifyEntityRef, + UserEntity, +} from '@backstage/catalog-model'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; import limiterFactory from 'p-limit'; import { MicrosoftGraphClient } from './client'; @@ -332,9 +336,7 @@ export function resolveRelations( retrieveItems(groupMember, id).forEach(m => { const childGroup = groupMap.get(m); if (childGroup) { - // TODO: This break when groups are transformed into different namespaces, use full entity refs instead - - group.spec.children.push(childGroup.metadata.name); + group.spec.children.push(stringifyEntityRef(childGroup)); } }); @@ -342,9 +344,7 @@ export function resolveRelations( const parentGroup = groupMap.get(p); if (parentGroup) { // TODO: Only having a single parent group might not match every companies model, but fine for now. - - // TODO: use full entity refs - group.spec.parent = parentGroup.metadata.name; + group.spec.parent = stringifyEntityRef(parentGroup); } }); }); @@ -359,8 +359,7 @@ export function resolveRelations( retrieveItems(groupMemberOf, id).forEach(p => { const parentGroup = groupMap.get(p); if (parentGroup) { - // TODO: use full entity refs - user.spec.memberOf.push(parentGroup.metadata.name); + user.spec.memberOf.push(stringifyEntityRef(parentGroup)); } }); }); From 8a63e6a523e4f4f2bb436d3aefbdce2d22908a5c Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 10 Jun 2021 12:48:41 +0200 Subject: [PATCH 07/17] Rename from `plugin-catalog-backend-extension-msgraph` to `plugin-catalog-backend-module-msgraph` Signed-off-by: Oliver Sand --- .changeset/metal-badgers-carry.md | 4 ++-- .changeset/silent-ways-laugh.md | 2 +- packages/backend/package.json | 2 +- packages/backend/src/plugins/catalog.ts | 2 +- .../.eslintrc.js | 0 .../README.md | 4 ++-- .../config.d.ts | 0 .../package.json | 4 ++-- .../src/index.ts | 0 .../src/microsoftGraph/client.test.ts | 0 .../src/microsoftGraph/client.ts | 0 .../src/microsoftGraph/config.test.ts | 0 .../src/microsoftGraph/config.ts | 0 .../src/microsoftGraph/constants.ts | 0 .../src/microsoftGraph/helper.test.ts | 0 .../src/microsoftGraph/helper.ts | 0 .../src/microsoftGraph/index.ts | 0 .../src/microsoftGraph/org.test.ts | 0 .../src/microsoftGraph/org.ts | 0 .../src/microsoftGraph/read.test.ts | 0 .../src/microsoftGraph/read.ts | 0 .../src/microsoftGraph/types.ts | 0 .../src/processors/MicrosoftGraphOrgReaderProcessor.ts | 0 .../src/processors/index.ts | 0 .../src/setupTests.ts | 0 .../ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts | 4 ++-- 26 files changed, 11 insertions(+), 11 deletions(-) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/.eslintrc.js (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/README.md (94%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/config.d.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/package.json (90%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/index.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/client.test.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/client.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/config.test.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/config.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/constants.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/helper.test.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/helper.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/index.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/org.test.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/org.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/read.test.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/read.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/microsoftGraph/types.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/processors/MicrosoftGraphOrgReaderProcessor.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/processors/index.ts (100%) rename plugins/{catalog-backend-extension-msgraph => catalog-backend-module-msgraph}/src/setupTests.ts (100%) diff --git a/.changeset/metal-badgers-carry.md b/.changeset/metal-badgers-carry.md index 17592e062f..8d0cc3715d 100644 --- a/.changeset/metal-badgers-carry.md +++ b/.changeset/metal-badgers-carry.md @@ -1,10 +1,10 @@ --- '@backstage/plugin-catalog-backend': patch -'@backstage/plugin-catalog-backend-extension-msgraph': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch --- Move `MicrosoftGraphOrgReaderProcessor` from `@backstage/plugin-catalog-backend` -to `@backstage/plugin-catalog-backend-extension-msgraph`. +to `@backstage/plugin-catalog-backend-module-msgraph`. For now `MicrosoftGraphOrgReaderProcessor` is only deprecated in `@backstage/plugin-catalog-backend`, but will be removed in the future. While it diff --git a/.changeset/silent-ways-laugh.md b/.changeset/silent-ways-laugh.md index cdb28a520c..ea152c68ef 100644 --- a/.changeset/silent-ways-laugh.md +++ b/.changeset/silent-ways-laugh.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend-extension-msgraph': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch --- Allow customizations of `MicrosoftGraphOrgReaderProcessor` by passing an diff --git a/packages/backend/package.json b/packages/backend/package.json index b0fb1ec9d4..aceace8764 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -36,7 +36,7 @@ "@backstage/plugin-auth-backend": "^0.3.12", "@backstage/plugin-badges-backend": "^0.1.6", "@backstage/plugin-catalog-backend": "^0.10.2", - "@backstage/plugin-catalog-backend-extension-msgraph": "^0.1.0", + "@backstage/plugin-catalog-backend-module-msgraph": "^0.1.0", "@backstage/plugin-code-coverage-backend": "^0.1.6", "@backstage/plugin-graphql-backend": "^0.1.8", "@backstage/plugin-kubernetes-backend": "^0.3.8", diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 57afe4fefd..e64fd1f2da 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -16,7 +16,7 @@ import { CatalogBuilder, - createRouter, + createRouter } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; diff --git a/plugins/catalog-backend-extension-msgraph/.eslintrc.js b/plugins/catalog-backend-module-msgraph/.eslintrc.js similarity index 100% rename from plugins/catalog-backend-extension-msgraph/.eslintrc.js rename to plugins/catalog-backend-module-msgraph/.eslintrc.js diff --git a/plugins/catalog-backend-extension-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md similarity index 94% rename from plugins/catalog-backend-extension-msgraph/README.md rename to plugins/catalog-backend-module-msgraph/README.md index 8296d277df..e2f13ebac3 100644 --- a/plugins/catalog-backend-extension-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -8,13 +8,13 @@ users and groups from Office 365. ## Getting Started 1. The processor is not installed by default, therefore you have to add a - dependency to `@backstage/plugin-catalog-backend-extension-msgraph` to your + dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your backend package. ```bash # From your Backstage root directory cd packages/backend -yarn add @backstage/plugin-catalog-backend-extension-msgraph +yarn add @backstage/plugin-catalog-backend-module-msgraph ``` 2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you have to register it in the catalog plugin: diff --git a/plugins/catalog-backend-extension-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/config.d.ts rename to plugins/catalog-backend-module-msgraph/config.d.ts diff --git a/plugins/catalog-backend-extension-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json similarity index 90% rename from plugins/catalog-backend-extension-msgraph/package.json rename to plugins/catalog-backend-module-msgraph/package.json index 9feb8e973f..65794cbd91 100644 --- a/plugins/catalog-backend-extension-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -1,5 +1,5 @@ { - "name": "@backstage/plugin-catalog-backend-extension-msgraph", + "name": "@backstage/plugin-catalog-backend-module-msgraph", "version": "0.1.0", "main": "src/index.ts", "types": "src/index.ts", @@ -14,7 +14,7 @@ "repository": { "type": "git", "url": "https://github.com/backstage/backstage", - "directory": "plugins/catalog-backend-extension-msgraph" + "directory": "plugins/catalog-backend-module-msgraph" }, "keywords": [ "backstage" diff --git a/plugins/catalog-backend-extension-msgraph/src/index.ts b/plugins/catalog-backend-module-msgraph/src/index.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/index.ts rename to plugins/catalog-backend-module-msgraph/src/index.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/constants.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/constants.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.test.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.test.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.test.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.test.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts rename to plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/processors/index.ts b/plugins/catalog-backend-module-msgraph/src/processors/index.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/processors/index.ts rename to plugins/catalog-backend-module-msgraph/src/processors/index.ts diff --git a/plugins/catalog-backend-extension-msgraph/src/setupTests.ts b/plugins/catalog-backend-module-msgraph/src/setupTests.ts similarity index 100% rename from plugins/catalog-backend-extension-msgraph/src/setupTests.ts rename to plugins/catalog-backend-module-msgraph/src/setupTests.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts index a25b3bc506..40a7251fcf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -33,7 +33,7 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; * Extracts teams and users out of a the Microsoft Graph API. * * @deprecated Use the MicrosoftGraphOrgReaderProcessor from package - * @backstage/plugin-catalog-backend-extension-msgraph instead. + * @backstage/plugin-catalog-backend-module-msgraph instead. */ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly providers: MicrosoftGraphProviderConfig[]; @@ -65,7 +65,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { } this.logger.warn( - 'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-extension-msgraph instead.', + 'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-module-msgraph instead.', ); const provider = this.providers.find(p => From 265227f32067bf1101d56f777c6606d88422f11a Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 10 Jun 2021 13:00:09 +0200 Subject: [PATCH 08/17] Remove example code Signed-off-by: Oliver Sand --- .changeset/metal-badgers-carry.md | 6 ------ packages/backend/package.json | 1 - 2 files changed, 7 deletions(-) diff --git a/.changeset/metal-badgers-carry.md b/.changeset/metal-badgers-carry.md index 8d0cc3715d..11f1e2f01b 100644 --- a/.changeset/metal-badgers-carry.md +++ b/.changeset/metal-badgers-carry.md @@ -9,9 +9,3 @@ to `@backstage/plugin-catalog-backend-module-msgraph`. For now `MicrosoftGraphOrgReaderProcessor` is only deprecated in `@backstage/plugin-catalog-backend`, but will be removed in the future. While it is now registered by default, it has to be registered manually in the future. - -TODO: Do we really want to deprecate the transformer before removing it? -It is actually pretty hard to switch to the new transformer as one has to call -`builder.replaceProcessors()` to replace ALL transformers. -As an alternative we can do a breaking change directly with the migration steps -(adding the dependency, adding an import and calling `builder.addProcessor()`). diff --git a/packages/backend/package.json b/packages/backend/package.json index aceace8764..5195a7c28f 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -36,7 +36,6 @@ "@backstage/plugin-auth-backend": "^0.3.12", "@backstage/plugin-badges-backend": "^0.1.6", "@backstage/plugin-catalog-backend": "^0.10.2", - "@backstage/plugin-catalog-backend-module-msgraph": "^0.1.0", "@backstage/plugin-code-coverage-backend": "^0.1.6", "@backstage/plugin-graphql-backend": "^0.1.8", "@backstage/plugin-kubernetes-backend": "^0.3.8", From 4384bc4ec1839a924a9065f95e6141533a16f3e1 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 15 Jun 2021 17:57:07 +0200 Subject: [PATCH 09/17] Fix formatting Signed-off-by: Oliver Sand --- packages/backend/src/plugins/catalog.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index e64fd1f2da..57afe4fefd 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -16,7 +16,7 @@ import { CatalogBuilder, - createRouter + createRouter, } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; From d4d9f13693d6e3a5915ca1be0cf63ae08b5f36cf Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 16 Jun 2021 09:43:40 +0200 Subject: [PATCH 10/17] Improve README Signed-off-by: Oliver Sand --- .../catalog-backend-module-msgraph/README.md | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index e2f13ebac3..0f1fc360e6 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -1,8 +1,8 @@ -# Catalog Backend Extension for Microsoft Graph +# Catalog Backend Module for Microsoft Graph -This is an extension to the `plugin-catalog-backend` plugin, providing a +This is an extension module to the `plugin-catalog-backend` plugin, providing a `MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data -from the Microsoft Graph API. This processor is useful, if you want to import +from the Microsoft Graph API. This processor is useful if you want to import users and groups from Office 365. ## Getting Started @@ -17,7 +17,8 @@ cd packages/backend yarn add @backstage/plugin-catalog-backend-module-msgraph ``` -2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you have to register it in the catalog plugin: +2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you + have to register it in the catalog plugin: ```typescript // packages/backend/src/plugins/catalog.ts @@ -28,7 +29,13 @@ builder.addProcessor( ); ``` -3. Configure the processor: +3. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/). + The App registration requires at least the API permissions `Group.Read.All`, + `GroupMember.Read.All`, `User.Read` and `User.Read.All` for Microsoft Graph + (if you still run into errors about insufficient privileges, add + `Team.ReadBasic.All` and `TeamMember.Read.All` too). + +4. Configure the processor: ```yaml # app-config.yaml @@ -38,18 +45,41 @@ catalog: providers: - target: https://graph.microsoft.com/v1.0 authority: https://login.microsoftonline.com + # If you don't know you tenantId, you can use Microsoft Graph Explorer + # to query it tenantId: ${MICROSOFT_GRAPH_TENANT_ID} + # Client Id and Secret can be created under Certificates & secrets in + # the App registration in the Microsoft Azure Portal. clientId: ${MICROSOFT_GRAPH_CLIENT_ID} clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} # Optional filter for user, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties + # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter userFilter: accountEnabled eq true and userType eq 'member' # Optional filter for group, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') ``` +5. Add a location that ingests from Microsoft Graph: + +```yaml +# app-config.yaml +catalog: + locations: + - type: microsoft-graph-org + target: https://graph.microsoft.com/v1.0 + # If you catalog doesn't allow to import Group and User entities by + # default, allow them here + rules: + - allow: [Group, User] + … +``` + ## Customize the Processor -In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` allows to pass transformers for users, groups and the organization. +In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` +allows to pass transformers for users, groups and the organization. 1. Create a transformer: From 160181779b919f79f7b73b090bc7a42c9daef17a Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 16 Jun 2021 09:53:28 +0200 Subject: [PATCH 11/17] Add docs to microsite Closes #4627 Signed-off-by: Oliver Sand --- docs/integrations/azure/locations.md | 4 ++-- docs/integrations/azure/org.md | 14 ++++++++++++++ microsite/sidebars.json | 7 +++++-- mkdocs.yml | 3 ++- plugins/catalog-backend-module-msgraph/README.md | 2 +- 5 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 docs/integrations/azure/org.md diff --git a/docs/integrations/azure/locations.md b/docs/integrations/azure/locations.md index 163674f00a..3b3e3dd5ed 100644 --- a/docs/integrations/azure/locations.md +++ b/docs/integrations/azure/locations.md @@ -6,8 +6,8 @@ sidebar_label: Locations description: Integrating source code stored in Azure DevOps into the Backstage catalog --- -The Azure integration supports loading catalog entities from Azure DevOps. -Entities can be added to +The Azure DevOps integration supports loading catalog entities from Azure +DevOps. Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the [catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md new file mode 100644 index 0000000000..c359960f4d --- /dev/null +++ b/docs/integrations/azure/org.md @@ -0,0 +1,14 @@ +--- +id: org +title: Microsoft Azure Active Directory Organizational Data +sidebar_label: Org Data +# prettier-ignore +description: Importing users and groups from a Microsoft Azure Active Directory into Backstage +--- + +The Backstage catalog can be set up to ingest organizational data - users and +teams - directly from an tenant in Microsoft Azure Active Directory via the +Microsoft Graph API. + +More details on this are available in the +[README of the `@backstage/plugin-catalog-backend-module-msgraph` package](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-msgraph/README.md). diff --git a/microsite/sidebars.json b/microsite/sidebars.json index e0add7a007..9021d15884 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -102,8 +102,11 @@ "integrations/index", { "type": "subcategory", - "label": "Azure DevOps", - "ids": ["integrations/azure/locations"] + "label": "Azure", + "ids": [ + "integrations/azure/locations", + "integrations/azure/org" + ] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 4b2b29a997..6c4af107ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,8 +75,9 @@ nav: - FAQ: 'features/techdocs/FAQ.md' - Integrations: - Overview: 'integrations/index.md' - - Azure DevOps: + - Azure: - Locations: 'integrations/azure/locations.md' + - Org Data: 'integrations/azure/org.md' - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' - Discovery: 'integrations/bitbucket/discovery.md' diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 0f1fc360e6..df141901aa 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -3,7 +3,7 @@ This is an extension module to the `plugin-catalog-backend` plugin, providing a `MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data from the Microsoft Graph API. This processor is useful if you want to import -users and groups from Office 365. +users and groups from Azure Active Directory or Office 365. ## Getting Started From 579cf97f0219f8e36b760239dffcf97b160d680c Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 16 Jun 2021 11:14:53 +0200 Subject: [PATCH 12/17] Remove `MicrosoftGraphOrgReaderProcessor` from `plugin-catalog-backend` Signed-off-by: Oliver Sand --- .changeset/metal-badgers-carry.md | 20 +- plugins/catalog-backend/config.d.ts | 48 --- plugins/catalog-backend/package.json | 2 - .../MicrosoftGraphOrgReaderProcessor.ts | 110 ------ .../src/ingestion/processors/index.ts | 1 - .../processors/microsoftGraph/client.test.ts | 363 ------------------ .../processors/microsoftGraph/client.ts | 235 ------------ .../processors/microsoftGraph/config.test.ts | 75 ---- .../processors/microsoftGraph/config.ts | 91 ----- .../processors/microsoftGraph/constants.ts | 32 -- .../processors/microsoftGraph/index.ts | 24 -- .../processors/microsoftGraph/read.test.ts | 347 ----------------- .../processors/microsoftGraph/read.ts | 363 ------------------ .../src/next/NextCatalogBuilder.ts | 2 - .../src/service/CatalogBuilder.ts | 2 - yarn.lock | 2 +- 16 files changed, 18 insertions(+), 1699 deletions(-) delete mode 100644 plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts diff --git a/.changeset/metal-badgers-carry.md b/.changeset/metal-badgers-carry.md index 11f1e2f01b..b82e638c6f 100644 --- a/.changeset/metal-badgers-carry.md +++ b/.changeset/metal-badgers-carry.md @@ -6,6 +6,20 @@ Move `MicrosoftGraphOrgReaderProcessor` from `@backstage/plugin-catalog-backend` to `@backstage/plugin-catalog-backend-module-msgraph`. -For now `MicrosoftGraphOrgReaderProcessor` is only deprecated in -`@backstage/plugin-catalog-backend`, but will be removed in the future. While it -is now registered by default, it has to be registered manually in the future. +The `MicrosoftGraphOrgReaderProcessor` isn't registered by default anymore, if +you want to continue using it you have to register it manually at the catalog +builder: + +1. Add dependency to `@backstage/plugin-catalog-backend-module-msgraph` to the `package.json` of your backend. +2. Add the processor to the catalog builder: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + }), +); +``` + +For more configuration details, see the [README of the `@backstage/plugin-catalog-backend-module-msgraph` package](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-msgraph/README.md). diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 20fcd975cf..5417a84741 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -362,54 +362,6 @@ export interface Config { roleArn?: string; }; }; - - /** - * MicrosoftGraphOrgReaderProcessor configuration - */ - microsoftGraphOrg?: { - /** - * The configuration parameters for each single Microsoft Graph provider. - */ - providers: Array<{ - /** - * The prefix of the target that this matches on, e.g. - * "https://graph.microsoft.com/v1.0", with no trailing slash. - */ - target: string; - /** - * The auth authority used. - * - * Default value "https://login.microsoftonline.com" - */ - authority?: string; - /** - * The tenant whose org data we are interested in. - */ - tenantId: string; - /** - * The OAuth client ID to use for authenticating requests. - */ - clientId: string; - /** - * The OAuth client secret to use for authenticating requests. - * - * @visibility secret - */ - clientSecret: string; - /** - * The filter to apply to extract users. - * - * E.g. "accountEnabled eq true and userType eq 'member'" - */ - userFilter?: string; - /** - * The filter to apply to extract groups. - * - * E.g. "securityEnabled eq false and mailEnabled eq true" - */ - groupFilter?: string; - }>; - }; }; }; } diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 6bc53b43da..8e9f54504e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -29,7 +29,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@azure/msal-node": "^1.0.0-beta.3", "@backstage/backend-common": "^0.8.3", "@backstage/catalog-client": "^0.3.13", "@backstage/catalog-model": "^0.8.3", @@ -38,7 +37,6 @@ "@backstage/integration": "^0.5.6", "@backstage/plugin-search-backend-node": "^0.2.1", "@backstage/search-common": "^0.1.2", - "@microsoft/microsoft-graph-types": "^1.25.0", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts deleted file mode 100644 index 40a7251fcf..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import { Config } from '@backstage/config'; -import { Logger } from 'winston'; -import { - MicrosoftGraphClient, - MicrosoftGraphProviderConfig, - readMicrosoftGraphConfig, - readMicrosoftGraphOrg, -} from './microsoftGraph'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; - -// TODO: Remove this deprecated processor, the related code in -// ./microsoftGraph/, and the config section in the future. - -/** - * Extracts teams and users out of a the Microsoft Graph API. - * - * @deprecated Use the MicrosoftGraphOrgReaderProcessor from package - * @backstage/plugin-catalog-backend-module-msgraph instead. - */ -export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { - private readonly providers: MicrosoftGraphProviderConfig[]; - private readonly logger: Logger; - - static fromConfig(config: Config, options: { logger: Logger }) { - const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); - return new MicrosoftGraphOrgReaderProcessor({ - ...options, - providers: c ? readMicrosoftGraphConfig(c) : [], - }); - } - - constructor(options: { - providers: MicrosoftGraphProviderConfig[]; - logger: Logger; - }) { - this.providers = options.providers; - this.logger = options.logger; - } - - async readLocation( - location: LocationSpec, - _optional: boolean, - emit: CatalogProcessorEmit, - ): Promise { - if (location.type !== 'microsoft-graph-org') { - return false; - } - - this.logger.warn( - 'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-module-msgraph instead.', - ); - - const provider = this.providers.find(p => - location.target.startsWith(p.target), - ); - if (!provider) { - throw new Error( - `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`, - ); - } - - // Read out all of the raw data - const startTimestamp = Date.now(); - this.logger.info('Reading Microsoft Graph users and groups'); - - // We create a client each time as we need one that matches the specific provider - const client = MicrosoftGraphClient.create(provider); - const { users, groups } = await readMicrosoftGraphOrg( - client, - provider.tenantId, - { - userFilter: provider.userFilter, - groupFilter: provider.groupFilter, - }, - ); - - const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); - this.logger.debug( - `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`, - ); - - // Done! - for (const group of groups) { - emit(results.entity(location, group)); - } - for (const user of users) { - emit(results.entity(location, user)); - } - - return true; - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index a92cc9ed3b..8b87e18018 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -27,7 +27,6 @@ export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; -export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts deleted file mode 100644 index 5ef82432bb..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as msal from '@azure/msal-node'; -import { msw } from '@backstage/test-utils'; -import { rest } from 'msw'; -import { setupServer } from 'msw/node'; -import { MicrosoftGraphClient } from './client'; - -describe('MicrosoftGraphClient', () => { - const confidentialClientApplication: jest.Mocked = { - acquireTokenByClientCredential: jest.fn(), - } as any; - let client: MicrosoftGraphClient; - const worker = setupServer(); - - msw.setupDefaultHandlers(worker); - - beforeEach(() => { - confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue( - { token: 'ACCESS_TOKEN' } as any, - ); - client = new MicrosoftGraphClient( - 'https://example.com', - confidentialClientApplication, - ); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - it('should perform raw request', async () => { - worker.use( - rest.get('https://other.example.com/', (_, res, ctx) => - res(ctx.status(200), ctx.json({ value: 'example' })), - ), - ); - - const response = await client.requestRaw('https://other.example.com/'); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ value: 'example' }); - expect( - confidentialClientApplication.acquireTokenByClientCredential, - ).toBeCalledTimes(1); - expect( - confidentialClientApplication.acquireTokenByClientCredential, - ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] }); - }); - - it('should perform simple api request', async () => { - worker.use( - rest.get('https://example.com/users', (_, res, ctx) => - res(ctx.status(200), ctx.json({ value: 'example' })), - ), - ); - - const response = await client.requestApi('users'); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ value: 'example' }); - }); - - it('should perform api request with filter, select and expand', async () => { - worker.use( - rest.get('https://example.com/users', (req, res, ctx) => - res(ctx.status(200), ctx.json({ queryString: req.url.search })), - ), - ); - - const response = await client.requestApi('users', { - filter: 'test eq true', - expand: ['children'], - select: ['id', 'children'], - }); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ - queryString: - '?$filter=test%20eq%20true&$select=id,children&$expand=children', - }); - }); - - it('should perform collection request for a single page', async () => { - worker.use( - rest.get('https://example.com/users', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: ['first'], - }), - ), - ), - ); - - const values = await collectAsyncIterable( - client.requestCollection('users'), - ); - - expect(values).toEqual(['first']); - }); - - it('should perform collection request for multiple pages', async () => { - worker.use( - rest.get('https://example.com/users', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: ['first'], - '@odata.nextLink': 'https://example.com/users2', - }), - ), - ), - ); - worker.use( - rest.get('https://example.com/users2', (_, res, ctx) => - res(ctx.status(200), ctx.json({ value: ['second'] })), - ), - ); - - const values = await collectAsyncIterable( - client.requestCollection('users'), - ); - - expect(values).toEqual(['first', 'second']); - }); - - it('should load user profile', async () => { - worker.use( - rest.get('https://example.com/users/user-id', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - surname: 'Example', - }), - ), - ), - ); - - const userProfile = await client.getUserProfile('user-id'); - - expect(userProfile).toEqual({ surname: 'Example' }); - }); - - it('should throw expection if load user profile fails', async () => { - worker.use( - rest.get('https://example.com/users/user-id', (_, res, ctx) => - res(ctx.status(404)), - ), - ); - - await expect(() => client.getUserProfile('user-id')).rejects.toThrowError(); - }); - - it('should load user profile photo with max size of 120', async () => { - worker.use( - rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: [ - { - height: 120, - id: 120, - }, - { - height: 500, - id: 500, - }, - ], - }), - ), - ), - ); - worker.use( - rest.get( - 'https://example.com/users/user-id/photos/120/*', - (_, res, ctx) => res(ctx.status(200), ctx.text('911')), - ), - ); - - const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); - - expect(photo).toEqual('data:image/jpeg;base64,OTEx'); - }); - - it('should not fail if user has no profile photo', async () => { - worker.use( - rest.get('https://example.com/users/user-id/photos', (_, res, ctx) => - res(ctx.status(404)), - ), - ); - - const photo = await client.getUserPhotoWithSizeLimit('user-id', 120); - - expect(photo).toBeFalsy(); - }); - - it('should load user profile photo', async () => { - worker.use( - rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) => - res(ctx.status(200), ctx.text('911')), - ), - ); - - const photo = await client.getUserPhoto('user-id'); - - expect(photo).toEqual('data:image/jpeg;base64,OTEx'); - }); - - it('should load user profile photo for size 120', async () => { - worker.use( - rest.get( - 'https://example.com/users/user-id/photos/120/*', - (_, res, ctx) => res(ctx.status(200), ctx.text('911')), - ), - ); - - const photo = await client.getUserPhoto('user-id', '120'); - - expect(photo).toEqual('data:image/jpeg;base64,OTEx'); - }); - - it('should load users', async () => { - worker.use( - rest.get('https://example.com/users', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: [{ surname: 'Example' }], - }), - ), - ), - ); - - const values = await collectAsyncIterable(client.getUsers()); - - expect(values).toEqual([{ surname: 'Example' }]); - }); - - it('should load group profile photo with max size of 120', async () => { - worker.use( - rest.get('https://example.com/groups/group-id/photos', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: [ - { - height: 120, - id: 120, - }, - ], - }), - ), - ), - ); - worker.use( - rest.get( - 'https://example.com/groups/group-id/photos/120/*', - (_, res, ctx) => res(ctx.status(200), ctx.text('911')), - ), - ); - - const photo = await client.getGroupPhotoWithSizeLimit('group-id', 120); - - expect(photo).toEqual('data:image/jpeg;base64,OTEx'); - }); - - it('should load group profile photo', async () => { - worker.use( - rest.get('https://example.com/groups/group-id/photo/*', (_, res, ctx) => - res(ctx.status(200), ctx.text('911')), - ), - ); - - const photo = await client.getGroupPhoto('group-id'); - - expect(photo).toEqual('data:image/jpeg;base64,OTEx'); - }); - - it('should load groups', async () => { - worker.use( - rest.get('https://example.com/groups', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: [{ displayName: 'Example' }], - }), - ), - ), - ); - - const values = await collectAsyncIterable(client.getGroups()); - - expect(values).toEqual([{ displayName: 'Example' }]); - }); - - it('should load group members', async () => { - worker.use( - rest.get('https://example.com/groups/group-id/members', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - value: [ - { '@odata.type': '#microsoft.graph.user' }, - { '@odata.type': '#microsoft.graph.group' }, - ], - }), - ), - ), - ); - - const values = await collectAsyncIterable( - client.getGroupMembers('group-id'), - ); - - expect(values).toEqual([ - { '@odata.type': '#microsoft.graph.user' }, - { '@odata.type': '#microsoft.graph.group' }, - ]); - }); - - it('should load organization', async () => { - worker.use( - rest.get('https://example.com/organization/tentant-id', (_, res, ctx) => - res( - ctx.status(200), - ctx.json({ - displayName: 'Example', - }), - ), - ), - ); - - const organization = await client.getOrganization('tentant-id'); - - expect(organization).toEqual({ displayName: 'Example' }); - }); -}); - -async function collectAsyncIterable( - iterable: AsyncIterable, -): Promise { - const values = []; - for await (const value of iterable) { - values.push(value); - } - return values; -} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts deleted file mode 100644 index 3dfc58e773..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as msal from '@azure/msal-node'; -import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; -import fetch from 'cross-fetch'; -import qs from 'qs'; -import { MicrosoftGraphProviderConfig } from './config'; - -export type ODataQuery = { - filter?: string; - expand?: string[]; - select?: string[]; -}; - -export type GroupMember = - | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' }) - | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' }); - -export class MicrosoftGraphClient { - static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient { - const clientConfig: msal.Configuration = { - auth: { - clientId: config.clientId, - clientSecret: config.clientSecret, - authority: `${config.authority}/${config.tenantId}`, - }, - }; - const pca = new msal.ConfidentialClientApplication(clientConfig); - return new MicrosoftGraphClient(config.target, pca); - } - - constructor( - private readonly baseUrl: string, - private readonly pca: msal.ConfidentialClientApplication, - ) {} - - async *requestCollection( - path: string, - query?: ODataQuery, - ): AsyncIterable { - let response = await this.requestApi(path, query); - - for (;;) { - if (response.status !== 200) { - await this.handleError(path, response); - } - - const result = await response.json(); - const elements: T[] = result.value; - - yield* elements; - - // Follow cursor to the next page if one is available - if (!result['@odata.nextLink']) { - return; - } - - response = await this.requestRaw(result['@odata.nextLink']); - } - } - - async requestApi(path: string, query?: ODataQuery): Promise { - const queryString = qs.stringify( - { - $filter: query?.filter, - $select: query?.select?.join(','), - $expand: query?.expand?.join(','), - }, - { - addQueryPrefix: true, - // Microsoft Graph doesn't like an encoded query string - encode: false, - }, - ); - - return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`); - } - - async requestRaw(url: string): Promise { - // Make sure that we always have a valid access token (might be cached) - const token = await this.pca.acquireTokenByClientCredential({ - scopes: ['https://graph.microsoft.com/.default'], - }); - - if (!token) { - throw new Error('Error while requesting token for Microsoft Graph'); - } - - return await fetch(url, { - headers: { - Authorization: `Bearer ${token.accessToken}`, - }, - }); - } - - async getUserProfile(userId: string): Promise { - const response = await this.requestApi(`users/${userId}`); - - if (response.status !== 200) { - await this.handleError('user profile', response); - } - - return await response.json(); - } - - async getUserPhotoWithSizeLimit( - userId: string, - maxSize: number, - ): Promise { - return await this.getPhotoWithSizeLimit('users', userId, maxSize); - } - - async getUserPhoto( - userId: string, - sizeId?: string, - ): Promise { - return await this.getPhoto('users', userId, sizeId); - } - - async *getUsers(query?: ODataQuery): AsyncIterable { - yield* this.requestCollection(`users`, query); - } - - async getGroupPhotoWithSizeLimit( - groupId: string, - maxSize: number, - ): Promise { - return await this.getPhotoWithSizeLimit('groups', groupId, maxSize); - } - - async getGroupPhoto( - groupId: string, - sizeId?: string, - ): Promise { - return await this.getPhoto('groups', groupId, sizeId); - } - - async *getGroups(query?: ODataQuery): AsyncIterable { - yield* this.requestCollection(`groups`, query); - } - - async *getGroupMembers(groupId: string): AsyncIterable { - yield* this.requestCollection(`groups/${groupId}/members`); - } - - async getOrganization( - tenantId: string, - ): Promise { - const response = await this.requestApi(`organization/${tenantId}`); - - if (response.status !== 200) { - await this.handleError(`organization/${tenantId}`, response); - } - - return await response.json(); - } - - private async getPhotoWithSizeLimit( - entityName: string, - id: string, - maxSize: number, - ): Promise { - const response = await this.requestApi(`${entityName}/${id}/photos`); - - if (response.status === 404) { - return undefined; - } else if (response.status !== 200) { - await this.handleError(`${entityName} photos`, response); - } - - const result = await response.json(); - const photos = result.value as MicrosoftGraph.ProfilePhoto[]; - let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined; - - // Find the biggest picture that is smaller than the max size - for (const p of photos) { - if ( - !selectedPhoto || - (p.height! >= selectedPhoto.height! && p.height! <= maxSize) - ) { - selectedPhoto = p; - } - } - - if (!selectedPhoto) { - return undefined; - } - - return await this.getPhoto(entityName, id, selectedPhoto.id!); - } - - private async getPhoto( - entityName: string, - id: string, - sizeId?: string, - ): Promise { - const path = sizeId - ? `${entityName}/${id}/photos/${sizeId}/$value` - : `${entityName}/${id}/photo/$value`; - const response = await this.requestApi(path); - - if (response.status === 404) { - return undefined; - } else if (response.status !== 200) { - await this.handleError('photo', response); - } - - return `data:image/jpeg;base64,${Buffer.from( - await response.arrayBuffer(), - ).toString('base64')}`; - } - - private async handleError(path: string, response: Response): Promise { - const result = await response.json(); - const error = result.error as MicrosoftGraph.PublicError; - - throw new Error( - `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`, - ); - } -} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts deleted file mode 100644 index 4671fd23ae..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ConfigReader } from '@backstage/config'; -import { readMicrosoftGraphConfig } from './config'; - -describe('readMicrosoftGraphConfig', () => { - it('applies all of the defaults', () => { - const config = { - providers: [ - { - target: 'target', - tenantId: 'tenantId', - clientId: 'clientId', - clientSecret: 'clientSecret', - }, - ], - }; - const actual = readMicrosoftGraphConfig(new ConfigReader(config)); - const expected = [ - { - target: 'target', - tenantId: 'tenantId', - clientId: 'clientId', - clientSecret: 'clientSecret', - authority: 'https://login.microsoftonline.com', - userFilter: undefined, - groupFilter: undefined, - }, - ]; - expect(actual).toEqual(expected); - }); - - it('reads all the values', () => { - const config = { - providers: [ - { - target: 'target', - tenantId: 'tenantId', - clientId: 'clientId', - clientSecret: 'clientSecret', - authority: 'https://login.example.com/', - userFilter: 'accountEnabled eq true', - groupFilter: 'securityEnabled eq false', - }, - ], - }; - const actual = readMicrosoftGraphConfig(new ConfigReader(config)); - const expected = [ - { - target: 'target', - tenantId: 'tenantId', - clientId: 'clientId', - clientSecret: 'clientSecret', - authority: 'https://login.example.com', - userFilter: 'accountEnabled eq true', - groupFilter: 'securityEnabled eq false', - }, - ]; - expect(actual).toEqual(expected); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts deleted file mode 100644 index 72416a63ee..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Config } from '@backstage/config'; - -/** - * The configuration parameters for a single Microsoft Graph provider. - */ -export type MicrosoftGraphProviderConfig = { - /** - * The prefix of the target that this matches on, e.g. - * "https://graph.microsoft.com/v1.0", with no trailing slash. - */ - target: string; - /** - * The auth authority used. - * - * E.g. "https://login.microsoftonline.com" - */ - authority?: string; - /** - * The tenant whose org data we are interested in. - */ - tenantId: string; - /** - * The OAuth client ID to use for authenticating requests. - */ - clientId: string; - /** - * The OAuth client secret to use for authenticating requests. - * - * @visibility secret - */ - clientSecret: string; - /** - * The filter to apply to extract users. - * - * E.g. "accountEnabled eq true and userType eq 'member'" - */ - userFilter?: string; - /** - * The filter to apply to extract groups. - * - * E.g. "securityEnabled eq false and mailEnabled eq true" - */ - groupFilter?: string; -}; - -export function readMicrosoftGraphConfig( - config: Config, -): MicrosoftGraphProviderConfig[] { - const providers: MicrosoftGraphProviderConfig[] = []; - const providerConfigs = config.getOptionalConfigArray('providers') ?? []; - - for (const providerConfig of providerConfigs) { - const target = providerConfig.getString('target').replace(/\/+$/, ''); - const authority = - providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') || - 'https://login.microsoftonline.com'; - const tenantId = providerConfig.getString('tenantId'); - const clientId = providerConfig.getString('clientId'); - const clientSecret = providerConfig.getString('clientSecret'); - const userFilter = providerConfig.getOptionalString('userFilter'); - const groupFilter = providerConfig.getOptionalString('groupFilter'); - - providers.push({ - target, - authority, - tenantId, - clientId, - clientSecret, - userFilter, - groupFilter, - }); - } - - return providers; -} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts deleted file mode 100644 index 6d34d0c159..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The tenant id used by the Microsoft Graph API - */ -export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION = - 'graph.microsoft.com/tenant-id'; - -/** - * The group id used by the Microsoft Graph API - */ -export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION = - 'graph.microsoft.com/group-id'; - -/** - * The user id used by the Microsoft Graph API - */ -export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts deleted file mode 100644 index 882125fd84..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export { MicrosoftGraphClient } from './client'; -export type { MicrosoftGraphProviderConfig } from './config'; -export { readMicrosoftGraphConfig } from './config'; -export { readMicrosoftGraphOrg } from './read'; -export { - MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, - MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, - MICROSOFT_GRAPH_USER_ID_ANNOTATION, -} from './constants'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts deleted file mode 100644 index 07d540c848..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; -import merge from 'lodash/merge'; -import { RecursivePartial } from '../../../util'; -import { GroupMember, MicrosoftGraphClient } from './client'; -import { - normalizeEntityName, - readMicrosoftGraphGroups, - readMicrosoftGraphOrganization, - readMicrosoftGraphUsers, - resolveRelations, -} from './read'; - -function user(data: RecursivePartial): UserEntity { - return merge( - {}, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { name: 'name' }, - spec: { profile: {}, memberOf: [] }, - } as UserEntity, - data, - ); -} - -function group(data: RecursivePartial): GroupEntity { - return merge( - {}, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: 'name', - }, - spec: { - children: [], - type: 'team', - }, - } as GroupEntity, - data, - ); -} - -describe('read microsoft graph', () => { - const client: jest.Mocked = { - getUsers: jest.fn(), - getGroups: jest.fn(), - getGroupMembers: jest.fn(), - getUserPhotoWithSizeLimit: jest.fn(), - getGroupPhotoWithSizeLimit: jest.fn(), - getOrganization: jest.fn(), - } as any; - - afterEach(() => jest.resetAllMocks()); - - describe('normalizeEntityName', () => { - it('should normalize name to valid entity name', () => { - expect(normalizeEntityName('User Name')).toBe('user_name'); - }); - - it('should normalize e-mail to valid entity name', () => { - expect(normalizeEntityName('user.name@example.com')).toBe( - 'user.name_example.com', - ); - }); - }); - - describe('readMicrosoftGraphUsers', () => { - it('should read users', async () => { - async function* getExampleUsers() { - yield { - id: 'userid', - displayName: 'User Name', - mail: 'user.name@example.com', - }; - } - - client.getUsers.mockImplementation(getExampleUsers); - client.getUserPhotoWithSizeLimit.mockResolvedValue( - 'data:image/jpeg;base64,...', - ); - - const { users } = await readMicrosoftGraphUsers(client, { - userFilter: 'accountEnabled eq true', - }); - - expect(users).toEqual([ - user({ - metadata: { - annotations: { - 'graph.microsoft.com/user-id': 'userid', - }, - name: 'user.name_example.com', - }, - spec: { - profile: { - displayName: 'User Name', - email: 'user.name@example.com', - picture: 'data:image/jpeg;base64,...', - }, - }, - }), - ]); - - expect(client.getUsers).toBeCalledTimes(1); - expect(client.getUsers).toBeCalledWith({ - filter: 'accountEnabled eq true', - select: ['id', 'displayName', 'mail'], - }); - expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); - expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); - }); - }); - - describe('readMicrosoftGraphOrganization', () => { - it('should read organization', async () => { - client.getOrganization.mockResolvedValue({ - id: 'tenantid', - displayName: 'Organization Name', - }); - - const { rootGroup } = await readMicrosoftGraphOrganization( - client, - 'tenantid', - ); - - expect(rootGroup).toEqual( - group({ - metadata: { - annotations: { - 'graph.microsoft.com/tenant-id': 'tenantid', - }, - name: 'organization_name', - description: 'Organization Name', - }, - spec: { - type: 'root', - profile: { - displayName: 'Organization Name', - }, - }, - }), - ); - - expect(client.getOrganization).toBeCalledTimes(1); - expect(client.getOrganization).toBeCalledWith('tenantid'); - }); - }); - - describe('readMicrosoftGraphGroups', () => { - it('should read groups', async () => { - async function* getExampleGroups() { - yield { - id: 'groupid', - displayName: 'Group Name', - description: 'Group Description', - mail: 'group@example.com', - }; - } - - async function* getExampleGroupMembers(): AsyncIterable { - yield { - '@odata.type': '#microsoft.graph.group', - id: 'childgroupid', - }; - yield { - '@odata.type': '#microsoft.graph.user', - id: 'userid', - }; - } - - client.getGroups.mockImplementation(getExampleGroups); - client.getGroupMembers.mockImplementation(getExampleGroupMembers); - client.getOrganization.mockResolvedValue({ - id: 'tenantid', - displayName: 'Organization Name', - }); - client.getGroupPhotoWithSizeLimit.mockResolvedValue( - 'data:image/jpeg;base64,...', - ); - - const { - groups, - groupMember, - groupMemberOf, - rootGroup, - } = await readMicrosoftGraphGroups(client, 'tenantid', { - groupFilter: 'securityEnabled eq false', - }); - - const expectedRootGroup = group({ - metadata: { - annotations: { - 'graph.microsoft.com/tenant-id': 'tenantid', - }, - name: 'organization_name', - description: 'Organization Name', - }, - spec: { - type: 'root', - profile: { - displayName: 'Organization Name', - }, - }, - }); - expect(groups).toEqual([ - expectedRootGroup, - group({ - metadata: { - annotations: { - 'graph.microsoft.com/group-id': 'groupid', - }, - name: 'group_name', - description: 'Group Description', - }, - spec: { - type: 'team', - profile: { - displayName: 'Group Name', - email: 'group@example.com', - // TODO: Loading groups doesn't work right now as Microsoft Graph - // doesn't allows this yet - /* picture: 'data:image/jpeg;base64,...',*/ - }, - }, - }), - ]); - expect(rootGroup).toEqual(expectedRootGroup); - expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); - expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); - expect(groupMember.get('organization_name')).toEqual(new Set()); - - expect(client.getGroups).toBeCalledTimes(1); - expect(client.getGroups).toBeCalledWith({ - filter: 'securityEnabled eq false', - select: ['id', 'displayName', 'description', 'mail', 'mailNickname'], - }); - expect(client.getGroupMembers).toBeCalledTimes(1); - expect(client.getGroupMembers).toBeCalledWith('groupid'); - // TODO: Loading groups doesn't work right now as Microsoft Graph - // doesn't allows this yet - // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); - // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); - }); - }); - - describe('resolveRelations', () => { - it('should resolve relations', async () => { - const rootGroup = group({ - metadata: { - annotations: { - 'graph.microsoft.com/tenant-id': 'tenant-id-root', - }, - name: 'root', - }, - spec: { - type: 'root', - }, - }); - const groupA = group({ - metadata: { - annotations: { - 'graph.microsoft.com/group-id': 'group-id-a', - }, - name: 'a', - }, - }); - const groupB = group({ - metadata: { - annotations: { - 'graph.microsoft.com/group-id': 'group-id-b', - }, - name: 'b', - }, - }); - const groupC = group({ - metadata: { - annotations: { - 'graph.microsoft.com/group-id': 'group-id-c', - }, - name: 'c', - }, - }); - const user1 = user({ - metadata: { - annotations: { - 'graph.microsoft.com/user-id': 'user-id-1', - }, - name: 'user1', - }, - }); - const user2 = user({ - metadata: { - annotations: { - 'graph.microsoft.com/user-id': 'user-id-2', - }, - name: 'user2', - }, - }); - const groups = [rootGroup, groupA, groupB, groupC]; - const users = [user1, user2]; - const groupMember = new Map>(); - groupMember.set('group-id-b', new Set(['group-id-c'])); - const groupMemberOf = new Map>(); - groupMemberOf.set('user-id-1', new Set(['group-id-a'])); - groupMemberOf.set('user-id-2', new Set(['group-id-c'])); - - // We have a root groups - // We have three groups: a, b, c. c is child of b - // we have two users: u1, u2. u1 is member of a, u2 is member of c - resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); - - expect(rootGroup.spec.parent).toBeUndefined(); - expect(rootGroup.spec.children).toEqual( - expect.arrayContaining(['a', 'b']), - ); - - expect(groupA.spec.parent).toEqual('root'); - expect(groupA.spec.children).toEqual(expect.arrayContaining([])); - - expect(groupB.spec.parent).toEqual('root'); - expect(groupB.spec.children).toEqual(expect.arrayContaining(['c'])); - - expect(groupC.spec.parent).toEqual('b'); - expect(groupC.spec.children).toEqual(expect.arrayContaining([])); - - expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a'])); - expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c'])); - }); - }); -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts deleted file mode 100644 index 4409422645..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; -import limiterFactory from 'p-limit'; -import { buildMemberOf, buildOrgHierarchy } from '../util/org'; -import { MicrosoftGraphClient } from './client'; -import { - MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, - MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, - MICROSOFT_GRAPH_USER_ID_ANNOTATION, -} from './constants'; - -export function normalizeEntityName(name: string): string { - return name - .trim() - .toLocaleLowerCase() - .replace(/[^a-zA-Z0-9_\-\.]/g, '_'); -} - -export async function readMicrosoftGraphUsers( - client: MicrosoftGraphClient, - options?: { userFilter?: string }, -): Promise<{ - users: UserEntity[]; // With all relations empty -}> { - const entities: UserEntity[] = []; - const promises: Promise[] = []; - const limiter = limiterFactory(10); - - for await (const user of client.getUsers({ - filter: options?.userFilter, - select: ['id', 'displayName', 'mail'], - })) { - if (!user.id || !user.displayName || !user.mail) { - continue; - } - - const name = normalizeEntityName(user.mail); - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name, - annotations: { - [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!, - }, - }, - spec: { - profile: { - displayName: user.displayName!, - email: user.mail!, - - // TODO: Additional fields? - // jobTitle: user.jobTitle || undefined, - // officeLocation: user.officeLocation || undefined, - // mobilePhone: user.mobilePhone || undefined, - }, - memberOf: [], - }, - }; - - // Download the photos in parallel, otherwise it can take quite some time - const loadPhoto = limiter(async () => { - entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit( - user.id!, - // We are limiting the photo size, as users with full resolution photos - // can make the Backstage API slow - 120, - ); - }); - - promises.push(loadPhoto); - entities.push(entity); - } - - // Wait for all photos to be downloaded - await Promise.all(promises); - - return { users: entities }; -} - -export async function readMicrosoftGraphOrganization( - client: MicrosoftGraphClient, - tenantId: string, -): Promise<{ - rootGroup: GroupEntity; // With all relations empty -}> { - // For now we expect a single root organization - const organization = await client.getOrganization(tenantId); - const name = normalizeEntityName(organization.displayName!); - const rootGroup: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: name, - description: organization.displayName!, - annotations: { - [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!, - }, - }, - spec: { - type: 'root', - profile: { - displayName: organization.displayName!, - }, - children: [], - }, - }; - - return { rootGroup }; -} - -export async function readMicrosoftGraphGroups( - client: MicrosoftGraphClient, - tenantId: string, - options?: { groupFilter?: string }, -): Promise<{ - groups: GroupEntity[]; // With all relations empty - rootGroup: GroupEntity | undefined; // With all relations empty - groupMember: Map>; - groupMemberOf: Map>; -}> { - const groups: GroupEntity[] = []; - const groupMember: Map> = new Map(); - const groupMemberOf: Map> = new Map(); - const limiter = limiterFactory(10); - - const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId); - groupMember.set(rootGroup.metadata.name, new Set()); - groups.push(rootGroup); - - const promises: Promise[] = []; - - for await (const group of client.getGroups({ - filter: options?.groupFilter, - select: ['id', 'displayName', 'description', 'mail', 'mailNickname'], - })) { - if (!group.id || !group.displayName) { - continue; - } - - const name = normalizeEntityName(group.mailNickname || group.displayName); - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: name, - annotations: { - [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id, - }, - }, - spec: { - type: 'team', - profile: {}, - children: [], - }, - }; - - if (group.description) { - entity.metadata.description = group.description; - } - if (group.displayName) { - entity.spec.profile!.displayName = group.displayName; - } - if (group.mail) { - entity.spec.profile!.email = group.mail; - } - - // Download the members in parallel, otherwise it can take quite some time - const loadGroupMembers = limiter(async () => { - for await (const member of client.getGroupMembers(group.id!)) { - if (!member.id) { - continue; - } - - if (member['@odata.type'] === '#microsoft.graph.user') { - ensureItem(groupMemberOf, member.id, group.id!); - } - - if (member['@odata.type'] === '#microsoft.graph.group') { - ensureItem(groupMember, group.id!, member.id); - } - } - }); - - // TODO: Loading groups doesn't work right now as Microsoft Graph doesn't - // allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo - /*/ / Download the photos in parallel, otherwise it can take quite some time - const loadPhoto = limiter(async () => { - entity.spec.profile!.picture = await client.getGroupPhotoWithSizeLimit( - group.id!, - // We are limiting the photo size, as groups with full resolution photos - // can make the Backstage API slow - 120, - ); - }); - - promises.push(loadPhoto);*/ - promises.push(loadGroupMembers); - groups.push(entity); - } - - // Wait for all group members and photos to be loaded - await Promise.all(promises); - - return { - groups, - rootGroup, - groupMember, - groupMemberOf, - }; -} - -export function resolveRelations( - rootGroup: GroupEntity | undefined, - groups: GroupEntity[], - users: UserEntity[], - groupMember: Map>, - groupMemberOf: Map>, -) { - // Build reference lookup tables, we reference them by the id the the graph - const groupMap: Map = new Map(); // by group-id or tenant-id - - for (const group of groups) { - if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) { - groupMap.set( - group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION], - group, - ); - } - if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) { - groupMap.set( - group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION], - group, - ); - } - } - - // Resolve all member relationships into the reverse direction - const parentGroups = new Map>(); - - groupMember.forEach((members, groupId) => - members.forEach(m => ensureItem(parentGroups, m, groupId)), - ); - - // Make sure every group (except root) has at least one parent. If the parent is missing, add the root. - if (rootGroup) { - const tenantId = rootGroup.metadata.annotations![ - MICROSOFT_GRAPH_TENANT_ID_ANNOTATION - ]; - - groups.forEach(group => { - const groupId = group.metadata.annotations![ - MICROSOFT_GRAPH_GROUP_ID_ANNOTATION - ]; - - if (!groupId) { - return; - } - - if (retrieveItems(parentGroups, groupId).size === 0) { - ensureItem(parentGroups, groupId, tenantId); - ensureItem(groupMember, tenantId, groupId); - } - }); - } - - groups.forEach(group => { - const id = - group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ?? - group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]; - - retrieveItems(groupMember, id).forEach(m => { - const childGroup = groupMap.get(m); - if (childGroup) { - group.spec.children.push(childGroup.metadata.name); - } - }); - - retrieveItems(parentGroups, id).forEach(p => { - const parentGroup = groupMap.get(p); - if (parentGroup) { - // TODO: Only having a single parent group might not match every companies model, but fine for now. - group.spec.parent = parentGroup.metadata.name; - } - }); - }); - - // Make sure that all groups have proper parents and children - buildOrgHierarchy(groups); - - // Set relations for all users - users.forEach(user => { - const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION]; - - retrieveItems(groupMemberOf, id).forEach(p => { - const parentGroup = groupMap.get(p); - if (parentGroup) { - user.spec.memberOf.push(parentGroup.metadata.name); - } - }); - }); - - // Make sure all transitive memberships are available - buildMemberOf(groups, users); -} - -export async function readMicrosoftGraphOrg( - client: MicrosoftGraphClient, - tenantId: string, - options?: { userFilter?: string; groupFilter?: string }, -): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { - const { users } = await readMicrosoftGraphUsers(client, { - userFilter: options?.userFilter, - }); - const { - groups, - rootGroup, - groupMember, - groupMemberOf, - } = await readMicrosoftGraphGroups(client, tenantId, { - groupFilter: options?.groupFilter, - }); - - resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); - users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); - groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); - - return { users, groups }; -} - -function ensureItem( - target: Map>, - key: string, - value: string, -) { - let set = target.get(key); - if (!set) { - set = new Set(); - target.set(key, set); - } - set!.add(value); -} - -function retrieveItems( - target: Map>, - key: string, -): Set { - return target.get(key) ?? new Set(); -} diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 2551c16879..0bb5cd0aee 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,7 +50,6 @@ import { GithubDiscoveryProcessor, GithubOrgReaderProcessor, LdapOrgReaderProcessor, - MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -373,7 +372,6 @@ export class NextCatalogBuilder { GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), - MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9bec4442d7..1901d18f72 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -51,7 +51,6 @@ import { LdapOrgReaderProcessor, LocationEntityProcessor, LocationReaders, - MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, StaticLocationProcessor, @@ -319,7 +318,6 @@ export class CatalogBuilder { GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), - MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }), diff --git a/yarn.lock b/yarn.lock index ab538fb58a..0b1af46a72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -225,7 +225,7 @@ dependencies: debug "^4.1.1" -"@azure/msal-node@1.0.0-beta.3", "@azure/msal-node@^1.0.0-beta.3": +"@azure/msal-node@1.0.0-beta.3": version "1.0.0-beta.3" resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-beta.3.tgz#c84c7948028b39e48b901f5fac35bdedcbc8772e" integrity sha512-/KfYRfrsOIrZONvo/0Vi5umuqbPBtCWNtmRvkse64uI0C4CP/W4WXwRD42VMws/8LtKvr1I5rYlYgFzt5zDz/A== From 84cb4410eb87d6383aad57cc0faf8447b4802653 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 16 Jun 2021 11:35:38 +0200 Subject: [PATCH 13/17] Format sidebars.json Signed-off-by: Oliver Sand --- microsite/sidebars.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 9021d15884..ea1e1a4e89 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -103,10 +103,7 @@ { "type": "subcategory", "label": "Azure", - "ids": [ - "integrations/azure/locations", - "integrations/azure/org" - ] + "ids": ["integrations/azure/locations", "integrations/azure/org"] }, { "type": "subcategory", From d6fb1cf1ba56905b40a9034787e8d51afacdc611 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 17 Jun 2021 17:37:58 +0200 Subject: [PATCH 14/17] Upgrade packages Signed-off-by: Oliver Sand --- packages/backend/package.json | 1 - plugins/catalog-backend-module-msgraph/package.json | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 5195a7c28f..f4f304d714 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,7 +31,6 @@ "@backstage/catalog-client": "^0.3.13", "@backstage/catalog-model": "^0.8.2", "@backstage/config": "^0.1.5", - "@backstage/integration": "^0.5.6", "@backstage/plugin-app-backend": "^0.3.13", "@backstage/plugin-auth-backend": "^0.3.12", "@backstage/plugin-badges-backend": "^0.1.6", diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json index 65794cbd91..83bb5dd802 100644 --- a/plugins/catalog-backend-module-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -29,7 +29,7 @@ }, "dependencies": { "@azure/msal-node": "^1.1.0", - "@backstage/catalog-model": "^0.8.2", + "@backstage/catalog-model": "^0.8.3", "@backstage/config": "^0.1.5", "@backstage/plugin-catalog-backend": "^0.10.2", "@microsoft/microsoft-graph-types": "^1.25.0", @@ -40,7 +40,7 @@ "qs": "^6.9.4" }, "devDependencies": { - "@backstage/cli": "^0.7.0", + "@backstage/cli": "^0.7.1", "@backstage/test-utils": "^0.1.13", "@types/lodash": "^4.14.151", "msw": "^0.21.2" From 79ec37d1bafed9f20ffdf000a79171720ad3b3de Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Fri, 18 Jun 2021 09:48:26 +0200 Subject: [PATCH 15/17] Update api reports Signed-off-by: Oliver Sand --- .github/styles/vocab.txt | 1 + .../api-report.md | 121 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 plugins/catalog-backend-module-msgraph/api-report.md diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 4ff766e5e2..78b2a9b75f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -149,6 +149,7 @@ Mkdocs monorepo Monorepo monorepos +msgraph msw mysql namespace diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md new file mode 100644 index 0000000000..702601b469 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -0,0 +1,121 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-msgraph" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; +import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { GroupEntity } from '@backstage/catalog-model'; +import { LocationSpec } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import * as msal from '@azure/msal-node'; +import { UserEntity } from '@backstage/catalog-model'; + +// @public (undocumented) +export function defaultGroupTransformer(group: MicrosoftGraph.Group, groupPhoto?: string): Promise; + +// @public (undocumented) +export function defaultOrganizationTransformer(organization: MicrosoftGraph.Organization): Promise; + +// @public (undocumented) +export function defaultUserTransformer(user: MicrosoftGraph.User, userPhoto?: string): Promise; + +// @public (undocumented) +export type GroupTransformer = (group: MicrosoftGraph.Group, groupPhoto?: string) => Promise; + +// @public +export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION = "graph.microsoft.com/group-id"; + +// @public +export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION = "graph.microsoft.com/tenant-id"; + +// @public +export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = "graph.microsoft.com/user-id"; + +// @public (undocumented) +export class MicrosoftGraphClient { + constructor(baseUrl: string, pca: msal.ConfidentialClientApplication); + // (undocumented) + static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient; + // (undocumented) + getGroupMembers(groupId: string): AsyncIterable; + // (undocumented) + getGroupPhoto(groupId: string, sizeId?: string): Promise; + // (undocumented) + getGroupPhotoWithSizeLimit(groupId: string, maxSize: number): Promise; + // (undocumented) + getGroups(query?: ODataQuery): AsyncIterable; + // (undocumented) + getOrganization(tenantId: string): Promise; + // (undocumented) + getUserPhoto(userId: string, sizeId?: string): Promise; + // (undocumented) + getUserPhotoWithSizeLimit(userId: string, maxSize: number): Promise; + // (undocumented) + getUserProfile(userId: string): Promise; + // (undocumented) + getUsers(query?: ODataQuery): AsyncIterable; + // (undocumented) + requestApi(path: string, query?: ODataQuery): Promise; + // (undocumented) + requestCollection(path: string, query?: ODataQuery): AsyncIterable; + // (undocumented) + requestRaw(url: string): Promise; +} + +// @public +export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { + constructor(options: { + providers: MicrosoftGraphProviderConfig[]; + logger: Logger; + groupTransformer?: GroupTransformer; + }); + // (undocumented) + static fromConfig(config: Config, options: { + logger: Logger; + groupTransformer?: GroupTransformer; + }): MicrosoftGraphOrgReaderProcessor; + // (undocumented) + readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise; +} + +// @public +export type MicrosoftGraphProviderConfig = { + target: string; + authority?: string; + tenantId: string; + clientId: string; + clientSecret: string; + userFilter?: string; + groupFilter?: string; +}; + +// @public (undocumented) +export function normalizeEntityName(name: string): string; + +// @public (undocumented) +export type OrganizationTransformer = (organization: MicrosoftGraph.Organization) => Promise; + +// @public (undocumented) +export function readMicrosoftGraphConfig(config: Config): MicrosoftGraphProviderConfig[]; + +// @public (undocumented) +export function readMicrosoftGraphOrg(client: MicrosoftGraphClient, tenantId: string, options?: { + userFilter?: string; + groupFilter?: string; + groupTransformer?: GroupTransformer; +}): Promise<{ + users: UserEntity[]; + groups: GroupEntity[]; +}>; + +// @public (undocumented) +export type UserTransformer = (user: MicrosoftGraph.User, userPhoto?: string) => Promise; + + +// (No @packageDocumentation comment for this package) + +``` From 176f28b712eedcee2ba08cbfc8f21f28a23fdb9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Jun 2021 08:32:15 +0000 Subject: [PATCH 16/17] chore(deps): bump @google-cloud/container from 2.2.2 to 2.3.0 Bumps [@google-cloud/container](https://github.com/googleapis/nodejs-cloud-container) from 2.2.2 to 2.3.0. - [Release notes](https://github.com/googleapis/nodejs-cloud-container/releases) - [Changelog](https://github.com/googleapis/nodejs-cloud-container/blob/master/CHANGELOG.md) - [Commits](https://github.com/googleapis/nodejs-cloud-container/compare/v2.2.2...v2.3.0) --- updated-dependencies: - dependency-name: "@google-cloud/container" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 208ad9e952..012d1a1a9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1347,7 +1347,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.7.4": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1360,7 +1360,7 @@ yup "^0.29.3" "@backstage/catalog-model@^0.7.9": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1389,16 +1389,16 @@ react-use "^17.2.4" "@backstage/plugin-catalog@^0.5.1": - version "0.6.2" + version "0.6.3" dependencies: "@backstage/catalog-client" "^0.3.13" - "@backstage/catalog-model" "^0.8.2" - "@backstage/core" "^0.7.12" + "@backstage/catalog-model" "^0.8.3" + "@backstage/core" "^0.7.13" "@backstage/core-plugin-api" "^0.1.2" "@backstage/errors" "^0.1.1" "@backstage/integration" "^0.5.6" "@backstage/integration-react" "^0.1.3" - "@backstage/plugin-catalog-react" "^0.2.2" + "@backstage/plugin-catalog-react" "^0.2.3" "@backstage/theme" "^0.2.8" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -1890,9 +1890,9 @@ teeny-request "^7.0.0" "@google-cloud/container@^2.2.0": - version "2.2.2" - resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.2.tgz#2b02e2cd3a446cfde3189c6018fad00244767b5b" - integrity sha512-r5DBAqKZtbU+DF/WLjldQfIuXiVLBBZJ7lJ/rrg9z20CvhFmv+ATVLkadh8018I2Xggor8+0ePp2Q6xstyFwMA== + version "2.3.0" + resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.3.0.tgz#a23f046948dbaf8cced008d419580cb600334efc" + integrity sha512-Tv8fR7JjlZr3oh476hMsf9yqGXbb/+81n0Va1Uc3reWjAdUXCYztH3/o/HMvh6yvd06j8VLLUxyBwAIb5PtW5g== dependencies: google-gax "^2.12.0" From 93529a5f11c53066ddd96d48bfd7073ae955a25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 18 Jun 2021 10:35:47 +0200 Subject: [PATCH 17/17] Permit, and prefer, "The Backstage Authors" in the copyright header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .eslintrc.js | 6 ++++++ scripts/copyright-header.txt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.eslintrc.js b/.eslintrc.js index 4dfcf5317f..c8108f7289 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -25,6 +25,12 @@ module.exports = { { // eslint-disable-next-line no-restricted-syntax templateFile: path.resolve(__dirname, './scripts/copyright-header.txt'), + templateVars: { + NAME: 'The Backstage Authors', + }, + varRegexps: { + NAME: /(The Backstage Authors)|(Spotify AB)/, + }, onNonMatchingHeader: 'replace', }, ], diff --git a/scripts/copyright-header.txt b/scripts/copyright-header.txt index 4376d55847..478635997f 100644 --- a/scripts/copyright-header.txt +++ b/scripts/copyright-header.txt @@ -1,5 +1,5 @@ /* - * Copyright <%= YEAR %> Spotify AB + * Copyright <%= YEAR %> <%= NAME %> * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.