diff --git a/.changeset/metal-badgers-carry.md b/.changeset/metal-badgers-carry.md new file mode 100644 index 0000000000..b82e638c6f --- /dev/null +++ b/.changeset/metal-badgers-carry.md @@ -0,0 +1,25 @@ +--- +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Move `MicrosoftGraphOrgReaderProcessor` from `@backstage/plugin-catalog-backend` +to `@backstage/plugin-catalog-backend-module-msgraph`. + +The `MicrosoftGraphOrgReaderProcessor` isn't registered by default anymore, if +you want to continue using it you have to register it manually at the catalog +builder: + +1. Add dependency to `@backstage/plugin-catalog-backend-module-msgraph` to the `package.json` of your backend. +2. Add the processor to the catalog builder: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + }), +); +``` + +For more configuration details, see the [README of the `@backstage/plugin-catalog-backend-module-msgraph` package](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-msgraph/README.md). diff --git a/.changeset/perfect-gifts-compete.md b/.changeset/perfect-gifts-compete.md new file mode 100644 index 0000000000..7c03798888 --- /dev/null +++ b/.changeset/perfect-gifts-compete.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Don't export the `defaultGoogleAuthProvider` diff --git a/.changeset/silent-ways-laugh.md b/.changeset/silent-ways-laugh.md new file mode 100644 index 0000000000..ea152c68ef --- /dev/null +++ b/.changeset/silent-ways-laugh.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Allow customizations of `MicrosoftGraphOrgReaderProcessor` by passing an +optional `groupTransformer`, `userTransformer`, and `organizationTransformer`. diff --git a/.eslintrc.js b/.eslintrc.js index 4dfcf5317f..c8108f7289 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -25,6 +25,12 @@ module.exports = { { // eslint-disable-next-line no-restricted-syntax templateFile: path.resolve(__dirname, './scripts/copyright-header.txt'), + templateVars: { + NAME: 'The Backstage Authors', + }, + varRegexps: { + NAME: /(The Backstage Authors)|(Spotify AB)/, + }, onNonMatchingHeader: 'replace', }, ], diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 4ff766e5e2..78b2a9b75f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -149,6 +149,7 @@ Mkdocs monorepo Monorepo monorepos +msgraph msw mysql namespace diff --git a/docs/integrations/azure/locations.md b/docs/integrations/azure/locations.md index 163674f00a..3b3e3dd5ed 100644 --- a/docs/integrations/azure/locations.md +++ b/docs/integrations/azure/locations.md @@ -6,8 +6,8 @@ sidebar_label: Locations description: Integrating source code stored in Azure DevOps into the Backstage catalog --- -The Azure integration supports loading catalog entities from Azure DevOps. -Entities can be added to +The Azure DevOps integration supports loading catalog entities from Azure +DevOps. Entities can be added to [static catalog configuration](../../features/software-catalog/configuration.md), or registered with the [catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md new file mode 100644 index 0000000000..c359960f4d --- /dev/null +++ b/docs/integrations/azure/org.md @@ -0,0 +1,14 @@ +--- +id: org +title: Microsoft Azure Active Directory Organizational Data +sidebar_label: Org Data +# prettier-ignore +description: Importing users and groups from a Microsoft Azure Active Directory into Backstage +--- + +The Backstage catalog can be set up to ingest organizational data - users and +teams - directly from an tenant in Microsoft Azure Active Directory via the +Microsoft Graph API. + +More details on this are available in the +[README of the `@backstage/plugin-catalog-backend-module-msgraph` package](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-msgraph/README.md). diff --git a/microsite/sidebars.json b/microsite/sidebars.json index e0add7a007..ea1e1a4e89 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -102,8 +102,8 @@ "integrations/index", { "type": "subcategory", - "label": "Azure DevOps", - "ids": ["integrations/azure/locations"] + "label": "Azure", + "ids": ["integrations/azure/locations", "integrations/azure/org"] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 4b2b29a997..6c4af107ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,8 +75,9 @@ nav: - FAQ: 'features/techdocs/FAQ.md' - Integrations: - Overview: 'integrations/index.md' - - Azure DevOps: + - Azure: - Locations: 'integrations/azure/locations.md' + - Org Data: 'integrations/azure/org.md' - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' - Discovery: 'integrations/bitbucket/discovery.md' diff --git a/plugins/auth-backend/api-report.md b/plugins/auth-backend/api-report.md index dea5500d53..6ae878accf 100644 --- a/plugins/auth-backend/api-report.md +++ b/plugins/auth-backend/api-report.md @@ -6,12 +6,14 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { JSONWebKey } from 'jose'; import { Logger } from 'winston'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Profile } from 'passport'; +import { UserEntity } from '@backstage/catalog-model'; // @public (undocumented) export type AuthProviderFactory = (options: AuthProviderFactoryOptions) => AuthProviderRouteHandlers; @@ -47,8 +49,13 @@ export type AuthResponse = { export type BackstageIdentity = { id: string; idToken?: string; + token?: string; + entity?: Entity; }; +// @public (undocumented) +export const createGoogleProvider: (options?: GoogleProviderOptions | undefined) => AuthProviderFactory; + // @public (undocumented) export function createRouter({ logger, config, discovery, database, providerFactories, }: RouterOptions): Promise; @@ -63,6 +70,17 @@ export const encodeState: (state: OAuthState) => string; // @public (undocumented) export const ensuresXRequestedWith: (req: express.Request) => boolean; +// @public (undocumented) +export const googleEmailSignInResolver: SignInResolver; + +// @public (undocumented) +export type GoogleProviderOptions = { + authHandler?: AuthHandler; + signIn?: { + resolver?: SignInResolver; + }; +}; + // @public export class IdentityClient { constructor(options: { diff --git a/plugins/auth-backend/src/providers/google/index.ts b/plugins/auth-backend/src/providers/google/index.ts index 8bff8b250b..61805b2dcd 100644 --- a/plugins/auth-backend/src/providers/google/index.ts +++ b/plugins/auth-backend/src/providers/google/index.ts @@ -14,9 +14,5 @@ * limitations under the License. */ -export { - createGoogleProvider, - googleDefaultSignInResolver, - googleEmailSignInResolver, -} from './provider'; +export { createGoogleProvider, googleEmailSignInResolver } from './provider'; export type { GoogleProviderOptions } from './provider'; diff --git a/plugins/auth-backend/src/providers/google/provider.ts b/plugins/auth-backend/src/providers/google/provider.ts index ace0c8e132..b47399e1c6 100644 --- a/plugins/auth-backend/src/providers/google/provider.ts +++ b/plugins/auth-backend/src/providers/google/provider.ts @@ -198,7 +198,7 @@ export const googleEmailSignInResolver: SignInResolver = async ( return { id: entity.metadata.name, entity, token }; }; -export const googleDefaultSignInResolver: SignInResolver = async ( +const googleDefaultSignInResolver: SignInResolver = async ( info, ctx, ) => { diff --git a/plugins/catalog-backend-module-msgraph/.eslintrc.js b/plugins/catalog-backend-module-msgraph/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md new file mode 100644 index 0000000000..df141901aa --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -0,0 +1,116 @@ +# Catalog Backend Module for Microsoft Graph + +This is an extension module to the `plugin-catalog-backend` plugin, providing a +`MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data +from the Microsoft Graph API. This processor is useful if you want to import +users and groups from Azure Active Directory or Office 365. + +## Getting Started + +1. The processor is not installed by default, therefore you have to add a + dependency to `@backstage/plugin-catalog-backend-module-msgraph` to your + backend package. + +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-catalog-backend-module-msgraph +``` + +2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you + have to register it in the catalog plugin: + +```typescript +// packages/backend/src/plugins/catalog.ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + }), +); +``` + +3. Create or use an existing App registration in the [Microsoft Azure Portal](https://portal.azure.com/). + The App registration requires at least the API permissions `Group.Read.All`, + `GroupMember.Read.All`, `User.Read` and `User.Read.All` for Microsoft Graph + (if you still run into errors about insufficient privileges, add + `Team.ReadBasic.All` and `TeamMember.Read.All` too). + +4. Configure the processor: + +```yaml +# app-config.yaml +catalog: + processors: + microsoftGraphOrg: + providers: + - target: https://graph.microsoft.com/v1.0 + authority: https://login.microsoftonline.com + # If you don't know you tenantId, you can use Microsoft Graph Explorer + # to query it + tenantId: ${MICROSOFT_GRAPH_TENANT_ID} + # Client Id and Secret can be created under Certificates & secrets in + # the App registration in the Microsoft Azure Portal. + clientId: ${MICROSOFT_GRAPH_CLIENT_ID} + clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional filter for user, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties + # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter + userFilter: accountEnabled eq true and userType eq 'member' + # Optional filter for group, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties + groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') +``` + +5. Add a location that ingests from Microsoft Graph: + +```yaml +# app-config.yaml +catalog: + locations: + - type: microsoft-graph-org + target: https://graph.microsoft.com/v1.0 + # If you catalog doesn't allow to import Group and User entities by + # default, allow them here + rules: + - allow: [Group, User] + … +``` + +## Customize the Processor + +In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` +allows to pass transformers for users, groups and the organization. + +1. Create a transformer: + +```ts +export async function myGroupTransformer( + group: MicrosoftGraph.Group, + groupPhoto?: string, +): Promise { + if ( + ((group as unknown) as { + creationOptions: string[]; + }).creationOptions.includes('ProvisionGroupHomepage') + ) { + return undefined; + } + + // Transformations may change namespace, change entity naming pattern, fill + // profile with more or other details... + + // Create the group entity on your own, or wrap the default transformer + return await defaultGroupTransformer(group, groupPhoto); +} +``` + +2. Configure the processor with the transformer: + +```ts +builder.addProcessor( + MicrosoftGraphOrgReaderProcessor.fromConfig(config, { + logger, + groupTransformer: myGroupTransformer, + }), +); +``` diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md new file mode 100644 index 0000000000..702601b469 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -0,0 +1,121 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-msgraph" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts + +import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; +import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; +import { Config } from '@backstage/config'; +import { GroupEntity } from '@backstage/catalog-model'; +import { LocationSpec } from '@backstage/catalog-model'; +import { Logger } from 'winston'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import * as msal from '@azure/msal-node'; +import { UserEntity } from '@backstage/catalog-model'; + +// @public (undocumented) +export function defaultGroupTransformer(group: MicrosoftGraph.Group, groupPhoto?: string): Promise; + +// @public (undocumented) +export function defaultOrganizationTransformer(organization: MicrosoftGraph.Organization): Promise; + +// @public (undocumented) +export function defaultUserTransformer(user: MicrosoftGraph.User, userPhoto?: string): Promise; + +// @public (undocumented) +export type GroupTransformer = (group: MicrosoftGraph.Group, groupPhoto?: string) => Promise; + +// @public +export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION = "graph.microsoft.com/group-id"; + +// @public +export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION = "graph.microsoft.com/tenant-id"; + +// @public +export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = "graph.microsoft.com/user-id"; + +// @public (undocumented) +export class MicrosoftGraphClient { + constructor(baseUrl: string, pca: msal.ConfidentialClientApplication); + // (undocumented) + static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient; + // (undocumented) + getGroupMembers(groupId: string): AsyncIterable; + // (undocumented) + getGroupPhoto(groupId: string, sizeId?: string): Promise; + // (undocumented) + getGroupPhotoWithSizeLimit(groupId: string, maxSize: number): Promise; + // (undocumented) + getGroups(query?: ODataQuery): AsyncIterable; + // (undocumented) + getOrganization(tenantId: string): Promise; + // (undocumented) + getUserPhoto(userId: string, sizeId?: string): Promise; + // (undocumented) + getUserPhotoWithSizeLimit(userId: string, maxSize: number): Promise; + // (undocumented) + getUserProfile(userId: string): Promise; + // (undocumented) + getUsers(query?: ODataQuery): AsyncIterable; + // (undocumented) + requestApi(path: string, query?: ODataQuery): Promise; + // (undocumented) + requestCollection(path: string, query?: ODataQuery): AsyncIterable; + // (undocumented) + requestRaw(url: string): Promise; +} + +// @public +export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { + constructor(options: { + providers: MicrosoftGraphProviderConfig[]; + logger: Logger; + groupTransformer?: GroupTransformer; + }); + // (undocumented) + static fromConfig(config: Config, options: { + logger: Logger; + groupTransformer?: GroupTransformer; + }): MicrosoftGraphOrgReaderProcessor; + // (undocumented) + readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise; +} + +// @public +export type MicrosoftGraphProviderConfig = { + target: string; + authority?: string; + tenantId: string; + clientId: string; + clientSecret: string; + userFilter?: string; + groupFilter?: string; +}; + +// @public (undocumented) +export function normalizeEntityName(name: string): string; + +// @public (undocumented) +export type OrganizationTransformer = (organization: MicrosoftGraph.Organization) => Promise; + +// @public (undocumented) +export function readMicrosoftGraphConfig(config: Config): MicrosoftGraphProviderConfig[]; + +// @public (undocumented) +export function readMicrosoftGraphOrg(client: MicrosoftGraphClient, tenantId: string, options?: { + userFilter?: string; + groupFilter?: string; + groupTransformer?: GroupTransformer; +}): Promise<{ + users: UserEntity[]; + groups: GroupEntity[]; +}>; + +// @public (undocumented) +export type UserTransformer = (user: MicrosoftGraph.User, userPhoto?: string) => Promise; + + +// (No @packageDocumentation comment for this package) + +``` diff --git a/plugins/catalog-backend-module-msgraph/config.d.ts b/plugins/catalog-backend-module-msgraph/config.d.ts new file mode 100644 index 0000000000..d1b9630477 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/config.d.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface Config { + /** + * Configuration options for the catalog plugin. + */ + catalog?: { + /** + * List of processor-specific options and attributes + */ + processors?: { + /** + * MicrosoftGraphOrgReaderProcessor configuration + */ + microsoftGraphOrg?: { + /** + * The configuration parameters for each single Microsoft Graph provider. + */ + providers: Array<{ + /** + * The prefix of the target that this matches on, e.g. + * "https://graph.microsoft.com/v1.0", with no trailing slash. + */ + target: string; + /** + * The auth authority used. + * + * Default value "https://login.microsoftonline.com" + */ + authority?: string; + /** + * The tenant whose org data we are interested in. + */ + tenantId: string; + /** + * The OAuth client ID to use for authenticating requests. + */ + clientId: string; + /** + * The OAuth client secret to use for authenticating requests. + * + * @visibility secret + */ + clientSecret: string; + + // TODO: Consider not making these config options and pass them in the + // constructor instead. They are probably not environment specifc, so + // they could also be configured "in code". + + /** + * The filter to apply to extract users. + * + * E.g. "accountEnabled eq true and userType eq 'member'" + */ + userFilter?: string; + /** + * The filter to apply to extract groups. + * + * E.g. "securityEnabled eq false and mailEnabled eq true" + */ + groupFilter?: string; + }>; + }; + }; + }; +} diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json new file mode 100644 index 0000000000..83bb5dd802 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -0,0 +1,53 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-msgraph", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-msgraph" + }, + "keywords": [ + "backstage" + ], + "scripts": { + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@azure/msal-node": "^1.1.0", + "@backstage/catalog-model": "^0.8.3", + "@backstage/config": "^0.1.5", + "@backstage/plugin-catalog-backend": "^0.10.2", + "@microsoft/microsoft-graph-types": "^1.25.0", + "cross-fetch": "^3.0.6", + "lodash": "^4.17.15", + "p-limit": "^3.0.2", + "winston": "^3.2.1", + "qs": "^6.9.4" + }, + "devDependencies": { + "@backstage/cli": "^0.7.1", + "@backstage/test-utils": "^0.1.13", + "@types/lodash": "^4.14.151", + "msw": "^0.21.2" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-msgraph/src/index.ts b/plugins/catalog-backend-module-msgraph/src/index.ts new file mode 100644 index 0000000000..83c8c14e37 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './processors'; +export * from './microsoftGraph'; diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/constants.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/constants.ts diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.test.ts new file mode 100644 index 0000000000..48a07bbce8 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { normalizeEntityName } from './helper'; + +describe('normalizeEntityName', () => { + it('should normalize name to valid entity name', () => { + expect(normalizeEntityName('User Name')).toBe('user_name'); + }); + + it('should normalize e-mail to valid entity name', () => { + expect(normalizeEntityName('user.name@example.com')).toBe( + 'user.name_example.com', + ); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.ts new file mode 100644 index 0000000000..e7a804cbff --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/helper.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function normalizeEntityName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9_\-\.]/g, '_'); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts similarity index 76% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts index 882125fd84..89b1a35a79 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/index.ts @@ -14,11 +14,22 @@ * limitations under the License. */ export { MicrosoftGraphClient } from './client'; -export type { MicrosoftGraphProviderConfig } from './config'; export { readMicrosoftGraphConfig } from './config'; -export { readMicrosoftGraphOrg } from './read'; +export type { MicrosoftGraphProviderConfig } from './config'; export { MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, MICROSOFT_GRAPH_USER_ID_ANNOTATION, } from './constants'; +export { normalizeEntityName } from './helper'; +export { + defaultGroupTransformer, + defaultOrganizationTransformer, + defaultUserTransformer, + readMicrosoftGraphOrg, +} from './read'; +export type { + GroupTransformer, + OrganizationTransformer, + UserTransformer, +} from './types'; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.test.ts new file mode 100644 index 0000000000..e264847478 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { buildMemberOf, buildOrgHierarchy } from './org'; + +function g( + name: string, + parent: string | undefined, + children: string[], +): GroupEntity { + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { name }, + spec: { type: 'team', parent, children }, + }; +} + +describe('buildOrgHierarchy', () => { + it('adds groups to their parent.children', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const d = g('d', 'a', []); + buildOrgHierarchy([a, b, c, d]); + expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd'])); + expect(b.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(c.spec.children).toEqual([]); + expect(d.spec.children).toEqual([]); + }); + + it('sets parent of groups children', () => { + const a = g('a', undefined, ['b', 'd']); + const b = g('b', undefined, ['c']); + const c = g('c', undefined, []); + const d = g('d', undefined, []); + buildOrgHierarchy([a, b, c, d]); + expect(a.spec.parent).toBeUndefined(); + expect(b.spec.parent).toBe('a'); + expect(c.spec.parent).toBe('b'); + expect(d.spec.parent).toBe('a'); + }); +}); + +describe('buildMemberOf', () => { + it('fills indirect member of groups', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + const u: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'n' }, + spec: { profile: {}, memberOf: ['c'] }, + }; + + const groups = [a, b, c]; + buildOrgHierarchy(groups); + buildMemberOf(groups, [u]); + expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + }); + + it('takes group spec.members into account', () => { + const a = g('a', undefined, []); + const b = g('b', 'a', []); + const c = g('c', 'b', []); + c.spec.members = ['n']; + const u: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'n' }, + spec: { profile: {}, memberOf: [] }, + }; + + const groups = [a, b, c]; + buildOrgHierarchy(groups); + buildMemberOf(groups, [u]); + expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c'])); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.ts new file mode 100644 index 0000000000..69ccc9fb77 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/org.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; + +// TODO: Copied from plugin-catalog-backend, but we could also export them from +// there. Or move them to catalog-model. + +export function buildOrgHierarchy(groups: GroupEntity[]) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + + // + // Make sure that g.parent.children contain g + // + + for (const group of groups) { + const selfName = group.metadata.name; + const parentName = group.spec.parent; + if (parentName) { + const parent = groupsByName.get(parentName); + if (parent && !parent.spec.children.includes(selfName)) { + parent.spec.children.push(selfName); + } + } + } + + // + // Make sure that g.children.parent is g + // + + for (const group of groups) { + const selfName = group.metadata.name; + for (const childName of group.spec.children) { + const child = groupsByName.get(childName); + if (child && !child.spec.parent) { + child.spec.parent = selfName; + } + } + } +} + +// Ensure that users have their transitive group memberships. Requires that +// the groups were previously processed with buildOrgHierarchy() +export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) { + const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); + + users.forEach(user => { + const transitiveMemberOf = new Set(); + + const todo = [ + ...user.spec.memberOf, + ...groups + .filter(g => g.spec.members?.includes(user.metadata.name)) + .map(g => g.metadata.name), + ]; + + for (;;) { + const current = todo.pop(); + if (!current) { + break; + } + + if (!transitiveMemberOf.has(current)) { + transitiveMemberOf.add(current); + const group = groupsByName.get(current); + if (group?.spec.parent) { + todo.push(group.spec.parent); + } + } + } + + user.spec.memberOf = [...transitiveMemberOf]; + }); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts similarity index 85% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index 07d540c848..8a5716f277 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -16,17 +16,15 @@ import { GroupEntity, UserEntity } from '@backstage/catalog-model'; import merge from 'lodash/merge'; -import { RecursivePartial } from '../../../util'; import { GroupMember, MicrosoftGraphClient } from './client'; import { - normalizeEntityName, readMicrosoftGraphGroups, readMicrosoftGraphOrganization, readMicrosoftGraphUsers, resolveRelations, } from './read'; -function user(data: RecursivePartial): UserEntity { +function user(data: Partial): UserEntity { return merge( {}, { @@ -39,7 +37,7 @@ function user(data: RecursivePartial): UserEntity { ); } -function group(data: RecursivePartial): GroupEntity { +function group(data: Partial): GroupEntity { return merge( {}, { @@ -69,18 +67,6 @@ describe('read microsoft graph', () => { afterEach(() => jest.resetAllMocks()); - describe('normalizeEntityName', () => { - it('should normalize name to valid entity name', () => { - expect(normalizeEntityName('User Name')).toBe('user_name'); - }); - - it('should normalize e-mail to valid entity name', () => { - expect(normalizeEntityName('user.name@example.com')).toBe( - 'user.name_example.com', - ); - }); - }); - describe('readMicrosoftGraphUsers', () => { it('should read users', async () => { async function* getExampleUsers() { @@ -114,6 +100,7 @@ describe('read microsoft graph', () => { email: 'user.name@example.com', picture: 'data:image/jpeg;base64,...', }, + memberOf: [], }, }), ]); @@ -121,7 +108,6 @@ describe('read microsoft graph', () => { expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ filter: 'accountEnabled eq true', - select: ['id', 'displayName', 'mail'], }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); @@ -154,6 +140,7 @@ describe('read microsoft graph', () => { profile: { displayName: 'Organization Name', }, + children: [], }, }), ); @@ -161,6 +148,24 @@ describe('read microsoft graph', () => { expect(client.getOrganization).toBeCalledTimes(1); expect(client.getOrganization).toBeCalledWith('tenantid'); }); + + it('should read organization with custom transformer', async () => { + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + + const { rootGroup } = await readMicrosoftGraphOrganization( + client, + 'tenantid', + { transformer: async _ => undefined }, + ); + + expect(rootGroup).toEqual(undefined); + + expect(client.getOrganization).toBeCalledTimes(1); + expect(client.getOrganization).toBeCalledWith('tenantid'); + }); }); describe('readMicrosoftGraphGroups', () => { @@ -217,6 +222,7 @@ describe('read microsoft graph', () => { profile: { displayName: 'Organization Name', }, + children: [], }, }); expect(groups).toEqual([ @@ -234,10 +240,11 @@ describe('read microsoft graph', () => { profile: { displayName: 'Group Name', email: 'group@example.com', - // TODO: Loading groups doesn't work right now as Microsoft Graph - // doesn't allows this yet + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet /* picture: 'data:image/jpeg;base64,...',*/ }, + children: [], }, }), ]); @@ -249,11 +256,10 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledTimes(1); expect(client.getGroups).toBeCalledWith({ filter: 'securityEnabled eq false', - select: ['id', 'displayName', 'description', 'mail', 'mailNickname'], }); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); - // TODO: Loading groups doesn't work right now as Microsoft Graph + // TODO: Loading groups photos doesn't work right now as Microsoft Graph // doesn't allows this yet // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); @@ -271,6 +277,7 @@ describe('read microsoft graph', () => { }, spec: { type: 'root', + children: [], }, }); const groupA = group({ @@ -328,20 +335,26 @@ describe('read microsoft graph', () => { expect(rootGroup.spec.parent).toBeUndefined(); expect(rootGroup.spec.children).toEqual( - expect.arrayContaining(['a', 'b']), + expect.arrayContaining(['group:default/a', 'group:default/b']), ); - expect(groupA.spec.parent).toEqual('root'); + expect(groupA.spec.parent).toEqual('group:default/root'); expect(groupA.spec.children).toEqual(expect.arrayContaining([])); - expect(groupB.spec.parent).toEqual('root'); - expect(groupB.spec.children).toEqual(expect.arrayContaining(['c'])); + expect(groupB.spec.parent).toEqual('group:default/root'); + expect(groupB.spec.children).toEqual( + expect.arrayContaining(['group:default/c']), + ); - expect(groupC.spec.parent).toEqual('b'); + expect(groupC.spec.parent).toEqual('group:default/b'); expect(groupC.spec.children).toEqual(expect.arrayContaining([])); - expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a'])); - expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c'])); + expect(user1.spec.memberOf).toEqual( + expect.arrayContaining(['group:default/a']), + ); + expect(user2.spec.memberOf).toEqual( + expect.arrayContaining(['group:default/c']), + ); }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts similarity index 55% rename from plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts rename to plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 4409422645..b047a57836 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -13,95 +13,117 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { + GroupEntity, + stringifyEntityRef, + UserEntity, +} from '@backstage/catalog-model'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; import limiterFactory from 'p-limit'; -import { buildMemberOf, buildOrgHierarchy } from '../util/org'; import { MicrosoftGraphClient } from './client'; import { MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, MICROSOFT_GRAPH_USER_ID_ANNOTATION, } from './constants'; +import { normalizeEntityName } from './helper'; +import { buildMemberOf, buildOrgHierarchy } from './org'; +import { + GroupTransformer, + OrganizationTransformer, + UserTransformer, +} from './types'; -export function normalizeEntityName(name: string): string { - return name - .trim() - .toLocaleLowerCase() - .replace(/[^a-zA-Z0-9_\-\.]/g, '_'); +export async function defaultUserTransformer( + user: MicrosoftGraph.User, + userPhoto?: string, +): Promise { + if (!user.id || !user.displayName || !user.mail) { + return undefined; + } + + const name = normalizeEntityName(user.mail); + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name, + annotations: { + [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!, + }, + }, + spec: { + profile: { + displayName: user.displayName!, + email: user.mail!, + + // TODO: Additional fields? + // jobTitle: user.jobTitle || undefined, + // officeLocation: user.officeLocation || undefined, + // mobilePhone: user.mobilePhone || undefined, + }, + memberOf: [], + }, + }; + + if (userPhoto) { + entity.spec.profile!.picture = userPhoto; + } + + return entity; } export async function readMicrosoftGraphUsers( client: MicrosoftGraphClient, - options?: { userFilter?: string }, + options?: { userFilter?: string; transformer?: UserTransformer }, ): Promise<{ users: UserEntity[]; // With all relations empty }> { - const entities: UserEntity[] = []; - const promises: Promise[] = []; + const users: UserEntity[] = []; const limiter = limiterFactory(10); + const transformer = options?.transformer ?? defaultUserTransformer; + const promises: Promise[] = []; + for await (const user of client.getUsers({ filter: options?.userFilter, - select: ['id', 'displayName', 'mail'], })) { - if (!user.id || !user.displayName || !user.mail) { - continue; - } + // Process all users in parallel, otherwise it can take quite some time + promises.push( + limiter(async () => { + const userPhoto = await client.getUserPhotoWithSizeLimit( + user.id!, + // We are limiting the photo size, as users with full resolution photos + // can make the Backstage API slow + 120, + ); - const name = normalizeEntityName(user.mail); - const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { - name, - annotations: { - [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!, - }, - }, - spec: { - profile: { - displayName: user.displayName!, - email: user.mail!, + const entity = await transformer(user, userPhoto); - // TODO: Additional fields? - // jobTitle: user.jobTitle || undefined, - // officeLocation: user.officeLocation || undefined, - // mobilePhone: user.mobilePhone || undefined, - }, - memberOf: [], - }, - }; + if (!entity) { + return; + } - // Download the photos in parallel, otherwise it can take quite some time - const loadPhoto = limiter(async () => { - entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit( - user.id!, - // We are limiting the photo size, as users with full resolution photos - // can make the Backstage API slow - 120, - ); - }); - - promises.push(loadPhoto); - entities.push(entity); + users.push(entity); + }), + ); } - // Wait for all photos to be downloaded + // Wait for all users and photos to be downloaded await Promise.all(promises); - return { users: entities }; + return { users }; } -export async function readMicrosoftGraphOrganization( - client: MicrosoftGraphClient, - tenantId: string, -): Promise<{ - rootGroup: GroupEntity; // With all relations empty -}> { - // For now we expect a single root organization - const organization = await client.getOrganization(tenantId); +export async function defaultOrganizationTransformer( + organization: MicrosoftGraph.Organization, +): Promise { + if (!organization.id || !organization.displayName) { + return undefined; + } + const name = normalizeEntityName(organization.displayName!); - const rootGroup: GroupEntity = { + return { apiVersion: 'backstage.io/v1alpha1', kind: 'Group', metadata: { @@ -119,14 +141,68 @@ export async function readMicrosoftGraphOrganization( children: [], }, }; +} + +export async function readMicrosoftGraphOrganization( + client: MicrosoftGraphClient, + tenantId: string, + options?: { transformer?: OrganizationTransformer }, +): Promise<{ + rootGroup?: GroupEntity; // With all relations empty +}> { + // For now we expect a single root organization + const organization = await client.getOrganization(tenantId); + const transformer = options?.transformer ?? defaultOrganizationTransformer; + const rootGroup = await transformer(organization); return { rootGroup }; } +export async function defaultGroupTransformer( + group: MicrosoftGraph.Group, + groupPhoto?: string, +): Promise { + if (!group.id || !group.displayName) { + return undefined; + } + + const name = normalizeEntityName(group.mailNickname || group.displayName); + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: name, + annotations: { + [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id, + }, + }, + spec: { + type: 'team', + profile: {}, + children: [], + }, + }; + + if (group.description) { + entity.metadata.description = group.description; + } + if (group.displayName) { + entity.spec.profile!.displayName = group.displayName; + } + if (group.mail) { + entity.spec.profile!.email = group.mail; + } + if (groupPhoto) { + entity.spec.profile!.picture = groupPhoto; + } + + return entity; +} + export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, - options?: { groupFilter?: string }, + options?: { groupFilter?: string; transformer?: GroupTransformer }, ): Promise<{ groups: GroupEntity[]; // With all relations empty rootGroup: GroupEntity | undefined; // With all relations empty @@ -139,78 +215,52 @@ export async function readMicrosoftGraphGroups( const limiter = limiterFactory(10); const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId); - groupMember.set(rootGroup.metadata.name, new Set()); - groups.push(rootGroup); + if (rootGroup) { + groupMember.set(rootGroup.metadata.name, new Set()); + groups.push(rootGroup); + } + const transformer = options?.transformer ?? defaultGroupTransformer; const promises: Promise[] = []; for await (const group of client.getGroups({ filter: options?.groupFilter, - select: ['id', 'displayName', 'description', 'mail', 'mailNickname'], })) { - if (!group.id || !group.displayName) { - continue; - } + // Process all groups in parallel, otherwise it can take quite some time + promises.push( + limiter(async () => { + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo + /* const groupPhoto = await client.getGroupPhotoWithSizeLimit( + group.id!, + // We are limiting the photo size, as groups with full resolution photos + // can make the Backstage API slow + 120, + );*/ - const name = normalizeEntityName(group.mailNickname || group.displayName); - const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Group', - metadata: { - name: name, - annotations: { - [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id, - }, - }, - spec: { - type: 'team', - profile: {}, - children: [], - }, - }; + const entity = await transformer(group /* , groupPhoto*/); - if (group.description) { - entity.metadata.description = group.description; - } - if (group.displayName) { - entity.spec.profile!.displayName = group.displayName; - } - if (group.mail) { - entity.spec.profile!.email = group.mail; - } - - // Download the members in parallel, otherwise it can take quite some time - const loadGroupMembers = limiter(async () => { - for await (const member of client.getGroupMembers(group.id!)) { - if (!member.id) { - continue; + if (!entity) { + return; } - if (member['@odata.type'] === '#microsoft.graph.user') { - ensureItem(groupMemberOf, member.id, group.id!); + for await (const member of client.getGroupMembers(group.id!)) { + if (!member.id) { + continue; + } + + if (member['@odata.type'] === '#microsoft.graph.user') { + ensureItem(groupMemberOf, member.id, group.id!); + } + + if (member['@odata.type'] === '#microsoft.graph.group') { + ensureItem(groupMember, group.id!, member.id); + } } - if (member['@odata.type'] === '#microsoft.graph.group') { - ensureItem(groupMember, group.id!, member.id); - } - } - }); - - // TODO: Loading groups doesn't work right now as Microsoft Graph doesn't - // allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo - /*/ / Download the photos in parallel, otherwise it can take quite some time - const loadPhoto = limiter(async () => { - entity.spec.profile!.picture = await client.getGroupPhotoWithSizeLimit( - group.id!, - // We are limiting the photo size, as groups with full resolution photos - // can make the Backstage API slow - 120, - ); - }); - - promises.push(loadPhoto);*/ - promises.push(loadGroupMembers); - groups.push(entity); + groups.push(entity); + }), + ); } // Wait for all group members and photos to be loaded @@ -286,7 +336,7 @@ export function resolveRelations( retrieveItems(groupMember, id).forEach(m => { const childGroup = groupMap.get(m); if (childGroup) { - group.spec.children.push(childGroup.metadata.name); + group.spec.children.push(stringifyEntityRef(childGroup)); } }); @@ -294,7 +344,7 @@ export function resolveRelations( const parentGroup = groupMap.get(p); if (parentGroup) { // TODO: Only having a single parent group might not match every companies model, but fine for now. - group.spec.parent = parentGroup.metadata.name; + group.spec.parent = stringifyEntityRef(parentGroup); } }); }); @@ -309,7 +359,7 @@ export function resolveRelations( retrieveItems(groupMemberOf, id).forEach(p => { const parentGroup = groupMap.get(p); if (parentGroup) { - user.spec.memberOf.push(parentGroup.metadata.name); + user.spec.memberOf.push(stringifyEntityRef(parentGroup)); } }); }); @@ -321,7 +371,11 @@ export function resolveRelations( export async function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, - options?: { userFilter?: string; groupFilter?: string }, + options?: { + userFilter?: string; + groupFilter?: string; + groupTransformer?: GroupTransformer; + }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { const { users } = await readMicrosoftGraphUsers(client, { userFilter: options?.userFilter, @@ -333,6 +387,7 @@ export async function readMicrosoftGraphOrg( groupMemberOf, } = await readMicrosoftGraphGroups(client, tenantId, { groupFilter: options?.groupFilter, + transformer: options?.groupTransformer, }); resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts new file mode 100644 index 0000000000..55c28b8d7a --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/types.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; + +export type UserTransformer = ( + user: MicrosoftGraph.User, + userPhoto?: string, +) => Promise; + +export type OrganizationTransformer = ( + organization: MicrosoftGraph.Organization, +) => Promise; + +export type GroupTransformer = ( + group: MicrosoftGraph.Group, + groupPhoto?: string, +) => Promise; diff --git a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts similarity index 84% rename from plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts rename to plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 3456f2cafa..5d8b071663 100644 --- a/plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -16,24 +16,32 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; +import { + CatalogProcessor, + CatalogProcessorEmit, + results, +} from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { + GroupTransformer, MicrosoftGraphClient, MicrosoftGraphProviderConfig, readMicrosoftGraphConfig, readMicrosoftGraphOrg, -} from './microsoftGraph'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; +} from '../microsoftGraph'; /** - * Extracts teams and users out of an LDAP server. + * Extracts teams and users out of a the Microsoft Graph API. */ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly providers: MicrosoftGraphProviderConfig[]; private readonly logger: Logger; + private readonly groupTransformer?: GroupTransformer; - static fromConfig(config: Config, options: { logger: Logger }) { + static fromConfig( + config: Config, + options: { logger: Logger; groupTransformer?: GroupTransformer }, + ) { const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); return new MicrosoftGraphOrgReaderProcessor({ ...options, @@ -44,9 +52,11 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { constructor(options: { providers: MicrosoftGraphProviderConfig[]; logger: Logger; + groupTransformer?: GroupTransformer; }) { this.providers = options.providers; this.logger = options.logger; + this.groupTransformer = options.groupTransformer; } async readLocation( @@ -79,6 +89,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { { userFilter: provider.userFilter, groupFilter: provider.groupFilter, + groupTransformer: this.groupTransformer, }, ); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/index.ts b/plugins/catalog-backend-module-msgraph/src/processors/index.ts new file mode 100644 index 0000000000..46a0cce6f5 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/processors/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; diff --git a/plugins/catalog-backend-module-msgraph/src/setupTests.ts b/plugins/catalog-backend-module-msgraph/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/plugins/catalog-backend-module-msgraph/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 20fcd975cf..5417a84741 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -362,54 +362,6 @@ export interface Config { roleArn?: string; }; }; - - /** - * MicrosoftGraphOrgReaderProcessor configuration - */ - microsoftGraphOrg?: { - /** - * The configuration parameters for each single Microsoft Graph provider. - */ - providers: Array<{ - /** - * The prefix of the target that this matches on, e.g. - * "https://graph.microsoft.com/v1.0", with no trailing slash. - */ - target: string; - /** - * The auth authority used. - * - * Default value "https://login.microsoftonline.com" - */ - authority?: string; - /** - * The tenant whose org data we are interested in. - */ - tenantId: string; - /** - * The OAuth client ID to use for authenticating requests. - */ - clientId: string; - /** - * The OAuth client secret to use for authenticating requests. - * - * @visibility secret - */ - clientSecret: string; - /** - * The filter to apply to extract users. - * - * E.g. "accountEnabled eq true and userType eq 'member'" - */ - userFilter?: string; - /** - * The filter to apply to extract groups. - * - * E.g. "securityEnabled eq false and mailEnabled eq true" - */ - groupFilter?: string; - }>; - }; }; }; } diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 6bc53b43da..8e9f54504e 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -29,7 +29,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@azure/msal-node": "^1.0.0-beta.3", "@backstage/backend-common": "^0.8.3", "@backstage/catalog-client": "^0.3.13", "@backstage/catalog-model": "^0.8.3", @@ -38,7 +37,6 @@ "@backstage/integration": "^0.5.6", "@backstage/plugin-search-backend-node": "^0.2.1", "@backstage/search-common": "^0.1.2", - "@microsoft/microsoft-graph-types": "^1.25.0", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index a92cc9ed3b..8b87e18018 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -27,7 +27,6 @@ export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; -export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index 2551c16879..0bb5cd0aee 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,7 +50,6 @@ import { GithubDiscoveryProcessor, GithubOrgReaderProcessor, LdapOrgReaderProcessor, - MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -373,7 +372,6 @@ export class NextCatalogBuilder { GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), - MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9bec4442d7..1901d18f72 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -51,7 +51,6 @@ import { LdapOrgReaderProcessor, LocationEntityProcessor, LocationReaders, - MicrosoftGraphOrgReaderProcessor, PlaceholderProcessor, PlaceholderResolver, StaticLocationProcessor, @@ -319,7 +318,6 @@ export class CatalogBuilder { GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), LdapOrgReaderProcessor.fromConfig(config, { logger }), - MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }), diff --git a/plugins/explore/api-report.md b/plugins/explore/api-report.md index 590af53de8..4116247061 100644 --- a/plugins/explore/api-report.md +++ b/plugins/explore/api-report.md @@ -5,8 +5,10 @@ ```ts import { BackstagePlugin } from '@backstage/core'; +import { default } from 'react'; import { ExternalRouteRef } from '@backstage/core'; import { RouteRef } from '@backstage/core'; +import { TabProps } from '@material-ui/core'; // @public (undocumented) export const catalogEntityRouteRef: ExternalRouteRef<{ @@ -15,11 +17,22 @@ export const catalogEntityRouteRef: ExternalRouteRef<{ namespace: string; }, false>; +// @public (undocumented) +export const DomainExplorerContent: ({ title, }: { + title?: string | undefined; +}) => JSX.Element; + +// @public +export const ExploreLayout: { + ({ title, subtitle, children, }: ExploreLayoutProps): JSX.Element; + Route: (props: SubRoute) => null; +}; + // @public (undocumented) export const ExplorePage: () => JSX.Element; // @public (undocumented) -export const explorePlugin: BackstagePlugin<{ +const explorePlugin: BackstagePlugin<{ explore: RouteRef; }, { catalogEntity: ExternalRouteRef<{ @@ -29,9 +42,23 @@ export const explorePlugin: BackstagePlugin<{ }, false>; }>; +export { explorePlugin } + +export { explorePlugin as plugin } + // @public (undocumented) export const exploreRouteRef: RouteRef; +// @public (undocumented) +export const GroupsExplorerContent: ({ title, }: { + title?: string | undefined; +}) => JSX.Element; + +// @public (undocumented) +export const ToolExplorerContent: ({ title }: { + title?: string | undefined; +}) => JSX.Element; + // (No @packageDocumentation comment for this package) diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 801eabf368..69a62b4362 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -43,6 +43,9 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element; // @public (undocumented) export const Router: () => JSX.Element; +// @public (undocumented) +export type SyncResult = 'cached' | 'updated' | 'timeout'; + // @public (undocumented) export interface TechDocsApi { // (undocumented) @@ -109,7 +112,7 @@ export interface TechDocsStorageApi { // (undocumented) getStorageUrl(): Promise; // (undocumented) - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName): Promise; } // @public (undocumented) @@ -137,7 +140,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; - syncEntityDocs(entityId: EntityName): Promise; + syncEntityDocs(entityId: EntityName): Promise; } diff --git a/scripts/copyright-header.txt b/scripts/copyright-header.txt index 4376d55847..478635997f 100644 --- a/scripts/copyright-header.txt +++ b/scripts/copyright-header.txt @@ -1,5 +1,5 @@ /* - * Copyright <%= YEAR %> Spotify AB + * Copyright <%= YEAR %> <%= NAME %> * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/yarn.lock b/yarn.lock index 208ad9e952..edf42e6320 100644 --- a/yarn.lock +++ b/yarn.lock @@ -218,7 +218,14 @@ dependencies: debug "^4.1.1" -"@azure/msal-node@1.0.0-beta.3", "@azure/msal-node@^1.0.0-beta.3": +"@azure/msal-common@^4.3.0": + version "4.3.0" + resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-4.3.0.tgz#b540e92748656724088bf77192e59943a93135bc" + integrity sha512-jFqUWe83wVb6O8cNGGBFg2QlKvqM1ezUgJTEV7kIsAPX0RXhGFE4B1DLNt6hCnkTXDbw+KGW0zgxOEr4MJQwLw== + dependencies: + debug "^4.1.1" + +"@azure/msal-node@1.0.0-beta.3": version "1.0.0-beta.3" resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-beta.3.tgz#c84c7948028b39e48b901f5fac35bdedcbc8772e" integrity sha512-/KfYRfrsOIrZONvo/0Vi5umuqbPBtCWNtmRvkse64uI0C4CP/W4WXwRD42VMws/8LtKvr1I5rYlYgFzt5zDz/A== @@ -228,6 +235,16 @@ jsonwebtoken "^8.5.1" uuid "^8.3.0" +"@azure/msal-node@^1.1.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.1.0.tgz#e472cfadead169f8832066ae6c2d6b8eef4e89e4" + integrity sha512-gMO9aZdWOzufp1PcdD5ID25DdS9eInxgeCqx4Tk8PVU6Z7RxJQhoMzS64cJhGdpYgeIQwKljtF0CLCcPFxew/w== + dependencies: + "@azure/msal-common" "^4.3.0" + axios "^0.21.1" + jsonwebtoken "^8.5.1" + uuid "^8.3.0" + "@azure/storage-blob@^12.4.0": version "12.4.0" resolved "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.4.0.tgz#7127ddd9f413105e2c3688691bc4c6245d0806b3" @@ -1347,7 +1364,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.7.4": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1360,7 +1377,7 @@ yup "^0.29.3" "@backstage/catalog-model@^0.7.9": - version "0.8.2" + version "0.8.3" dependencies: "@backstage/config" "^0.1.5" "@backstage/errors" "^0.1.1" @@ -1389,16 +1406,16 @@ react-use "^17.2.4" "@backstage/plugin-catalog@^0.5.1": - version "0.6.2" + version "0.6.3" dependencies: "@backstage/catalog-client" "^0.3.13" - "@backstage/catalog-model" "^0.8.2" - "@backstage/core" "^0.7.12" + "@backstage/catalog-model" "^0.8.3" + "@backstage/core" "^0.7.13" "@backstage/core-plugin-api" "^0.1.2" "@backstage/errors" "^0.1.1" "@backstage/integration" "^0.5.6" "@backstage/integration-react" "^0.1.3" - "@backstage/plugin-catalog-react" "^0.2.2" + "@backstage/plugin-catalog-react" "^0.2.3" "@backstage/theme" "^0.2.8" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -1890,9 +1907,9 @@ teeny-request "^7.0.0" "@google-cloud/container@^2.2.0": - version "2.2.2" - resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.2.2.tgz#2b02e2cd3a446cfde3189c6018fad00244767b5b" - integrity sha512-r5DBAqKZtbU+DF/WLjldQfIuXiVLBBZJ7lJ/rrg9z20CvhFmv+ATVLkadh8018I2Xggor8+0ePp2Q6xstyFwMA== + version "2.3.0" + resolved "https://registry.npmjs.org/@google-cloud/container/-/container-2.3.0.tgz#a23f046948dbaf8cced008d419580cb600334efc" + integrity sha512-Tv8fR7JjlZr3oh476hMsf9yqGXbb/+81n0Va1Uc3reWjAdUXCYztH3/o/HMvh6yvd06j8VLLUxyBwAIb5PtW5g== dependencies: google-gax "^2.12.0"