Split MicrosoftGraphOrgReaderProcessor into a separate package and make it customizable

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>

plugins/catalog-backend-extension-msgraph/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/catalog-backend-extension-msgraph/README.md

@@ -0,0 +1,86 @@
+# Catalog Backend Extension for Microsoft Graph
+
+This is an extension to the `plugin-catalog-backend` plugin, providing a
+`MicrosoftGraphOrgReaderProcessor` that can be used to ingest organization data
+from the Microsoft Graph API. This processor is useful, if you want to import
+users and groups from Office 365.
+
+## Getting Started
+
+1. The processor is not installed by default, therefore you have to add a
+   dependency to `@backstage/plugin-catalog-backend-extension-msgraph` to your
+   backend package.
+
+```bash
+# From your Backstage root directory
+cd packages/backend
+yarn add @backstage/plugin-catalog-backend-extension-msgraph
+```
+
+2. The `MicrosoftGraphOrgReaderProcessor` is not registered by default, so you have to register it in the catalog plugin:
+
+```typescript
+// packages/backend/src/plugins/catalog.ts
+builder.addProcessor(
+  MicrosoftGraphOrgReaderProcessor.fromConfig(config, {
+    logger,
+  }),
+);
+```
+
+3. Configure the processor:
+
+```yaml
+# app-config.yaml
+catalog:
+  processors:
+    microsoftGraphOrg:
+      providers:
+        - target: https://graph.microsoft.com/v1.0
+          authority: https://login.microsoftonline.com
+          tenantId: ${MICROSOFT_GRAPH_TENANT_ID}
+          clientId: ${MICROSOFT_GRAPH_CLIENT_ID}
+          clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN}
+          # Optional filter for user, see Microsoft Graph API for the syntax
+          userFilter: accountEnabled eq true and userType eq 'member'
+          # Optional filter for group, see Microsoft Graph API for the syntax
+          groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
+```
+
+## Customize the Processor
+
+In case you want to customize the ingested entities, the `MicrosoftGraphOrgReaderProcessor` allows to pass transformers for users, groups and the organization.
+
+1. Create a transformer:
+
+```ts
+export async function myGroupTransformer(
+  group: MicrosoftGraph.Group,
+  groupPhoto?: string,
+): Promise<GroupEntity | undefined> {
+  if (
+    ((group as unknown) as {
+      creationOptions: string[];
+    }).creationOptions.includes('ProvisionGroupHomepage')
+  ) {
+    return undefined;
+  }
+
+  // Transformations may change namespace, change entity naming pattern, fill
+  // profile with more or other details...
+
+  // Create the group entity on your own, or wrap the default transformer
+  return await defaultGroupTransformer(group, groupPhoto);
+}
+```
+
+2. Configure the processor with the transformer:
+
+```ts
+builder.addProcessor(
+  MicrosoftGraphOrgReaderProcessor.fromConfig(config, {
+    logger,
+    groupTransformer: myGroupTransformer,
+  }),
+);
+```

plugins/catalog-backend-extension-msgraph/config.d.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface Config {
+  /**
+   * Configuration options for the catalog plugin.
+   */
+  catalog?: {
+    /**
+     * List of processor-specific options and attributes
+     */
+    processors?: {
+      /**
+       * MicrosoftGraphOrgReaderProcessor configuration
+       */
+      microsoftGraphOrg?: {
+        /**
+         * The configuration parameters for each single Microsoft Graph provider.
+         */
+        providers: Array<{
+          /**
+           * The prefix of the target that this matches on, e.g.
+           * "https://graph.microsoft.com/v1.0", with no trailing slash.
+           */
+          target: string;
+          /**
+           * The auth authority used.
+           *
+           * Default value "https://login.microsoftonline.com"
+           */
+          authority?: string;
+          /**
+           * The tenant whose org data we are interested in.
+           */
+          tenantId: string;
+          /**
+           * The OAuth client ID to use for authenticating requests.
+           */
+          clientId: string;
+          /**
+           * The OAuth client secret to use for authenticating requests.
+           *
+           * @visibility secret
+           */
+          clientSecret: string;
+
+          // TODO: Consider not making these config options and pass them in the
+          // constructor instead. They are probably not environment specifc, so
+          // they could also be configured "in code".
+
+          /**
+           * The filter to apply to extract users.
+           *
+           * E.g. "accountEnabled eq true and userType eq 'member'"
+           */
+          userFilter?: string;
+          /**
+           * The filter to apply to extract groups.
+           *
+           * E.g. "securityEnabled eq false and mailEnabled eq true"
+           */
+          groupFilter?: string;
+        }>;
+      };
+    };
+  };
+}

plugins/catalog-backend-extension-msgraph/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@backstage/plugin-catalog-backend-extension-msgraph",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-backend-extension-msgraph"
+  },
+  "keywords": [
+    "backstage"
+  ],
+  "scripts": {
+    "build": "backstage-cli backend:build",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@azure/msal-node": "^1.1.0",
+    "@backstage/catalog-model": "^0.8.2",
+    "@backstage/config": "^0.1.5",
+    "@backstage/plugin-catalog-backend": "^0.10.2",
+    "@microsoft/microsoft-graph-types": "^1.25.0",
+    "cross-fetch": "^3.0.6",
+    "lodash": "^4.17.15",
+    "p-limit": "^3.0.2",
+    "winston": "^3.2.1",
+    "qs": "^6.9.4"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.7.0",
+    "@backstage/test-utils": "^0.1.13",
+    "@types/lodash": "^4.14.151",
+    "msw": "^0.21.2"
+  },
+  "files": [
+    "dist",
+    "config.d.ts"
+  ],
+  "configSchema": "config.d.ts"
+}

plugins/catalog-backend-extension-msgraph/src/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './processors';
+export * from './microsoftGraph';

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.test.ts

@@ -0,0 +1,363 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import { msw } from '@backstage/test-utils';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { MicrosoftGraphClient } from './client';
+
+describe('MicrosoftGraphClient', () => {
+  const confidentialClientApplication: jest.Mocked<msal.ConfidentialClientApplication> = {
+    acquireTokenByClientCredential: jest.fn(),
+  } as any;
+  let client: MicrosoftGraphClient;
+  const worker = setupServer();
+
+  msw.setupDefaultHandlers(worker);
+
+  beforeEach(() => {
+    confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue(
+      { token: 'ACCESS_TOKEN' } as any,
+    );
+    client = new MicrosoftGraphClient(
+      'https://example.com',
+      confidentialClientApplication,
+    );
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  it('should perform raw request', async () => {
+    worker.use(
+      rest.get('https://other.example.com/', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestRaw('https://other.example.com/');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledTimes(1);
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] });
+  });
+
+  it('should perform simple api request', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestApi('users');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+  });
+
+  it('should perform api request with filter, select and expand', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (req, res, ctx) =>
+        res(ctx.status(200), ctx.json({ queryString: req.url.search })),
+      ),
+    );
+
+    const response = await client.requestApi('users', {
+      filter: 'test eq true',
+      expand: ['children'],
+      select: ['id', 'children'],
+    });
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      queryString:
+        '?$filter=test%20eq%20true&$select=id,children&$expand=children',
+    });
+  });
+
+  it('should perform collection request for a single page', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first']);
+  });
+
+  it('should perform collection request for multiple pages', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+            '@odata.nextLink': 'https://example.com/users2',
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get('https://example.com/users2', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: ['second'] })),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first', 'second']);
+  });
+
+  it('should load user profile', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            surname: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const userProfile = await client.getUserProfile('user-id');
+
+    expect(userProfile).toEqual({ surname: 'Example' });
+  });
+
+  it('should throw expection if load user profile fails', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    await expect(() => client.getUserProfile('user-id')).rejects.toThrowError();
+  });
+
+  it('should load user profile photo with max size of 120', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              {
+                height: 120,
+                id: 120,
+              },
+              {
+                height: 500,
+                id: 500,
+              },
+            ],
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should not fail if user has no profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toBeFalsy();
+  });
+
+  it('should load user profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) =>
+        res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load user profile photo for size 120', async () => {
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id', '120');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load users', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ surname: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getUsers());
+
+    expect(values).toEqual([{ surname: 'Example' }]);
+  });
+
+  it('should load group profile photo with max size of 120', async () => {
+    worker.use(
+      rest.get('https://example.com/groups/group-id/photos', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              {
+                height: 120,
+                id: 120,
+              },
+            ],
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get(
+        'https://example.com/groups/group-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getGroupPhotoWithSizeLimit('group-id', 120);
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load group profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/groups/group-id/photo/*', (_, res, ctx) =>
+        res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getGroupPhoto('group-id');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load groups', async () => {
+    worker.use(
+      rest.get('https://example.com/groups', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ displayName: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getGroups());
+
+    expect(values).toEqual([{ displayName: 'Example' }]);
+  });
+
+  it('should load group members', async () => {
+    worker.use(
+      rest.get('https://example.com/groups/group-id/members', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              { '@odata.type': '#microsoft.graph.user' },
+              { '@odata.type': '#microsoft.graph.group' },
+            ],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.getGroupMembers('group-id'),
+    );
+
+    expect(values).toEqual([
+      { '@odata.type': '#microsoft.graph.user' },
+      { '@odata.type': '#microsoft.graph.group' },
+    ]);
+  });
+
+  it('should load organization', async () => {
+    worker.use(
+      rest.get('https://example.com/organization/tentant-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            displayName: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const organization = await client.getOrganization('tentant-id');
+
+    expect(organization).toEqual({ displayName: 'Example' });
+  });
+});
+
+async function collectAsyncIterable<T>(
+  iterable: AsyncIterable<T>,
+): Promise<T[]> {
+  const values = [];
+  for await (const value of iterable) {
+    values.push(value);
+  }
+  return values;
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/client.ts

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
+import fetch from 'cross-fetch';
+import qs from 'qs';
+import { MicrosoftGraphProviderConfig } from './config';
+
+export type ODataQuery = {
+  filter?: string;
+  expand?: string[];
+  select?: string[];
+};
+
+export type GroupMember =
+  | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' })
+  | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' });
+
+export class MicrosoftGraphClient {
+  static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient {
+    const clientConfig: msal.Configuration = {
+      auth: {
+        clientId: config.clientId,
+        clientSecret: config.clientSecret,
+        authority: `${config.authority}/${config.tenantId}`,
+      },
+    };
+    const pca = new msal.ConfidentialClientApplication(clientConfig);
+    return new MicrosoftGraphClient(config.target, pca);
+  }
+
+  constructor(
+    private readonly baseUrl: string,
+    private readonly pca: msal.ConfidentialClientApplication,
+  ) {}
+
+  async *requestCollection<T>(
+    path: string,
+    query?: ODataQuery,
+  ): AsyncIterable<T> {
+    let response = await this.requestApi(path, query);
+
+    for (;;) {
+      if (response.status !== 200) {
+        await this.handleError(path, response);
+      }
+
+      const result = await response.json();
+      const elements: T[] = result.value;
+
+      yield* elements;
+
+      // Follow cursor to the next page if one is available
+      if (!result['@odata.nextLink']) {
+        return;
+      }
+
+      response = await this.requestRaw(result['@odata.nextLink']);
+    }
+  }
+
+  async requestApi(path: string, query?: ODataQuery): Promise<Response> {
+    const queryString = qs.stringify(
+      {
+        $filter: query?.filter,
+        $select: query?.select?.join(','),
+        $expand: query?.expand?.join(','),
+      },
+      {
+        addQueryPrefix: true,
+        // Microsoft Graph doesn't like an encoded query string
+        encode: false,
+      },
+    );
+
+    return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`);
+  }
+
+  async requestRaw(url: string): Promise<Response> {
+    // Make sure that we always have a valid access token (might be cached)
+    const token = await this.pca.acquireTokenByClientCredential({
+      scopes: ['https://graph.microsoft.com/.default'],
+    });
+
+    if (!token) {
+      throw new Error('Error while requesting token for Microsoft Graph');
+    }
+
+    return await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${token.accessToken}`,
+      },
+    });
+  }
+
+  async getUserProfile(userId: string): Promise<MicrosoftGraph.User> {
+    const response = await this.requestApi(`users/${userId}`);
+
+    if (response.status !== 200) {
+      await this.handleError('user profile', response);
+    }
+
+    return await response.json();
+  }
+
+  async getUserPhotoWithSizeLimit(
+    userId: string,
+    maxSize: number,
+  ): Promise<string | undefined> {
+    return await this.getPhotoWithSizeLimit('users', userId, maxSize);
+  }
+
+  async getUserPhoto(
+    userId: string,
+    sizeId?: string,
+  ): Promise<string | undefined> {
+    return await this.getPhoto('users', userId, sizeId);
+  }
+
+  async *getUsers(query?: ODataQuery): AsyncIterable<MicrosoftGraph.User> {
+    yield* this.requestCollection<MicrosoftGraph.User>(`users`, query);
+  }
+
+  async getGroupPhotoWithSizeLimit(
+    groupId: string,
+    maxSize: number,
+  ): Promise<string | undefined> {
+    return await this.getPhotoWithSizeLimit('groups', groupId, maxSize);
+  }
+
+  async getGroupPhoto(
+    groupId: string,
+    sizeId?: string,
+  ): Promise<string | undefined> {
+    return await this.getPhoto('groups', groupId, sizeId);
+  }
+
+  async *getGroups(query?: ODataQuery): AsyncIterable<MicrosoftGraph.Group> {
+    yield* this.requestCollection<MicrosoftGraph.Group>(`groups`, query);
+  }
+
+  async *getGroupMembers(groupId: string): AsyncIterable<GroupMember> {
+    yield* this.requestCollection<GroupMember>(`groups/${groupId}/members`);
+  }
+
+  async getOrganization(
+    tenantId: string,
+  ): Promise<MicrosoftGraph.Organization> {
+    const response = await this.requestApi(`organization/${tenantId}`);
+
+    if (response.status !== 200) {
+      await this.handleError(`organization/${tenantId}`, response);
+    }
+
+    return await response.json();
+  }
+
+  private async getPhotoWithSizeLimit(
+    entityName: string,
+    id: string,
+    maxSize: number,
+  ): Promise<string | undefined> {
+    const response = await this.requestApi(`${entityName}/${id}/photos`);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError(`${entityName} photos`, response);
+    }
+
+    const result = await response.json();
+    const photos = result.value as MicrosoftGraph.ProfilePhoto[];
+    let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined;
+
+    // Find the biggest picture that is smaller than the max size
+    for (const p of photos) {
+      if (
+        !selectedPhoto ||
+        (p.height! >= selectedPhoto.height! && p.height! <= maxSize)
+      ) {
+        selectedPhoto = p;
+      }
+    }
+
+    if (!selectedPhoto) {
+      return undefined;
+    }
+
+    return await this.getPhoto(entityName, id, selectedPhoto.id!);
+  }
+
+  private async getPhoto(
+    entityName: string,
+    id: string,
+    sizeId?: string,
+  ): Promise<string | undefined> {
+    const path = sizeId
+      ? `${entityName}/${id}/photos/${sizeId}/$value`
+      : `${entityName}/${id}/photo/$value`;
+    const response = await this.requestApi(path);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError('photo', response);
+    }
+
+    return `data:image/jpeg;base64,${Buffer.from(
+      await response.arrayBuffer(),
+    ).toString('base64')}`;
+  }
+
+  private async handleError(path: string, response: Response): Promise<void> {
+    const result = await response.json();
+    const error = result.error as MicrosoftGraph.PublicError;
+
+    throw new Error(
+      `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`,
+    );
+  }
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.test.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readMicrosoftGraphConfig } from './config';
+
+describe('readMicrosoftGraphConfig', () => {
+  it('applies all of the defaults', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(new ConfigReader(config));
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.microsoftonline.com',
+        userFilter: undefined,
+        groupFilter: undefined,
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+
+  it('reads all the values', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+          authority: 'https://login.example.com/',
+          userFilter: 'accountEnabled eq true',
+          groupFilter: 'securityEnabled eq false',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(new ConfigReader(config));
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.example.com',
+        userFilter: 'accountEnabled eq true',
+        groupFilter: 'securityEnabled eq false',
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+});

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/config.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+/**
+ * The configuration parameters for a single Microsoft Graph provider.
+ */
+export type MicrosoftGraphProviderConfig = {
+  /**
+   * The prefix of the target that this matches on, e.g.
+   * "https://graph.microsoft.com/v1.0", with no trailing slash.
+   */
+  target: string;
+  /**
+   * The auth authority used.
+   *
+   * E.g. "https://login.microsoftonline.com"
+   */
+  authority?: string;
+  /**
+   * The tenant whose org data we are interested in.
+   */
+  tenantId: string;
+  /**
+   * The OAuth client ID to use for authenticating requests.
+   */
+  clientId: string;
+  /**
+   * The OAuth client secret to use for authenticating requests.
+   *
+   * @visibility secret
+   */
+  clientSecret: string;
+  /**
+   * The filter to apply to extract users.
+   *
+   * E.g. "accountEnabled eq true and userType eq 'member'"
+   */
+  userFilter?: string;
+  /**
+   * The filter to apply to extract groups.
+   *
+   * E.g. "securityEnabled eq false and mailEnabled eq true"
+   */
+  groupFilter?: string;
+};
+
+export function readMicrosoftGraphConfig(
+  config: Config,
+): MicrosoftGraphProviderConfig[] {
+  const providers: MicrosoftGraphProviderConfig[] = [];
+  const providerConfigs = config.getOptionalConfigArray('providers') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    const authority =
+      providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') ||
+      'https://login.microsoftonline.com';
+    const tenantId = providerConfig.getString('tenantId');
+    const clientId = providerConfig.getString('clientId');
+    const clientSecret = providerConfig.getString('clientSecret');
+    const userFilter = providerConfig.getOptionalString('userFilter');
+    const groupFilter = providerConfig.getOptionalString('groupFilter');
+
+    providers.push({
+      target,
+      authority,
+      tenantId,
+      clientId,
+      clientSecret,
+      userFilter,
+      groupFilter,
+    });
+  }
+
+  return providers;
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/constants.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The tenant id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION =
+  'graph.microsoft.com/tenant-id';
+
+/**
+ * The group id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION =
+  'graph.microsoft.com/group-id';
+
+/**
+ * The user id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id';

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.test.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { normalizeEntityName } from './helper';
+
+describe('normalizeEntityName', () => {
+  it('should normalize name to valid entity name', () => {
+    expect(normalizeEntityName('User Name')).toBe('user_name');
+  });
+
+  it('should normalize e-mail to valid entity name', () => {
+    expect(normalizeEntityName('user.name@example.com')).toBe(
+      'user.name_example.com',
+    );
+  });
+});

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/helper.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function normalizeEntityName(name: string): string {
+  return name
+    .trim()
+    .toLocaleLowerCase()
+    .replace(/[^a-zA-Z0-9_\-\.]/g, '_');
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { MicrosoftGraphClient } from './client';
+export { readMicrosoftGraphConfig } from './config';
+export type { MicrosoftGraphProviderConfig } from './config';
+export {
+  MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
+  MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+} from './constants';
+export { normalizeEntityName } from './helper';
+export {
+  defaultGroupTransformer,
+  defaultOrganizationTransformer,
+  defaultUserTransformer,
+  readMicrosoftGraphOrg,
+} from './read';
+export type {
+  GroupTransformer,
+  OrganizationTransformer,
+  UserTransformer,
+} from './types';

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.test.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildMemberOf, buildOrgHierarchy } from './org';
+
+function g(
+  name: string,
+  parent: string | undefined,
+  children: string[],
+): GroupEntity {
+  return {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: { name },
+    spec: { type: 'team', parent, children },
+  };
+}
+
+describe('buildOrgHierarchy', () => {
+  it('adds groups to their parent.children', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const d = g('d', 'a', []);
+    buildOrgHierarchy([a, b, c, d]);
+    expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd']));
+    expect(b.spec.children).toEqual(expect.arrayContaining(['c']));
+    expect(c.spec.children).toEqual([]);
+    expect(d.spec.children).toEqual([]);
+  });
+
+  it('sets parent of groups children', () => {
+    const a = g('a', undefined, ['b', 'd']);
+    const b = g('b', undefined, ['c']);
+    const c = g('c', undefined, []);
+    const d = g('d', undefined, []);
+    buildOrgHierarchy([a, b, c, d]);
+    expect(a.spec.parent).toBeUndefined();
+    expect(b.spec.parent).toBe('a');
+    expect(c.spec.parent).toBe('b');
+    expect(d.spec.parent).toBe('a');
+  });
+});
+
+describe('buildMemberOf', () => {
+  it('fills indirect member of groups', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const u: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name: 'n' },
+      spec: { profile: {}, memberOf: ['c'] },
+    };
+
+    const groups = [a, b, c];
+    buildOrgHierarchy(groups);
+    buildMemberOf(groups, [u]);
+    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+  });
+
+  it('takes group spec.members into account', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    c.spec.members = ['n'];
+    const u: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name: 'n' },
+      spec: { profile: {}, memberOf: [] },
+    };
+
+    const groups = [a, b, c];
+    buildOrgHierarchy(groups);
+    buildMemberOf(groups, [u]);
+    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+  });
+});

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/org.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+
+// TODO: Copied from plugin-catalog-backend, but we could also export them from
+// there. Or move them to catalog-model.
+
+export function buildOrgHierarchy(groups: GroupEntity[]) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+
+  //
+  // Make sure that g.parent.children contain g
+  //
+
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    const parentName = group.spec.parent;
+    if (parentName) {
+      const parent = groupsByName.get(parentName);
+      if (parent && !parent.spec.children.includes(selfName)) {
+        parent.spec.children.push(selfName);
+      }
+    }
+  }
+
+  //
+  // Make sure that g.children.parent is g
+  //
+
+  for (const group of groups) {
+    const selfName = group.metadata.name;
+    for (const childName of group.spec.children) {
+      const child = groupsByName.get(childName);
+      if (child && !child.spec.parent) {
+        child.spec.parent = selfName;
+      }
+    }
+  }
+}
+
+// Ensure that users have their transitive group memberships. Requires that
+// the groups were previously processed with buildOrgHierarchy()
+export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+
+  users.forEach(user => {
+    const transitiveMemberOf = new Set<string>();
+
+    const todo = [
+      ...user.spec.memberOf,
+      ...groups
+        .filter(g => g.spec.members?.includes(user.metadata.name))
+        .map(g => g.metadata.name),
+    ];
+
+    for (;;) {
+      const current = todo.pop();
+      if (!current) {
+        break;
+      }
+
+      if (!transitiveMemberOf.has(current)) {
+        transitiveMemberOf.add(current);
+        const group = groupsByName.get(current);
+        if (group?.spec.parent) {
+          todo.push(group.spec.parent);
+        }
+      }
+    }
+
+    user.spec.memberOf = [...transitiveMemberOf];
+  });
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.test.ts

@@ -0,0 +1,354 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import merge from 'lodash/merge';
+import { GroupMember, MicrosoftGraphClient } from './client';
+import {
+  readMicrosoftGraphGroups,
+  readMicrosoftGraphOrganization,
+  readMicrosoftGraphUsers,
+  resolveRelations,
+} from './read';
+
+function user(data: Partial<UserEntity>): UserEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name: 'name' },
+      spec: { profile: {}, memberOf: [] },
+    } as UserEntity,
+    data,
+  );
+}
+
+function group(data: Partial<GroupEntity>): GroupEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: 'name',
+      },
+      spec: {
+        children: [],
+        type: 'team',
+      },
+    } as GroupEntity,
+    data,
+  );
+}
+
+describe('read microsoft graph', () => {
+  const client: jest.Mocked<MicrosoftGraphClient> = {
+    getUsers: jest.fn(),
+    getGroups: jest.fn(),
+    getGroupMembers: jest.fn(),
+    getUserPhotoWithSizeLimit: jest.fn(),
+    getGroupPhotoWithSizeLimit: jest.fn(),
+    getOrganization: jest.fn(),
+  } as any;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('readMicrosoftGraphUsers', () => {
+    it('should read users', async () => {
+      async function* getExampleUsers() {
+        yield {
+          id: 'userid',
+          displayName: 'User Name',
+          mail: 'user.name@example.com',
+        };
+      }
+
+      client.getUsers.mockImplementation(getExampleUsers);
+      client.getUserPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { users } = await readMicrosoftGraphUsers(client, {
+        userFilter: 'accountEnabled eq true',
+      });
+
+      expect(users).toEqual([
+        user({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/user-id': 'userid',
+            },
+            name: 'user.name_example.com',
+          },
+          spec: {
+            profile: {
+              displayName: 'User Name',
+              email: 'user.name@example.com',
+              picture: 'data:image/jpeg;base64,...',
+            },
+            memberOf: [],
+          },
+        }),
+      ]);
+
+      expect(client.getUsers).toBeCalledTimes(1);
+      expect(client.getUsers).toBeCalledWith({
+        filter: 'accountEnabled eq true',
+      });
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
+    });
+  });
+
+  describe('readMicrosoftGraphOrganization', () => {
+    it('should read organization', async () => {
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const { rootGroup } = await readMicrosoftGraphOrganization(
+        client,
+        'tenantid',
+      );
+
+      expect(rootGroup).toEqual(
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/tenant-id': 'tenantid',
+            },
+            name: 'organization_name',
+            description: 'Organization Name',
+          },
+          spec: {
+            type: 'root',
+            profile: {
+              displayName: 'Organization Name',
+            },
+            children: [],
+          },
+        }),
+      );
+
+      expect(client.getOrganization).toBeCalledTimes(1);
+      expect(client.getOrganization).toBeCalledWith('tenantid');
+    });
+
+    it('should read organization with custom transformer', async () => {
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const { rootGroup } = await readMicrosoftGraphOrganization(
+        client,
+        'tenantid',
+        { transformer: async _ => undefined },
+      );
+
+      expect(rootGroup).toEqual(undefined);
+
+      expect(client.getOrganization).toBeCalledTimes(1);
+      expect(client.getOrganization).toBeCalledWith('tenantid');
+    });
+  });
+
+  describe('readMicrosoftGraphGroups', () => {
+    it('should read groups', async () => {
+      async function* getExampleGroups() {
+        yield {
+          id: 'groupid',
+          displayName: 'Group Name',
+          description: 'Group Description',
+          mail: 'group@example.com',
+        };
+      }
+
+      async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.group',
+          id: 'childgroupid',
+        };
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementation(getExampleGroupMembers);
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+      client.getGroupPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const {
+        groups,
+        groupMember,
+        groupMemberOf,
+        rootGroup,
+      } = await readMicrosoftGraphGroups(client, 'tenantid', {
+        groupFilter: 'securityEnabled eq false',
+      });
+
+      const expectedRootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenantid',
+          },
+          name: 'organization_name',
+          description: 'Organization Name',
+        },
+        spec: {
+          type: 'root',
+          profile: {
+            displayName: 'Organization Name',
+          },
+          children: [],
+        },
+      });
+      expect(groups).toEqual([
+        expectedRootGroup,
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/group-id': 'groupid',
+            },
+            name: 'group_name',
+            description: 'Group Description',
+          },
+          spec: {
+            type: 'team',
+            profile: {
+              displayName: 'Group Name',
+              email: 'group@example.com',
+              // TODO: Loading groups photos doesn't work right now as Microsoft
+              // Graph doesn't allows this yet
+              /* picture: 'data:image/jpeg;base64,...',*/
+            },
+            children: [],
+          },
+        }),
+      ]);
+      expect(rootGroup).toEqual(expectedRootGroup);
+      expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid']));
+      expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid']));
+      expect(groupMember.get('organization_name')).toEqual(new Set());
+
+      expect(client.getGroups).toBeCalledTimes(1);
+      expect(client.getGroups).toBeCalledWith({
+        filter: 'securityEnabled eq false',
+      });
+      expect(client.getGroupMembers).toBeCalledTimes(1);
+      expect(client.getGroupMembers).toBeCalledWith('groupid');
+      // TODO: Loading groups photos doesn't work right now as Microsoft Graph
+      // doesn't allows this yet
+      // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1);
+      // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120);
+    });
+  });
+
+  describe('resolveRelations', () => {
+    it('should resolve relations', async () => {
+      const rootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenant-id-root',
+          },
+          name: 'root',
+        },
+        spec: {
+          type: 'root',
+          children: [],
+        },
+      });
+      const groupA = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-a',
+          },
+          name: 'a',
+        },
+      });
+      const groupB = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-b',
+          },
+          name: 'b',
+        },
+      });
+      const groupC = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-c',
+          },
+          name: 'c',
+        },
+      });
+      const user1 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-1',
+          },
+          name: 'user1',
+        },
+      });
+      const user2 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-2',
+          },
+          name: 'user2',
+        },
+      });
+      const groups = [rootGroup, groupA, groupB, groupC];
+      const users = [user1, user2];
+      const groupMember = new Map<string, Set<string>>();
+      groupMember.set('group-id-b', new Set(['group-id-c']));
+      const groupMemberOf = new Map<string, Set<string>>();
+      groupMemberOf.set('user-id-1', new Set(['group-id-a']));
+      groupMemberOf.set('user-id-2', new Set(['group-id-c']));
+
+      // We have a root groups
+      // We have three groups: a, b, c. c is child of b
+      // we have two users: u1, u2. u1 is member of a, u2 is member of c
+      resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+
+      expect(rootGroup.spec.parent).toBeUndefined();
+      expect(rootGroup.spec.children).toEqual(
+        expect.arrayContaining(['a', 'b']),
+      );
+
+      expect(groupA.spec.parent).toEqual('root');
+      expect(groupA.spec.children).toEqual(expect.arrayContaining([]));
+
+      expect(groupB.spec.parent).toEqual('root');
+      expect(groupB.spec.children).toEqual(expect.arrayContaining(['c']));
+
+      expect(groupC.spec.parent).toEqual('b');
+      expect(groupC.spec.children).toEqual(expect.arrayContaining([]));
+
+      expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a']));
+      expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c']));
+    });
+  });
+});

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/read.ts

@@ -0,0 +1,419 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
+import limiterFactory from 'p-limit';
+import { MicrosoftGraphClient } from './client';
+import {
+  MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
+  MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+} from './constants';
+import { normalizeEntityName } from './helper';
+import { buildMemberOf, buildOrgHierarchy } from './org';
+import {
+  GroupTransformer,
+  OrganizationTransformer,
+  UserTransformer,
+} from './types';
+
+export async function defaultUserTransformer(
+  user: MicrosoftGraph.User,
+  userPhoto?: string,
+): Promise<UserEntity | undefined> {
+  if (!user.id || !user.displayName || !user.mail) {
+    return undefined;
+  }
+
+  const name = normalizeEntityName(user.mail);
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name,
+      annotations: {
+        [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!,
+      },
+    },
+    spec: {
+      profile: {
+        displayName: user.displayName!,
+        email: user.mail!,
+
+        // TODO: Additional fields?
+        // jobTitle: user.jobTitle || undefined,
+        // officeLocation: user.officeLocation || undefined,
+        // mobilePhone: user.mobilePhone || undefined,
+      },
+      memberOf: [],
+    },
+  };
+
+  if (userPhoto) {
+    entity.spec.profile!.picture = userPhoto;
+  }
+
+  return entity;
+}
+
+export async function readMicrosoftGraphUsers(
+  client: MicrosoftGraphClient,
+  options?: { userFilter?: string; transformer?: UserTransformer },
+): Promise<{
+  users: UserEntity[]; // With all relations empty
+}> {
+  const users: UserEntity[] = [];
+  const limiter = limiterFactory(10);
+
+  const transformer = options?.transformer ?? defaultUserTransformer;
+  const promises: Promise<void>[] = [];
+
+  for await (const user of client.getUsers({
+    filter: options?.userFilter,
+  })) {
+    // Process all users in parallel, otherwise it can take quite some time
+    promises.push(
+      limiter(async () => {
+        const userPhoto = await client.getUserPhotoWithSizeLimit(
+          user.id!,
+          // We are limiting the photo size, as users with full resolution photos
+          // can make the Backstage API slow
+          120,
+        );
+
+        const entity = await transformer(user, userPhoto);
+
+        if (!entity) {
+          return;
+        }
+
+        users.push(entity);
+      }),
+    );
+  }
+
+  // Wait for all users and photos to be downloaded
+  await Promise.all(promises);
+
+  return { users };
+}
+
+export async function defaultOrganizationTransformer(
+  organization: MicrosoftGraph.Organization,
+): Promise<GroupEntity | undefined> {
+  if (!organization.id || !organization.displayName) {
+    return undefined;
+  }
+
+  const name = normalizeEntityName(organization.displayName!);
+  return {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: {
+      name: name,
+      description: organization.displayName!,
+      annotations: {
+        [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!,
+      },
+    },
+    spec: {
+      type: 'root',
+      profile: {
+        displayName: organization.displayName!,
+      },
+      children: [],
+    },
+  };
+}
+
+export async function readMicrosoftGraphOrganization(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { transformer?: OrganizationTransformer },
+): Promise<{
+  rootGroup?: GroupEntity; // With all relations empty
+}> {
+  // For now we expect a single root organization
+  const organization = await client.getOrganization(tenantId);
+  const transformer = options?.transformer ?? defaultOrganizationTransformer;
+  const rootGroup = await transformer(organization);
+
+  return { rootGroup };
+}
+
+export async function defaultGroupTransformer(
+  group: MicrosoftGraph.Group,
+  groupPhoto?: string,
+): Promise<GroupEntity | undefined> {
+  if (!group.id || !group.displayName) {
+    return undefined;
+  }
+
+  const name = normalizeEntityName(group.mailNickname || group.displayName);
+  const entity: GroupEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: {
+      name: name,
+      annotations: {
+        [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id,
+      },
+    },
+    spec: {
+      type: 'team',
+      profile: {},
+      children: [],
+    },
+  };
+
+  if (group.description) {
+    entity.metadata.description = group.description;
+  }
+  if (group.displayName) {
+    entity.spec.profile!.displayName = group.displayName;
+  }
+  if (group.mail) {
+    entity.spec.profile!.email = group.mail;
+  }
+  if (groupPhoto) {
+    entity.spec.profile!.picture = groupPhoto;
+  }
+
+  return entity;
+}
+
+export async function readMicrosoftGraphGroups(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { groupFilter?: string; transformer?: GroupTransformer },
+): Promise<{
+  groups: GroupEntity[]; // With all relations empty
+  rootGroup: GroupEntity | undefined; // With all relations empty
+  groupMember: Map<string, Set<string>>;
+  groupMemberOf: Map<string, Set<string>>;
+}> {
+  const groups: GroupEntity[] = [];
+  const groupMember: Map<string, Set<string>> = new Map();
+  const groupMemberOf: Map<string, Set<string>> = new Map();
+  const limiter = limiterFactory(10);
+
+  const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId);
+  if (rootGroup) {
+    groupMember.set(rootGroup.metadata.name, new Set<string>());
+    groups.push(rootGroup);
+  }
+
+  const transformer = options?.transformer ?? defaultGroupTransformer;
+  const promises: Promise<void>[] = [];
+
+  for await (const group of client.getGroups({
+    filter: options?.groupFilter,
+  })) {
+    // Process all groups in parallel, otherwise it can take quite some time
+    promises.push(
+      limiter(async () => {
+        // TODO: Loading groups photos doesn't work right now as Microsoft Graph
+        // doesn't allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo
+        /* const groupPhoto = await client.getGroupPhotoWithSizeLimit(
+          group.id!,
+          // We are limiting the photo size, as groups with full resolution photos
+          // can make the Backstage API slow
+          120,
+        );*/
+
+        const entity = await transformer(group /* , groupPhoto*/);
+
+        if (!entity) {
+          return;
+        }
+
+        for await (const member of client.getGroupMembers(group.id!)) {
+          if (!member.id) {
+            continue;
+          }
+
+          if (member['@odata.type'] === '#microsoft.graph.user') {
+            ensureItem(groupMemberOf, member.id, group.id!);
+          }
+
+          if (member['@odata.type'] === '#microsoft.graph.group') {
+            ensureItem(groupMember, group.id!, member.id);
+          }
+        }
+
+        groups.push(entity);
+      }),
+    );
+  }
+
+  // Wait for all group members and photos to be loaded
+  await Promise.all(promises);
+
+  return {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  };
+}
+
+export function resolveRelations(
+  rootGroup: GroupEntity | undefined,
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMember: Map<string, Set<string>>,
+  groupMemberOf: Map<string, Set<string>>,
+) {
+  // Build reference lookup tables, we reference them by the id the the graph
+  const groupMap: Map<string, GroupEntity> = new Map(); // by group-id or tenant-id
+
+  for (const group of groups) {
+    if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION],
+        group,
+      );
+    }
+    if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION],
+        group,
+      );
+    }
+  }
+
+  // Resolve all member relationships into the reverse direction
+  const parentGroups = new Map<string, Set<string>>();
+
+  groupMember.forEach((members, groupId) =>
+    members.forEach(m => ensureItem(parentGroups, m, groupId)),
+  );
+
+  // Make sure every group (except root) has at least one parent. If the parent is missing, add the root.
+  if (rootGroup) {
+    const tenantId = rootGroup.metadata.annotations![
+      MICROSOFT_GRAPH_TENANT_ID_ANNOTATION
+    ];
+
+    groups.forEach(group => {
+      const groupId = group.metadata.annotations![
+        MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
+      ];
+
+      if (!groupId) {
+        return;
+      }
+
+      if (retrieveItems(parentGroups, groupId).size === 0) {
+        ensureItem(parentGroups, groupId, tenantId);
+        ensureItem(groupMember, tenantId, groupId);
+      }
+    });
+  }
+
+  groups.forEach(group => {
+    const id =
+      group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ??
+      group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION];
+
+    retrieveItems(groupMember, id).forEach(m => {
+      const childGroup = groupMap.get(m);
+      if (childGroup) {
+        // TODO: This break when groups are transformed into different namespaces, use full entity refs instead
+
+        group.spec.children.push(childGroup.metadata.name);
+      }
+    });
+
+    retrieveItems(parentGroups, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        // TODO: Only having a single parent group might not match every companies model, but fine for now.
+
+        // TODO: use full entity refs
+        group.spec.parent = parentGroup.metadata.name;
+      }
+    });
+  });
+
+  // Make sure that all groups have proper parents and children
+  buildOrgHierarchy(groups);
+
+  // Set relations for all users
+  users.forEach(user => {
+    const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION];
+
+    retrieveItems(groupMemberOf, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        // TODO: use full entity refs
+        user.spec.memberOf.push(parentGroup.metadata.name);
+      }
+    });
+  });
+
+  // Make sure all transitive memberships are available
+  buildMemberOf(groups, users);
+}
+
+export async function readMicrosoftGraphOrg(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: {
+    userFilter?: string;
+    groupFilter?: string;
+    groupTransformer?: GroupTransformer;
+  },
+): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
+  const { users } = await readMicrosoftGraphUsers(client, {
+    userFilter: options?.userFilter,
+  });
+  const {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  } = await readMicrosoftGraphGroups(client, tenantId, {
+    groupFilter: options?.groupFilter,
+    transformer: options?.groupTransformer,
+  });
+
+  resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+  users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+  groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+
+  return { users, groups };
+}
+
+function ensureItem(
+  target: Map<string, Set<string>>,
+  key: string,
+  value: string,
+) {
+  let set = target.get(key);
+  if (!set) {
+    set = new Set();
+    target.set(key, set);
+  }
+  set!.add(value);
+}
+
+function retrieveItems(
+  target: Map<string, Set<string>>,
+  key: string,
+): Set<string> {
+  return target.get(key) ?? new Set();
+}

plugins/catalog-backend-extension-msgraph/src/microsoftGraph/types.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
+
+export type UserTransformer = (
+  user: MicrosoftGraph.User,
+  userPhoto?: string,
+) => Promise<UserEntity | undefined>;
+
+export type OrganizationTransformer = (
+  organization: MicrosoftGraph.Organization,
+) => Promise<GroupEntity | undefined>;
+
+export type GroupTransformer = (
+  group: MicrosoftGraph.Group,
+  groupPhoto?: string,
+) => Promise<GroupEntity | undefined>;

plugins/catalog-backend-extension-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  CatalogProcessor,
+  CatalogProcessorEmit,
+  results,
+} from '@backstage/plugin-catalog-backend';
+import { Logger } from 'winston';
+import {
+  GroupTransformer,
+  MicrosoftGraphClient,
+  MicrosoftGraphProviderConfig,
+  readMicrosoftGraphConfig,
+  readMicrosoftGraphOrg,
+} from '../microsoftGraph';
+
+/**
+ * Extracts teams and users out of a the Microsoft Graph API.
+ */
+export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
+  private readonly providers: MicrosoftGraphProviderConfig[];
+  private readonly logger: Logger;
+  private readonly groupTransformer?: GroupTransformer;
+
+  static fromConfig(
+    config: Config,
+    options: { logger: Logger; groupTransformer?: GroupTransformer },
+  ) {
+    const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
+    return new MicrosoftGraphOrgReaderProcessor({
+      ...options,
+      providers: c ? readMicrosoftGraphConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    providers: MicrosoftGraphProviderConfig[];
+    logger: Logger;
+    groupTransformer?: GroupTransformer;
+  }) {
+    this.providers = options.providers;
+    this.logger = options.logger;
+    this.groupTransformer = options.groupTransformer;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'microsoft-graph-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(p.target),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
+      );
+    }
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info('Reading Microsoft Graph users and groups');
+
+    // We create a client each time as we need one that matches the specific provider
+    const client = MicrosoftGraphClient.create(provider);
+    const { users, groups } = await readMicrosoftGraphOrg(
+      client,
+      provider.tenantId,
+      {
+        userFilter: provider.userFilter,
+        groupFilter: provider.groupFilter,
+        groupTransformer: this.groupTransformer,
+      },
+    );
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`,
+    );
+
+    // Done!
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend-extension-msgraph/src/processors/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';

plugins/catalog-backend-extension-msgraph/src/setupTests.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};

plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -26,8 +26,14 @@ import {
 import * as results from './results';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 
+// TODO: Remove this deprecated processor, the related code in
+// ./microsoftGraph/, and the config section in the future.
+
 /**
- * Extracts teams and users out of an LDAP server.
+ * Extracts teams and users out of a the Microsoft Graph API.
+ *
+ * @deprecated Use the MicrosoftGraphOrgReaderProcessor from package
+ * @backstage/plugin-catalog-backend-extension-msgraph instead.
  */
 export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
   private readonly providers: MicrosoftGraphProviderConfig[];
@@ -58,6 +64,10 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
       return false;
     }
 
+    this.logger.warn(
+      'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-extension-msgraph instead.',
+    );
+
     const provider = this.providers.find(p =>
       location.target.startsWith(p.target),
     );