Add support for reading groups and users from the Microsoft Graph API. (#3293)

* Add support for reading groups and users from the Microsoft Graph API.

* Limit amount of parallel requests

* Add helper for paging in odata collections

* Add tests for the microsoft graph reader

* Output the correct relations between groups and users

.changeset/curly-yaks-invite.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Add support for reading groups and users from the Microsoft Graph API.

app-config.yaml

@@ -140,6 +140,19 @@ catalog:
       #       dn: ou=access,ou=groups,ou=example,dc=example,dc=net
       #       options:
       #         filter: (&(objectClass=some-group-class)(!(groupType=email)))
+    microsoftGraphOrg:
+      ### Example for how to add your Microsoft Graph tenant
+      #providers:
+      #  - target: https://graph.microsoft.com/v1.0/
+      #    authority: https://login.microsoftonline.com/
+      #    tenantId:
+      #      $env: MICROSOFT_GRAPH_TENANT_ID
+      #    clientId:
+      #      $env: MICROSOFT_GRAPH_CLIENT_ID
+      #    clientSecret:
+      #      $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN
+      #    userFilter: accountEnabled eq true and userType eq 'member'
+      #    groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
 
   locations:
     # Backstage example components

docs/features/software-catalog/well-known-annotations.md

@@ -190,9 +190,25 @@ metadata:
 ```
 
 The value of these annotations are the corresponding attributes that were found
-when ingestion the entity from LDAP. Not all of them may be present, depending
+when ingesting the entity from LDAP. Not all of them may be present, depending
 on what attributes that the server presented at ingestion time.
 
+### graph.microsoft.com/tenant-id, graph.microsoft.com/group-id, graph.microsoft.com/user-id
+
+```yaml
+# Example:
+metadata:
+  annotations:
+    graph.microsoft.com/tenant-id: 6902611b-ffc1-463f-8af3-4d5285dc057b
+    graph.microsoft.com/group-id: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21
+    graph.microsoft.com/user-id: 2de244b5-104b-4e8f-a3b8-dce3c31e54b6
+```
+
+The value of these annotations are the corresponding attributes that were found
+when ingesting the entity from the Microsoft Graph API. Not all of them may be
+present, depending on what attributes that the server presented at ingestion
+time.
+
 ### sonarqube.org/project-key
 
 ```yaml

plugins/catalog-backend/package.json

@@ -20,6 +20,7 @@
     "clean": "backstage-cli clean"
   },
   "dependencies": {
+    "@azure/msal-node": "^1.0.0-alpha.8",
     "@backstage/backend-common": "^0.3.0",
     "@backstage/catalog-model": "^0.2.0",
     "@backstage/config": "^0.1.1",
@@ -37,6 +38,7 @@
     "lodash": "^4.17.15",
     "morgan": "^1.10.0",
     "p-limit": "^3.0.2",
+    "qs": "^6.9.4",
     "sqlite3": "^5.0.0",
     "uuid": "^8.0.0",
     "winston": "^3.2.1",

plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
+import {
+  MicrosoftGraphClient,
+  MicrosoftGraphProviderConfig,
+  readMicrosoftGraphConfig,
+  readMicrosoftGraphOrg,
+} from './microsoftGraph';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+/**
+ * Extracts teams and users out of an LDAP server.
+ */
+export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
+  private readonly providers: MicrosoftGraphProviderConfig[];
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
+    return new MicrosoftGraphOrgReaderProcessor({
+      ...options,
+      providers: c ? readMicrosoftGraphConfig(c) : [],
+    });
+  }
+
+  constructor(options: {
+    providers: MicrosoftGraphProviderConfig[];
+    logger: Logger;
+  }) {
+    this.providers = options.providers;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'microsoft-graph-org') {
+      return false;
+    }
+
+    const provider = this.providers.find(p =>
+      location.target.startsWith(p.target),
+    );
+    if (!provider) {
+      throw new Error(
+        `There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
+      );
+    }
+
+    // Read out all of the raw data
+    const startTimestamp = Date.now();
+    this.logger.info('Reading Microsoft Graph users and groups');
+
+    // We create a client each time as we need one that matches the specific provider
+    const client = MicrosoftGraphClient.create(provider);
+    const { users, groups } = await readMicrosoftGraphOrg(
+      client,
+      provider.tenantId,
+      {
+        userFilter: provider.userFilter,
+        groupFilter: provider.groupFilter,
+      },
+    );
+
+    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
+    this.logger.debug(
+      `Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`,
+    );
+
+    // Done!
+    for (const group of groups) {
+      emit(results.entity(location, group));
+    }
+    for (const user of users) {
+      emit(results.entity(location, user));
+    }
+
+    return true;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -27,6 +27,7 @@ export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { OwnerRelationProcessor } from './OwnerRelationProcessor';
 export { LocationRefProcessor } from './LocationEntityProcessor';
+export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';
 export { PlaceholderProcessor } from './PlaceholderProcessor';
 export type { PlaceholderResolver } from './PlaceholderProcessor';
 export { StaticLocationProcessor } from './StaticLocationProcessor';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts

@@ -0,0 +1,324 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import { msw } from '@backstage/test-utils';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { MicrosoftGraphClient } from './client';
+
+describe('MicrosoftGraphClient', () => {
+  const confidentialClientApplication: jest.Mocked<msal.ConfidentialClientApplication> = {
+    acquireTokenByClientCredential: jest.fn(),
+  } as any;
+  let client: MicrosoftGraphClient;
+  const worker = setupServer();
+
+  msw.setupDefaultHandlers(worker);
+
+  beforeEach(() => {
+    confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue(
+      { token: 'ACCESS_TOKEN' } as any,
+    );
+    client = new MicrosoftGraphClient(
+      'https://example.com',
+      confidentialClientApplication,
+    );
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+    worker.resetHandlers();
+  });
+
+  it('should perform raw request', async () => {
+    worker.use(
+      rest.get('https://other.example.com/', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestRaw('https://other.example.com/');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledTimes(1);
+    expect(
+      confidentialClientApplication.acquireTokenByClientCredential,
+    ).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] });
+  });
+
+  it('should perform simple api request', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: 'example' })),
+      ),
+    );
+
+    const response = await client.requestApi('users');
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ value: 'example' });
+  });
+
+  it('should perform api request with filter, select and expand', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (req, res, ctx) =>
+        res(ctx.status(200), ctx.json({ queryString: req.url.search })),
+      ),
+    );
+
+    const response = await client.requestApi('users', {
+      filter: 'test eq true',
+      expand: ['children'],
+      select: ['id', 'children'],
+    });
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      queryString:
+        '?$filter=test%20eq%20true&$select=id,children&$expand=children',
+    });
+  });
+
+  it('should perform collection request for a single page', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first']);
+  });
+
+  it('should perform collection request for multiple pages', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: ['first'],
+            '@odata.nextLink': 'https://example.com/users2',
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get('https://example.com/users2', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ value: ['second'] })),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.requestCollection<string>('users'),
+    );
+
+    expect(values).toEqual(['first', 'second']);
+  });
+
+  it('should load user profile', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            surname: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const userProfile = await client.getUserProfile('user-id');
+
+    expect(userProfile).toEqual({ surname: 'Example' });
+  });
+
+  it('should throw expection if load user profile fails', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    await expect(() => client.getUserProfile('user-id')).rejects.toThrowError();
+  });
+
+  it('should load user profile photo with max size of 120', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              {
+                height: 120,
+                id: 120,
+              },
+              {
+                height: 500,
+                id: 500,
+              },
+            ],
+          }),
+        ),
+      ),
+    );
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should not fail if user has no profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
+        res(ctx.status(404)),
+      ),
+    );
+
+    const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
+
+    expect(photo).toBeFalsy();
+  });
+
+  it('should load profile photo', async () => {
+    worker.use(
+      rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) =>
+        res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load profile photo for size 120', async () => {
+    worker.use(
+      rest.get(
+        'https://example.com/users/user-id/photos/120/*',
+        (_, res, ctx) => res(ctx.status(200), ctx.text('911')),
+      ),
+    );
+
+    const photo = await client.getUserPhoto('user-id', '120');
+
+    expect(photo).toEqual('data:image/jpeg;base64,OTEx');
+  });
+
+  it('should load users', async () => {
+    worker.use(
+      rest.get('https://example.com/users', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ surname: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getUsers());
+
+    expect(values).toEqual([{ surname: 'Example' }]);
+  });
+
+  it('should load groups', async () => {
+    worker.use(
+      rest.get('https://example.com/groups', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [{ displayName: 'Example' }],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(client.getGroups());
+
+    expect(values).toEqual([{ displayName: 'Example' }]);
+  });
+
+  it('should load group members', async () => {
+    worker.use(
+      rest.get('https://example.com/groups/group-id/members', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            value: [
+              { '@odata.type': '#microsoft.graph.user' },
+              { '@odata.type': '#microsoft.graph.group' },
+            ],
+          }),
+        ),
+      ),
+    );
+
+    const values = await collectAsyncIterable(
+      client.getGroupMembers('group-id'),
+    );
+
+    expect(values).toEqual([
+      { '@odata.type': '#microsoft.graph.user' },
+      { '@odata.type': '#microsoft.graph.group' },
+    ]);
+  });
+
+  it('should load organization', async () => {
+    worker.use(
+      rest.get('https://example.com/organization/tentant-id', (_, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            displayName: 'Example',
+          }),
+        ),
+      ),
+    );
+
+    const organization = await client.getOrganization('tentant-id');
+
+    expect(organization).toEqual({ displayName: 'Example' });
+  });
+});
+
+async function collectAsyncIterable<T>(
+  iterable: AsyncIterable<T>,
+): Promise<T[]> {
+  const values = [];
+  for await (const value of iterable) {
+    values.push(value);
+  }
+  return values;
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts

@@ -0,0 +1,201 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as msal from '@azure/msal-node';
+import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
+import fetch from 'cross-fetch';
+import qs from 'qs';
+import { MicrosoftGraphProviderConfig } from './config';
+
+export type ODataQuery = {
+  filter?: string;
+  expand?: string[];
+  select?: string[];
+};
+
+export type GroupMember =
+  | (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' })
+  | (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' });
+
+export class MicrosoftGraphClient {
+  static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient {
+    const clientConfig: msal.Configuration = {
+      auth: {
+        clientId: config.clientId,
+        clientSecret: config.clientSecret,
+        authority: `${config.authority}/${config.tenantId}`,
+      },
+    };
+    const pca = new msal.ConfidentialClientApplication(clientConfig);
+    return new MicrosoftGraphClient(config.target, pca);
+  }
+
+  constructor(
+    private readonly baseUrl: string,
+    private readonly pca: msal.ConfidentialClientApplication,
+  ) {}
+
+  async *requestCollection<T>(
+    path: string,
+    query?: ODataQuery,
+  ): AsyncIterable<T> {
+    let response = await this.requestApi(path, query);
+
+    for (;;) {
+      if (response.status !== 200) {
+        await this.handleError(path, response);
+      }
+
+      const result = await response.json();
+      const elements: T[] = result.value;
+
+      yield* elements;
+
+      // Follow cursor to the next page if one is available
+      if (!result['@odata.nextLink']) {
+        return;
+      }
+
+      response = await this.requestRaw(result['@odata.nextLink']);
+    }
+  }
+
+  async requestApi(path: string, query?: ODataQuery): Promise<Response> {
+    const queryString = qs.stringify(
+      {
+        $filter: query?.filter,
+        $select: query?.select?.join(','),
+        $expand: query?.expand?.join(','),
+      },
+      {
+        addQueryPrefix: true,
+        // Microsoft Graph doesn't like an encoded query string
+        encode: false,
+      },
+    );
+
+    return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`);
+  }
+
+  async requestRaw(url: string): Promise<Response> {
+    // Make sure that we always have a valid access token (might be cached)
+    const token = await this.pca.acquireTokenByClientCredential({
+      scopes: ['https://graph.microsoft.com/.default'],
+    });
+
+    return await fetch(url, {
+      headers: {
+        Authorization: `Bearer ${token.accessToken}`,
+      },
+    });
+  }
+
+  async getUserProfile(userId: string): Promise<MicrosoftGraph.User> {
+    const response = await this.requestApi(`users/${userId}`);
+
+    if (response.status !== 200) {
+      await this.handleError('user profile', response);
+    }
+
+    return await response.json();
+  }
+
+  async getUserPhotoWithSizeLimit(
+    userId: string,
+    maxSize: number,
+  ): Promise<string | undefined> {
+    const response = await this.requestApi(`users/${userId}/photos`);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError('user photos', response);
+    }
+
+    const result = await response.json();
+    const photos = result.value as MicrosoftGraph.ProfilePhoto[];
+    let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined;
+
+    // Find the biggest picture that is small than the max size
+    for (const p of photos) {
+      if (
+        !selectedPhoto ||
+        (p.height! >= selectedPhoto.height! && p.height! <= maxSize)
+      ) {
+        selectedPhoto = p;
+      }
+    }
+
+    if (!selectedPhoto) {
+      return undefined;
+    }
+
+    return await this.getUserPhoto(userId, selectedPhoto.id!);
+  }
+
+  async getUserPhoto(
+    userId: string,
+    sizeId?: string,
+  ): Promise<string | undefined> {
+    const path = sizeId
+      ? `users/${userId}/photos/${sizeId}/$value`
+      : `users/${userId}/photo/$value`;
+    const response = await this.requestApi(path);
+
+    if (response.status === 404) {
+      return undefined;
+    } else if (response.status !== 200) {
+      await this.handleError('photo', response);
+    }
+
+    return `data:image/jpeg;base64,${Buffer.from(
+      await response.arrayBuffer(),
+    ).toString('base64')}`;
+  }
+
+  async *getUsers(query?: ODataQuery): AsyncIterable<MicrosoftGraph.User> {
+    yield* this.requestCollection<MicrosoftGraph.User>(`users`, query);
+  }
+
+  async *getGroups(query?: ODataQuery): AsyncIterable<MicrosoftGraph.Group> {
+    yield* this.requestCollection<MicrosoftGraph.Group>(`groups`, query);
+  }
+
+  async *getGroupMembers(groupId: string): AsyncIterable<GroupMember> {
+    yield* this.requestCollection<GroupMember>(`groups/${groupId}/members`);
+  }
+
+  async getOrganization(
+    tenantId: string,
+  ): Promise<MicrosoftGraph.Organization> {
+    const response = await this.requestApi(`organization/${tenantId}`);
+
+    if (response.status !== 200) {
+      await this.handleError('organization/${tenantId}', response);
+    }
+
+    return await response.json();
+  }
+
+  private async handleError(path: string, response: Response): Promise<void> {
+    const result = await response.json();
+    const error = result.error as MicrosoftGraph.PublicError;
+
+    throw new Error(
+      `Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`,
+    );
+  }
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { readMicrosoftGraphConfig } from './config';
+
+describe('readMicrosoftGraphConfig', () => {
+  it('applies all of the defaults', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(
+      ConfigReader.fromConfigs([{ context: '', data: config }]),
+    );
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.microsoftonline.com',
+        userFilter: undefined,
+        groupFilter: undefined,
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+
+  it('reads all the values', () => {
+    const config = {
+      providers: [
+        {
+          target: 'target',
+          tenantId: 'tenantId',
+          clientId: 'clientId',
+          clientSecret: 'clientSecret',
+          authority: 'https://login.example.com/',
+          userFilter: 'accountEnabled eq true',
+          groupFilter: 'securityEnabled eq false',
+        },
+      ],
+    };
+    const actual = readMicrosoftGraphConfig(
+      ConfigReader.fromConfigs([{ context: '', data: config }]),
+    );
+    const expected = [
+      {
+        target: 'target',
+        tenantId: 'tenantId',
+        clientId: 'clientId',
+        clientSecret: 'clientSecret',
+        authority: 'https://login.example.com',
+        userFilter: 'accountEnabled eq true',
+        groupFilter: 'securityEnabled eq false',
+      },
+    ];
+    expect(actual).toEqual(expected);
+  });
+});

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+
+export type MicrosoftGraphProviderConfig = {
+  target: string;
+  authority: string;
+  tenantId: string;
+  clientId: string;
+  clientSecret: string;
+  userFilter?: string;
+  groupFilter?: string;
+};
+
+export function readMicrosoftGraphConfig(
+  config: Config,
+): MicrosoftGraphProviderConfig[] {
+  const providers: MicrosoftGraphProviderConfig[] = [];
+  const providerConfigs = config.getOptionalConfigArray('providers') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    const authority =
+      providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') ||
+      'https://login.microsoftonline.com';
+    const tenantId = providerConfig.getString('tenantId');
+    const clientId = providerConfig.getString('clientId');
+    const clientSecret = providerConfig.getString('clientSecret');
+    const userFilter = providerConfig.getOptionalString('userFilter');
+    const groupFilter = providerConfig.getOptionalString('groupFilter');
+
+    providers.push({
+      target,
+      authority,
+      tenantId,
+      clientId,
+      clientSecret,
+      userFilter,
+      groupFilter,
+    });
+  }
+
+  return providers;
+}

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The tenant id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION =
+  'graph.microsoft.com/tenant-id';
+
+/**
+ * The group id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION =
+  'graph.microsoft.com/group-id';
+
+/**
+ * The user id used by the Microsoft Graph API
+ */
+export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export { MicrosoftGraphClient } from './client';
+export type { MicrosoftGraphProviderConfig } from './config';
+export { readMicrosoftGraphConfig } from './config';
+export { readMicrosoftGraphOrg } from './read';

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts

@@ -0,0 +1,339 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import merge from 'lodash/merge';
+import { RecursivePartial } from '../../../util';
+import { GroupMember, MicrosoftGraphClient } from './client';
+import {
+  normalizeEntityName,
+  readMicrosoftGraphGroups,
+  readMicrosoftGraphOrganization,
+  readMicrosoftGraphUsers,
+  resolveRelations,
+} from './read';
+
+function user(data: RecursivePartial<UserEntity>): UserEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name: 'name' },
+      spec: { profile: {}, memberOf: [] },
+    } as UserEntity,
+    data,
+  );
+}
+
+function group(data: RecursivePartial<GroupEntity>): GroupEntity {
+  return merge(
+    {},
+    {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: 'name',
+      },
+      spec: {
+        ancestors: [],
+        children: [],
+        descendants: [],
+        type: 'team',
+      },
+    } as GroupEntity,
+    data,
+  );
+}
+
+describe('read microsoft graph', () => {
+  const client: jest.Mocked<MicrosoftGraphClient> = {
+    getUsers: jest.fn(),
+    getGroups: jest.fn(),
+    getGroupMembers: jest.fn(),
+    getUserPhotoWithSizeLimit: jest.fn(),
+    getOrganization: jest.fn(),
+  } as any;
+
+  afterEach(() => jest.resetAllMocks());
+
+  describe('normalizeEntityName', () => {
+    it('should normalize name to valid entity name', () => {
+      expect(normalizeEntityName('User Name')).toBe('user_name');
+    });
+
+    it('should normalize e-mail to valid entity name', () => {
+      expect(normalizeEntityName('user.name@example.com')).toBe(
+        'user.name_example.com',
+      );
+    });
+  });
+
+  describe('readMicrosoftGraphUsers', () => {
+    it('should read users', async () => {
+      async function* getExampleUsers() {
+        yield {
+          id: 'userid',
+          displayName: 'User Name',
+          mail: 'user.name@example.com',
+        };
+      }
+
+      client.getUsers.mockImplementation(getExampleUsers);
+      client.getUserPhotoWithSizeLimit.mockResolvedValue(
+        'data:image/jpeg;base64,...',
+      );
+
+      const { users } = await readMicrosoftGraphUsers(client, {
+        userFilter: 'accountEnabled eq true',
+      });
+
+      expect(users).toEqual([
+        user({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/user-id': 'userid',
+            },
+            name: 'user.name_example.com',
+          },
+          spec: {
+            profile: {
+              displayName: 'User Name',
+              email: 'user.name@example.com',
+              picture: 'data:image/jpeg;base64,...',
+            },
+          },
+        }),
+      ]);
+
+      expect(client.getUsers).toBeCalledTimes(1);
+      expect(client.getUsers).toBeCalledWith({
+        filter: 'accountEnabled eq true',
+        select: ['id', 'displayName', 'mail'],
+      });
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
+      expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
+    });
+  });
+
+  describe('readMicrosoftGraphOrganization', () => {
+    it('should read organization', async () => {
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const { rootGroup } = await readMicrosoftGraphOrganization(
+        client,
+        'tenantid',
+      );
+
+      expect(rootGroup).toEqual(
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/tenant-id': 'tenantid',
+            },
+            name: 'organization_name',
+            description: 'Organization Name',
+          },
+          spec: {
+            type: 'root',
+          },
+        }),
+      );
+
+      expect(client.getOrganization).toBeCalledTimes(1);
+      expect(client.getOrganization).toBeCalledWith('tenantid');
+    });
+  });
+
+  describe('readMicrosoftGraphGroups', () => {
+    it('should read groups', async () => {
+      async function* getExampleGroups() {
+        yield {
+          id: 'groupid',
+          displayName: 'Group Name',
+        };
+      }
+
+      async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
+        yield {
+          '@odata.type': '#microsoft.graph.group',
+          id: 'childgroupid',
+        };
+        yield {
+          '@odata.type': '#microsoft.graph.user',
+          id: 'userid',
+        };
+      }
+
+      client.getGroups.mockImplementation(getExampleGroups);
+      client.getGroupMembers.mockImplementation(getExampleGroupMembers);
+      client.getOrganization.mockResolvedValue({
+        id: 'tenantid',
+        displayName: 'Organization Name',
+      });
+
+      const {
+        groups,
+        groupMember,
+        groupMemberOf,
+        rootGroup,
+      } = await readMicrosoftGraphGroups(client, 'tenantid', {
+        groupFilter: 'securityEnabled eq false',
+      });
+
+      const expectedRootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenantid',
+          },
+          name: 'organization_name',
+          description: 'Organization Name',
+        },
+        spec: {
+          type: 'root',
+        },
+      });
+      expect(groups).toEqual([
+        expectedRootGroup,
+        group({
+          metadata: {
+            annotations: {
+              'graph.microsoft.com/group-id': 'groupid',
+            },
+            name: 'group_name',
+            description: 'Group Name',
+          },
+          spec: {
+            type: 'team',
+          },
+        }),
+      ]);
+      expect(rootGroup).toEqual(expectedRootGroup);
+      expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid']));
+      expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid']));
+      expect(groupMember.get('organization_name')).toEqual(new Set());
+
+      expect(client.getGroups).toBeCalledTimes(1);
+      expect(client.getGroups).toBeCalledWith({
+        filter: 'securityEnabled eq false',
+        select: ['id', 'displayName', 'mailNickname'],
+      });
+      expect(client.getGroupMembers).toBeCalledTimes(1);
+      expect(client.getGroupMembers).toBeCalledWith('groupid');
+    });
+  });
+
+  describe('resolveRelations', () => {
+    it('should resolve relations', async () => {
+      const rootGroup = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/tenant-id': 'tenant-id-root',
+          },
+          name: 'root',
+        },
+        spec: {
+          type: 'root',
+        },
+      });
+      const groupA = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-a',
+          },
+          name: 'a',
+        },
+      });
+      const groupB = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-b',
+          },
+          name: 'b',
+        },
+      });
+      const groupC = group({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/group-id': 'group-id-c',
+          },
+          name: 'c',
+        },
+      });
+      const user1 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-1',
+          },
+          name: 'user1',
+        },
+      });
+      const user2 = user({
+        metadata: {
+          annotations: {
+            'graph.microsoft.com/user-id': 'user-id-2',
+          },
+          name: 'user2',
+        },
+      });
+      const groups = [rootGroup, groupA, groupB, groupC];
+      const users = [user1, user2];
+      const groupMember = new Map<string, Set<string>>();
+      groupMember.set('group-id-b', new Set(['group-id-c']));
+      const groupMemberOf = new Map<string, Set<string>>();
+      groupMemberOf.set('user-id-1', new Set(['group-id-a']));
+      groupMemberOf.set('user-id-2', new Set(['group-id-c']));
+
+      // We have a root groups
+      // We have three groups: a, b, c. c is child of b
+      // we have two users: u1, u2. u1 is member of a, u2 is member of c
+      resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+
+      expect(rootGroup.spec.parent).toBeUndefined();
+      expect(rootGroup.spec.ancestors).toEqual(expect.arrayContaining([]));
+      expect(rootGroup.spec.children).toEqual(
+        expect.arrayContaining(['a', 'b']),
+      );
+      expect(rootGroup.spec.descendants).toEqual(
+        expect.arrayContaining(['a', 'b', 'c']),
+      );
+
+      expect(groupA.spec.parent).toEqual('root');
+      expect(groupA.spec.ancestors).toEqual(expect.arrayContaining(['root']));
+      expect(groupA.spec.children).toEqual(expect.arrayContaining([]));
+      expect(groupA.spec.descendants).toEqual(expect.arrayContaining([]));
+
+      expect(groupB.spec.parent).toEqual('root');
+      expect(groupB.spec.ancestors).toEqual(expect.arrayContaining(['root']));
+      expect(groupB.spec.children).toEqual(expect.arrayContaining(['c']));
+      expect(groupB.spec.descendants).toEqual(expect.arrayContaining(['c']));
+
+      expect(groupC.spec.parent).toEqual('b');
+      expect(groupC.spec.ancestors).toEqual(
+        expect.arrayContaining(['root', 'b']),
+      );
+      expect(groupC.spec.children).toEqual(expect.arrayContaining([]));
+      expect(groupC.spec.descendants).toEqual(expect.arrayContaining([]));
+
+      expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a']));
+
+      expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c']));
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts

@@ -0,0 +1,342 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildMemberOf, buildOrgHierarchy } from '../util/org';
+import { MicrosoftGraphClient } from './client';
+import {
+  MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
+  MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
+  MICROSOFT_GRAPH_USER_ID_ANNOTATION,
+} from './constants';
+import limiterFactory from 'p-limit';
+
+export function normalizeEntityName(name: string): string {
+  return name
+    .trim()
+    .toLocaleLowerCase()
+    .replace(/[^a-zA-Z0-9_\-\.]/g, '_');
+}
+
+export async function readMicrosoftGraphUsers(
+  client: MicrosoftGraphClient,
+  options?: { userFilter?: string },
+): Promise<{
+  users: UserEntity[]; // With all relations empty
+}> {
+  const entities: UserEntity[] = [];
+  const picturePromises: Promise<void>[] = [];
+  const limiter = limiterFactory(10);
+
+  for await (const user of client.getUsers({
+    filter: options?.userFilter,
+    select: ['id', 'displayName', 'mail'],
+  })) {
+    if (!user.id || !user.displayName || !user.mail) {
+      continue;
+    }
+
+    const name = normalizeEntityName(user.mail);
+    const entity: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: {
+        name,
+        annotations: {
+          [MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!,
+        },
+      },
+      spec: {
+        profile: {
+          displayName: user.displayName!,
+          email: user.mail!,
+
+          // TODO: Additional fields?
+          // jobTitle: user.jobTitle || undefined,
+          // officeLocation: user.officeLocation || undefined,
+          // mobilePhone: user.mobilePhone || undefined,
+        },
+        memberOf: [],
+      },
+    };
+
+    // Download the photos in parallel, otherwise it can take quite some time
+    const loadPhoto = limiter(async () => {
+      entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit(
+        user.id!,
+        // We are limiting the photo size, as users with full resolution photos
+        // can make the Backstage API slow
+        120,
+      );
+    });
+
+    picturePromises.push(loadPhoto);
+    entities.push(entity);
+  }
+
+  // Wait for all photos to be downloaded
+  await Promise.all(picturePromises);
+
+  return { users: entities };
+}
+
+export async function readMicrosoftGraphOrganization(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+): Promise<{
+  rootGroup: GroupEntity; // With all relations empty
+}> {
+  // For now we expect a single root orgranization
+  const organization = await client.getOrganization(tenantId);
+  const name = normalizeEntityName(organization.displayName!);
+  const rootGroup: GroupEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: {
+      name: name,
+      description: organization.displayName!,
+      annotations: {
+        [MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!,
+      },
+    },
+    spec: {
+      type: 'root',
+      ancestors: [],
+      children: [],
+      descendants: [],
+    },
+  };
+
+  return { rootGroup };
+}
+
+export async function readMicrosoftGraphGroups(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { groupFilter?: string },
+): Promise<{
+  groups: GroupEntity[]; // With all relations empty
+  rootGroup: GroupEntity | undefined; // With all relations empty
+  groupMember: Map<string, Set<string>>;
+  groupMemberOf: Map<string, Set<string>>;
+}> {
+  const groups: GroupEntity[] = [];
+  const groupMember: Map<string, Set<string>> = new Map();
+  const groupMemberOf: Map<string, Set<string>> = new Map();
+  const limiter = limiterFactory(10);
+
+  const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId);
+  groupMember.set(rootGroup.metadata.name, new Set<string>());
+  groups.push(rootGroup);
+
+  const groupMemberPromises: Promise<void>[] = [];
+
+  for await (const group of client.getGroups({
+    filter: options?.groupFilter,
+    select: ['id', 'displayName', 'mailNickname'],
+  })) {
+    if (!group.id || !group.displayName) {
+      continue;
+    }
+
+    const name = normalizeEntityName(group.mailNickname || group.displayName);
+    const entity: GroupEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Group',
+      metadata: {
+        name: name,
+        description: group.displayName,
+        annotations: {
+          [MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id,
+        },
+      },
+      spec: {
+        type: 'team',
+        // TODO: We could include a group email and picture
+        ancestors: [],
+        children: [],
+        descendants: [],
+      },
+    };
+
+    // Download the members in parallel, otherwise it can take quite some time
+    const loadGroupMembers = limiter(async () => {
+      for await (const member of client.getGroupMembers(group.id!)) {
+        if (!member.id) {
+          continue;
+        }
+
+        if (member['@odata.type'] === '#microsoft.graph.user') {
+          ensureItem(groupMemberOf, member.id, group.id!);
+        }
+
+        if (member['@odata.type'] === '#microsoft.graph.group') {
+          ensureItem(groupMember, group.id!, member.id);
+        }
+      }
+    });
+
+    groupMemberPromises.push(loadGroupMembers);
+    groups.push(entity);
+  }
+
+  // Wait for all group members to be loaded
+  await Promise.all(groupMemberPromises);
+
+  return {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  };
+}
+
+export function resolveRelations(
+  rootGroup: GroupEntity | undefined,
+  groups: GroupEntity[],
+  users: UserEntity[],
+  groupMember: Map<string, Set<string>>,
+  groupMemberOf: Map<string, Set<string>>,
+) {
+  // Build reference lookup tables, we reference them by the id the the graph
+  const groupMap: Map<string, GroupEntity> = new Map(); // by group-id or tenant-id
+
+  for (const group of groups) {
+    if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION],
+        group,
+      );
+    }
+    if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) {
+      groupMap.set(
+        group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION],
+        group,
+      );
+    }
+  }
+
+  // Resolve all member relationships into the reverse direction
+  const parentGroups = new Map<string, Set<string>>();
+
+  groupMember.forEach((members, groupId) =>
+    members.forEach(m => ensureItem(parentGroups, m, groupId)),
+  );
+
+  // Make sure every group (except root) has at least one parent. If the parent is missing, add the root.
+  if (rootGroup) {
+    const tenantId = rootGroup.metadata.annotations![
+      MICROSOFT_GRAPH_TENANT_ID_ANNOTATION
+    ];
+
+    groups.forEach(group => {
+      const groupId = group.metadata.annotations![
+        MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
+      ];
+
+      if (!groupId) {
+        return;
+      }
+
+      if (retrieveItems(parentGroups, groupId).size === 0) {
+        ensureItem(parentGroups, groupId, tenantId);
+        ensureItem(groupMember, tenantId, groupId);
+      }
+    });
+  }
+
+  groups.forEach(group => {
+    const id =
+      group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ??
+      group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION];
+
+    retrieveItems(groupMember, id).forEach(m => {
+      const childGroup = groupMap.get(m);
+      if (childGroup) {
+        group.spec.children.push(childGroup.metadata.name);
+      }
+    });
+
+    retrieveItems(parentGroups, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        // TODO: Only having a single parent group might not match every companies model, but fine for now.
+        group.spec.parent = parentGroup.metadata.name;
+      }
+    });
+  });
+
+  // Make sure that all groups have proper ancestors and descendants
+  buildOrgHierarchy(groups);
+
+  // Set relations for all users
+  users.forEach(user => {
+    const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION];
+
+    retrieveItems(groupMemberOf, id).forEach(p => {
+      const parentGroup = groupMap.get(p);
+      if (parentGroup) {
+        user.spec.memberOf.push(parentGroup.metadata.name);
+      }
+    });
+  });
+
+  // Make sure all transitive memberships are available
+  buildMemberOf(groups, users);
+}
+
+export async function readMicrosoftGraphOrg(
+  client: MicrosoftGraphClient,
+  tenantId: string,
+  options?: { userFilter?: string; groupFilter?: string },
+): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
+  const { users } = await readMicrosoftGraphUsers(client, {
+    userFilter: options?.userFilter,
+  });
+  const {
+    groups,
+    rootGroup,
+    groupMember,
+    groupMemberOf,
+  } = await readMicrosoftGraphGroups(client, tenantId, {
+    groupFilter: options?.groupFilter,
+  });
+
+  resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
+  users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+  groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
+
+  return { users, groups };
+}
+
+function ensureItem(
+  target: Map<string, Set<string>>,
+  key: string,
+  value: string,
+) {
+  let set = target.get(key);
+  if (!set) {
+    set = new Set();
+    target.set(key, set);
+  }
+  set!.add(value);
+}
+
+function retrieveItems(
+  target: Map<string, Set<string>>,
+  key: string,
+): Set<string> {
+  return target.get(key) ?? new Set();
+}

plugins/catalog-backend/src/ingestion/processors/util/org.test.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-import { GroupEntity } from '@backstage/catalog-model';
-import { buildOrgHierarchy } from './org';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { buildMemberOf, buildOrgHierarchy } from './org';
 
 function g(
   name: string,
@@ -67,3 +67,22 @@ describe('buildOrgHierarchy', () => {
     expect(d.spec.ancestors).toEqual(expect.arrayContaining(['a']));
   });
 });
+
+describe('buildMemberOf', () => {
+  it('fills indirect member of groups', () => {
+    const a = g('a', undefined, []);
+    const b = g('b', 'a', []);
+    const c = g('c', 'b', []);
+    const u: UserEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'User',
+      metadata: { name },
+      spec: { profile: {}, memberOf: ['c'] },
+    };
+
+    const groups = [a, b, c];
+    buildOrgHierarchy(groups);
+    buildMemberOf(groups, [u]);
+    expect(u.spec.memberOf).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+  });
+});

plugins/catalog-backend/src/ingestion/processors/util/org.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { GroupEntity } from '@backstage/catalog-model';
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
 
 export function buildOrgHierarchy(groups: GroupEntity[]) {
   const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
@@ -93,3 +93,22 @@ export function buildOrgHierarchy(groups: GroupEntity[]) {
     visitAncestors(group);
   }
 }
+
+// Ensure that users have their transitive group memberships. Requires that
+// the groups were previously processed with buildOrgHierarchy()
+export function buildMemberOf(groups: GroupEntity[], users: UserEntity[]) {
+  const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
+
+  users.forEach(user => {
+    const transitiveMemberOf = new Set([...user.spec.memberOf]);
+
+    user.spec.memberOf.forEach(groupName => {
+      const group = groupsByName.get(groupName);
+
+      if (group) {
+        group.spec.ancestors.forEach(g => transitiveMemberOf.add(g));
+      }
+    });
+    user.spec.memberOf = [...transitiveMemberOf];
+  });
+}

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -46,6 +46,7 @@ import {
   LocationReaders,
   LocationRefProcessor,
   OwnerRelationProcessor,
+  MicrosoftGraphOrgReaderProcessor,
   PlaceholderProcessor,
   PlaceholderResolver,
   StaticLocationProcessor,
@@ -278,6 +279,7 @@ export class CatalogBuilder {
         new FileReaderProcessor(),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
         LdapOrgReaderProcessor.fromConfig(config, { logger }),
+        MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),
         new UrlReaderProcessor({ reader, logger }),
         new CodeOwnersProcessor({ reader }),
         new LocationRefProcessor(),

plugins/catalog/package.json

@@ -46,6 +46,7 @@
     "@backstage/cli": "^0.3.0",
     "@backstage/dev-utils": "^0.1.4",
     "@backstage/test-utils": "^0.1.3",
+    "@microsoft/microsoft-graph-types": "^1.25.0",
     "@testing-library/jest-dom": "^5.10.1",
     "@testing-library/react": "^10.4.1",
     "@testing-library/react-hooks": "^3.3.0",

yarn.lock

@@ -87,6 +87,24 @@
   resolved "https://registry.npmjs.org/@asyncapi/specs/-/specs-2.7.5.tgz#3a516d198fc41a1103695bd889fdd4fbbebe7f5d"
   integrity sha512-T1Ham9sqZKCtSowXRPaBCRy2oz3KHglqqrKiaO7lEudpP6lwH5SwXaq4qliyKzWaqd22srJHE4szdsorbFZKlw==
 
+"@azure/msal-common@^1.6.2":
+  version "1.6.2"
+  resolved "https://registry.npmjs.org/@azure/msal-common/-/msal-common-1.6.2.tgz#91f3732866d727e20f1e142e6e88a981268fbff2"
+  integrity sha512-GShzp1q7Ld8SwYiDEjQZ9PmFOY4x+2stE86maiguylE9/d/c2muqKjc8aepmEqyjbV7o/omDvEf2Sr9QcIqkSA==
+  dependencies:
+    debug "^4.1.1"
+
+"@azure/msal-node@^1.0.0-alpha.8":
+  version "1.0.0-alpha.12"
+  resolved "https://registry.npmjs.org/@azure/msal-node/-/msal-node-1.0.0-alpha.12.tgz#09d8d52f5cea90b133c3d48fe4ec477693040c91"
+  integrity sha512-uGLOJRWiEhfJIrTv/lwdm4RxQFm++00h83zNgDn0O3NkXlzAoCCq9QFYW84PjMR/Q2PUvVy7uW+6yKL/Nq3gBA==
+  dependencies:
+    "@azure/msal-common" "^1.6.2"
+    axios "^0.19.2"
+    debug "^4.1.1"
+    jsonwebtoken "^8.5.1"
+    uuid "^8.3.0"
+
 "@babel/code-frame@7.0.0":
   version "7.0.0"
   resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
@@ -3372,6 +3390,11 @@
   resolved "https://registry.npmjs.org/@mdx-js/react/-/react-1.5.9.tgz#31873ab097fbe58c61c7585fc0be64e83182b6df"
   integrity sha512-rengdUSedIdIQbXPSeafItCacTYocARAjUA51b6R1KNHmz+59efz7UmyTKr73viJQZ98ouu7iRGmOTtjRrbbWA==
 
+"@microsoft/microsoft-graph-types@^1.25.0":
+  version "1.25.0"
+  resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-1.25.0.tgz#1f543ebc029a115dd1d48a1ae99d7ddd5ee9af57"
+  integrity sha512-RsuA+ROaU3voWzG9TVBkRKxmLatteRGduFDi5p0k3FUHho49rm9SvrA7DUyYbSXLy2xXRx9AnjKM9klYBeKEiQ==
+
 "@mrmlnc/readdir-enhanced@^2.2.1":
   version "2.2.1"
   resolved "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde"