paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 20 Packages Projects Releases Wiki Activity

Remove MicrosoftGraphOrgReaderProcessor from plugin-catalog-backend

Browse Source 
Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
This commit is contained in:
Oliver Sand
2021-06-16 11:14:53 +02:00
parent 160181779b
commit 579cf97f02
16 changed files with 18 additions and 1699 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/config.d.ts
Vendored
-48
View File
@@ -362,54 +362,6 @@ export interface Config {
roleArn?: string;
};
};
/**
* MicrosoftGraphOrgReaderProcessor configuration
*/
microsoftGraphOrg?: {
/**
* The configuration parameters for each single Microsoft Graph provider.
*/
providers: Array<{
/**
* The prefix of the target that this matches on, e.g.
* "https://graph.microsoft.com/v1.0", with no trailing slash.
*/
target: string;
/**
* The auth authority used.
*
* Default value "https://login.microsoftonline.com"
*/
authority?: string;
/**
* The tenant whose org data we are interested in.
*/
tenantId: string;
/**
* The OAuth client ID to use for authenticating requests.
*/
clientId: string;
/**
* The OAuth client secret to use for authenticating requests.
*
* @visibility secret
*/
clientSecret: string;
/**
* The filter to apply to extract users.
*
* E.g. "accountEnabled eq true and userType eq 'member'"
*/
userFilter?: string;
/**
* The filter to apply to extract groups.
*
* E.g. "securityEnabled eq false and mailEnabled eq true"
*/
groupFilter?: string;
}>;
};
};
};
}
plugins/catalog-backend/package.json
-2
View File
@@ -29,7 +29,6 @@
"clean": "backstage-cli clean"
},
"dependencies": {
"@azure/msal-node": "^1.0.0-beta.3",
"@backstage/backend-common": "^0.8.3",
"@backstage/catalog-client": "^0.3.13",
"@backstage/catalog-model": "^0.8.3",
@@ -38,7 +37,6 @@
"@backstage/integration": "^0.5.6",
"@backstage/plugin-search-backend-node": "^0.2.1",
"@backstage/search-common": "^0.1.2",
"@microsoft/microsoft-graph-types": "^1.25.0",
"@octokit/graphql": "^4.5.8",
"@types/express": "^4.17.6",
"@types/ldapjs": "^1.0.9",
plugins/catalog-backend/src/ingestion/processors/MicrosoftGraphOrgReaderProcessor.ts
-110
View File
@@ -1,110 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import {
MicrosoftGraphClient,
MicrosoftGraphProviderConfig,
readMicrosoftGraphConfig,
readMicrosoftGraphOrg,
} from './microsoftGraph';
import * as results from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
// TODO: Remove this deprecated processor, the related code in
// ./microsoftGraph/, and the config section in the future.
/**
* Extracts teams and users out of a the Microsoft Graph API.
*
* @deprecated Use the MicrosoftGraphOrgReaderProcessor from package
* @backstage/plugin-catalog-backend-module-msgraph instead.
*/
export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor {
private readonly providers: MicrosoftGraphProviderConfig[];
private readonly logger: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
return new MicrosoftGraphOrgReaderProcessor({
...options,
providers: c ? readMicrosoftGraphConfig(c) : [],
});
}
constructor(options: {
providers: MicrosoftGraphProviderConfig[];
logger: Logger;
}) {
this.providers = options.providers;
this.logger = options.logger;
}
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean> {
if (location.type !== 'microsoft-graph-org') {
return false;
}
this.logger.warn(
'MicrosoftGraphOrgReaderProcessor from @backstage/plugin-catalog is deprecated and will be removed in the future. Please migrate to the new one from @backstage/plugin-catalog-backend-module-msgraph instead.',
);
const provider = this.providers.find(p =>
location.target.startsWith(p.target),
);
if (!provider) {
throw new Error(
`There is no Microsoft Graph Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
);
}
// Read out all of the raw data
const startTimestamp = Date.now();
this.logger.info('Reading Microsoft Graph users and groups');
// We create a client each time as we need one that matches the specific provider
const client = MicrosoftGraphClient.create(provider);
const { users, groups } = await readMicrosoftGraphOrg(
client,
provider.tenantId,
{
userFilter: provider.userFilter,
groupFilter: provider.groupFilter,
},
);
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.debug(
`Read ${users.length} users and ${groups.length} groups from Microsoft Graph in ${duration} seconds`,
);
// Done!
for (const group of groups) {
emit(results.entity(location, group));
}
for (const user of users) {
emit(results.entity(location, user));
}
return true;
}
}
plugins/catalog-backend/src/ingestion/processors/index.ts
-1
View File
@@ -27,7 +27,6 @@ export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor';
export { LocationEntityProcessor } from './LocationEntityProcessor';
export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';
export { PlaceholderProcessor } from './PlaceholderProcessor';
export type { PlaceholderResolver } from './PlaceholderProcessor';
export { StaticLocationProcessor } from './StaticLocationProcessor';
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.test.ts
-363
View File
@@ -1,363 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as msal from '@azure/msal-node';
import { msw } from '@backstage/test-utils';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { MicrosoftGraphClient } from './client';
describe('MicrosoftGraphClient', () => {
const confidentialClientApplication: jest.Mocked<msal.ConfidentialClientApplication> = {
acquireTokenByClientCredential: jest.fn(),
} as any;
let client: MicrosoftGraphClient;
const worker = setupServer();
msw.setupDefaultHandlers(worker);
beforeEach(() => {
confidentialClientApplication.acquireTokenByClientCredential.mockResolvedValue(
{ token: 'ACCESS_TOKEN' } as any,
);
client = new MicrosoftGraphClient(
'https://example.com',
confidentialClientApplication,
);
});
afterEach(() => {
jest.resetAllMocks();
});
it('should perform raw request', async () => {
worker.use(
rest.get('https://other.example.com/', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ value: 'example' })),
),
);
const response = await client.requestRaw('https://other.example.com/');
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ value: 'example' });
expect(
confidentialClientApplication.acquireTokenByClientCredential,
).toBeCalledTimes(1);
expect(
confidentialClientApplication.acquireTokenByClientCredential,
).toBeCalledWith({ scopes: ['https://graph.microsoft.com/.default'] });
});
it('should perform simple api request', async () => {
worker.use(
rest.get('https://example.com/users', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ value: 'example' })),
),
);
const response = await client.requestApi('users');
expect(response.status).toBe(200);
expect(await response.json()).toEqual({ value: 'example' });
});
it('should perform api request with filter, select and expand', async () => {
worker.use(
rest.get('https://example.com/users', (req, res, ctx) =>
res(ctx.status(200), ctx.json({ queryString: req.url.search })),
),
);
const response = await client.requestApi('users', {
filter: 'test eq true',
expand: ['children'],
select: ['id', 'children'],
});
expect(response.status).toBe(200);
expect(await response.json()).toEqual({
queryString:
'?$filter=test%20eq%20true&$select=id,children&$expand=children',
});
});
it('should perform collection request for a single page', async () => {
worker.use(
rest.get('https://example.com/users', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: ['first'],
}),
),
),
);
const values = await collectAsyncIterable(
client.requestCollection<string>('users'),
);
expect(values).toEqual(['first']);
});
it('should perform collection request for multiple pages', async () => {
worker.use(
rest.get('https://example.com/users', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: ['first'],
'@odata.nextLink': 'https://example.com/users2',
}),
),
),
);
worker.use(
rest.get('https://example.com/users2', (_, res, ctx) =>
res(ctx.status(200), ctx.json({ value: ['second'] })),
),
);
const values = await collectAsyncIterable(
client.requestCollection<string>('users'),
);
expect(values).toEqual(['first', 'second']);
});
it('should load user profile', async () => {
worker.use(
rest.get('https://example.com/users/user-id', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
surname: 'Example',
}),
),
),
);
const userProfile = await client.getUserProfile('user-id');
expect(userProfile).toEqual({ surname: 'Example' });
});
it('should throw expection if load user profile fails', async () => {
worker.use(
rest.get('https://example.com/users/user-id', (_, res, ctx) =>
res(ctx.status(404)),
),
);
await expect(() => client.getUserProfile('user-id')).rejects.toThrowError();
});
it('should load user profile photo with max size of 120', async () => {
worker.use(
rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [
{
height: 120,
id: 120,
},
{
height: 500,
id: 500,
},
],
}),
),
),
);
worker.use(
rest.get(
'https://example.com/users/user-id/photos/120/*',
(_, res, ctx) => res(ctx.status(200), ctx.text('911')),
),
);
const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
expect(photo).toEqual('data:image/jpeg;base64,OTEx');
});
it('should not fail if user has no profile photo', async () => {
worker.use(
rest.get('https://example.com/users/user-id/photos', (_, res, ctx) =>
res(ctx.status(404)),
),
);
const photo = await client.getUserPhotoWithSizeLimit('user-id', 120);
expect(photo).toBeFalsy();
});
it('should load user profile photo', async () => {
worker.use(
rest.get('https://example.com/users/user-id/photo/*', (_, res, ctx) =>
res(ctx.status(200), ctx.text('911')),
),
);
const photo = await client.getUserPhoto('user-id');
expect(photo).toEqual('data:image/jpeg;base64,OTEx');
});
it('should load user profile photo for size 120', async () => {
worker.use(
rest.get(
'https://example.com/users/user-id/photos/120/*',
(_, res, ctx) => res(ctx.status(200), ctx.text('911')),
),
);
const photo = await client.getUserPhoto('user-id', '120');
expect(photo).toEqual('data:image/jpeg;base64,OTEx');
});
it('should load users', async () => {
worker.use(
rest.get('https://example.com/users', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [{ surname: 'Example' }],
}),
),
),
);
const values = await collectAsyncIterable(client.getUsers());
expect(values).toEqual([{ surname: 'Example' }]);
});
it('should load group profile photo with max size of 120', async () => {
worker.use(
rest.get('https://example.com/groups/group-id/photos', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [
{
height: 120,
id: 120,
},
],
}),
),
),
);
worker.use(
rest.get(
'https://example.com/groups/group-id/photos/120/*',
(_, res, ctx) => res(ctx.status(200), ctx.text('911')),
),
);
const photo = await client.getGroupPhotoWithSizeLimit('group-id', 120);
expect(photo).toEqual('data:image/jpeg;base64,OTEx');
});
it('should load group profile photo', async () => {
worker.use(
rest.get('https://example.com/groups/group-id/photo/*', (_, res, ctx) =>
res(ctx.status(200), ctx.text('911')),
),
);
const photo = await client.getGroupPhoto('group-id');
expect(photo).toEqual('data:image/jpeg;base64,OTEx');
});
it('should load groups', async () => {
worker.use(
rest.get('https://example.com/groups', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [{ displayName: 'Example' }],
}),
),
),
);
const values = await collectAsyncIterable(client.getGroups());
expect(values).toEqual([{ displayName: 'Example' }]);
});
it('should load group members', async () => {
worker.use(
rest.get('https://example.com/groups/group-id/members', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
value: [
{ '@odata.type': '#microsoft.graph.user' },
{ '@odata.type': '#microsoft.graph.group' },
],
}),
),
),
);
const values = await collectAsyncIterable(
client.getGroupMembers('group-id'),
);
expect(values).toEqual([
{ '@odata.type': '#microsoft.graph.user' },
{ '@odata.type': '#microsoft.graph.group' },
]);
});
it('should load organization', async () => {
worker.use(
rest.get('https://example.com/organization/tentant-id', (_, res, ctx) =>
res(
ctx.status(200),
ctx.json({
displayName: 'Example',
}),
),
),
);
const organization = await client.getOrganization('tentant-id');
expect(organization).toEqual({ displayName: 'Example' });
});
});
async function collectAsyncIterable<T>(
iterable: AsyncIterable<T>,
): Promise<T[]> {
const values = [];
for await (const value of iterable) {
values.push(value);
}
return values;
}
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts
-235
View File
@@ -1,235 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as msal from '@azure/msal-node';
import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
import fetch from 'cross-fetch';
import qs from 'qs';
import { MicrosoftGraphProviderConfig } from './config';
export type ODataQuery = {
filter?: string;
expand?: string[];
select?: string[];
};
export type GroupMember =
| (MicrosoftGraph.Group & { '@odata.type': '#microsoft.graph.user' })
| (MicrosoftGraph.User & { '@odata.type': '#microsoft.graph.group' });
export class MicrosoftGraphClient {
static create(config: MicrosoftGraphProviderConfig): MicrosoftGraphClient {
const clientConfig: msal.Configuration = {
auth: {
clientId: config.clientId,
clientSecret: config.clientSecret,
authority: `${config.authority}/${config.tenantId}`,
},
};
const pca = new msal.ConfidentialClientApplication(clientConfig);
return new MicrosoftGraphClient(config.target, pca);
}
constructor(
private readonly baseUrl: string,
private readonly pca: msal.ConfidentialClientApplication,
) {}
async *requestCollection<T>(
path: string,
query?: ODataQuery,
): AsyncIterable<T> {
let response = await this.requestApi(path, query);
for (;;) {
if (response.status !== 200) {
await this.handleError(path, response);
}
const result = await response.json();
const elements: T[] = result.value;
yield* elements;
// Follow cursor to the next page if one is available
if (!result['@odata.nextLink']) {
return;
}
response = await this.requestRaw(result['@odata.nextLink']);
}
}
async requestApi(path: string, query?: ODataQuery): Promise<Response> {
const queryString = qs.stringify(
{
$filter: query?.filter,
$select: query?.select?.join(','),
$expand: query?.expand?.join(','),
},
{
addQueryPrefix: true,
// Microsoft Graph doesn't like an encoded query string
encode: false,
},
);
return await this.requestRaw(`${this.baseUrl}/${path}${queryString}`);
}
async requestRaw(url: string): Promise<Response> {
// Make sure that we always have a valid access token (might be cached)
const token = await this.pca.acquireTokenByClientCredential({
scopes: ['https://graph.microsoft.com/.default'],
});
if (!token) {
throw new Error('Error while requesting token for Microsoft Graph');
}
return await fetch(url, {
headers: {
Authorization: `Bearer ${token.accessToken}`,
},
});
}
async getUserProfile(userId: string): Promise<MicrosoftGraph.User> {
const response = await this.requestApi(`users/${userId}`);
if (response.status !== 200) {
await this.handleError('user profile', response);
}
return await response.json();
}
async getUserPhotoWithSizeLimit(
userId: string,
maxSize: number,
): Promise<string | undefined> {
return await this.getPhotoWithSizeLimit('users', userId, maxSize);
}
async getUserPhoto(
userId: string,
sizeId?: string,
): Promise<string | undefined> {
return await this.getPhoto('users', userId, sizeId);
}
async *getUsers(query?: ODataQuery): AsyncIterable<MicrosoftGraph.User> {
yield* this.requestCollection<MicrosoftGraph.User>(`users`, query);
}
async getGroupPhotoWithSizeLimit(
groupId: string,
maxSize: number,
): Promise<string | undefined> {
return await this.getPhotoWithSizeLimit('groups', groupId, maxSize);
}
async getGroupPhoto(
groupId: string,
sizeId?: string,
): Promise<string | undefined> {
return await this.getPhoto('groups', groupId, sizeId);
}
async *getGroups(query?: ODataQuery): AsyncIterable<MicrosoftGraph.Group> {
yield* this.requestCollection<MicrosoftGraph.Group>(`groups`, query);
}
async *getGroupMembers(groupId: string): AsyncIterable<GroupMember> {
yield* this.requestCollection<GroupMember>(`groups/${groupId}/members`);
}
async getOrganization(
tenantId: string,
): Promise<MicrosoftGraph.Organization> {
const response = await this.requestApi(`organization/${tenantId}`);
if (response.status !== 200) {
await this.handleError(`organization/${tenantId}`, response);
}
return await response.json();
}
private async getPhotoWithSizeLimit(
entityName: string,
id: string,
maxSize: number,
): Promise<string | undefined> {
const response = await this.requestApi(`${entityName}/${id}/photos`);
if (response.status === 404) {
return undefined;
} else if (response.status !== 200) {
await this.handleError(`${entityName} photos`, response);
}
const result = await response.json();
const photos = result.value as MicrosoftGraph.ProfilePhoto[];
let selectedPhoto: MicrosoftGraph.ProfilePhoto | undefined = undefined;
// Find the biggest picture that is smaller than the max size
for (const p of photos) {
if (
!selectedPhoto ||
(p.height! >= selectedPhoto.height! && p.height! <= maxSize)
) {
selectedPhoto = p;
}
}
if (!selectedPhoto) {
return undefined;
}
return await this.getPhoto(entityName, id, selectedPhoto.id!);
}
private async getPhoto(
entityName: string,
id: string,
sizeId?: string,
): Promise<string | undefined> {
const path = sizeId
? `${entityName}/${id}/photos/${sizeId}/$value`
: `${entityName}/${id}/photo/$value`;
const response = await this.requestApi(path);
if (response.status === 404) {
return undefined;
} else if (response.status !== 200) {
await this.handleError('photo', response);
}
return `data:image/jpeg;base64,${Buffer.from(
await response.arrayBuffer(),
).toString('base64')}`;
}
private async handleError(path: string, response: Response): Promise<void> {
const result = await response.json();
const error = result.error as MicrosoftGraph.PublicError;
throw new Error(
`Error while reading ${path} from Microsoft Graph: ${error.code} - ${error.message}`,
);
}
}
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.test.ts
-75
View File
@@ -1,75 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readMicrosoftGraphConfig } from './config';
describe('readMicrosoftGraphConfig', () => {
it('applies all of the defaults', () => {
const config = {
providers: [
{
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
},
],
};
const actual = readMicrosoftGraphConfig(new ConfigReader(config));
const expected = [
{
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
authority: 'https://login.microsoftonline.com',
userFilter: undefined,
groupFilter: undefined,
},
];
expect(actual).toEqual(expected);
});
it('reads all the values', () => {
const config = {
providers: [
{
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
authority: 'https://login.example.com/',
userFilter: 'accountEnabled eq true',
groupFilter: 'securityEnabled eq false',
},
],
};
const actual = readMicrosoftGraphConfig(new ConfigReader(config));
const expected = [
{
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
authority: 'https://login.example.com',
userFilter: 'accountEnabled eq true',
groupFilter: 'securityEnabled eq false',
},
];
expect(actual).toEqual(expected);
});
});
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/config.ts
-91
View File
@@ -1,91 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
/**
* The configuration parameters for a single Microsoft Graph provider.
*/
export type MicrosoftGraphProviderConfig = {
/**
* The prefix of the target that this matches on, e.g.
* "https://graph.microsoft.com/v1.0", with no trailing slash.
*/
target: string;
/**
* The auth authority used.
*
* E.g. "https://login.microsoftonline.com"
*/
authority?: string;
/**
* The tenant whose org data we are interested in.
*/
tenantId: string;
/**
* The OAuth client ID to use for authenticating requests.
*/
clientId: string;
/**
* The OAuth client secret to use for authenticating requests.
*
* @visibility secret
*/
clientSecret: string;
/**
* The filter to apply to extract users.
*
* E.g. "accountEnabled eq true and userType eq 'member'"
*/
userFilter?: string;
/**
* The filter to apply to extract groups.
*
* E.g. "securityEnabled eq false and mailEnabled eq true"
*/
groupFilter?: string;
};
export function readMicrosoftGraphConfig(
config: Config,
): MicrosoftGraphProviderConfig[] {
const providers: MicrosoftGraphProviderConfig[] = [];
const providerConfigs = config.getOptionalConfigArray('providers') ?? [];
for (const providerConfig of providerConfigs) {
const target = providerConfig.getString('target').replace(/\/+$/, '');
const authority =
providerConfig.getOptionalString('authority')?.replace(/\/+$/, '') ||
'https://login.microsoftonline.com';
const tenantId = providerConfig.getString('tenantId');
const clientId = providerConfig.getString('clientId');
const clientSecret = providerConfig.getString('clientSecret');
const userFilter = providerConfig.getOptionalString('userFilter');
const groupFilter = providerConfig.getOptionalString('groupFilter');
providers.push({
target,
authority,
tenantId,
clientId,
clientSecret,
userFilter,
groupFilter,
});
}
return providers;
}
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/constants.ts
-32
View File
@@ -1,32 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* The tenant id used by the Microsoft Graph API
*/
export const MICROSOFT_GRAPH_TENANT_ID_ANNOTATION =
'graph.microsoft.com/tenant-id';
/**
* The group id used by the Microsoft Graph API
*/
export const MICROSOFT_GRAPH_GROUP_ID_ANNOTATION =
'graph.microsoft.com/group-id';
/**
* The user id used by the Microsoft Graph API
*/
export const MICROSOFT_GRAPH_USER_ID_ANNOTATION = 'graph.microsoft.com/user-id';
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/index.ts
-24
View File
@@ -1,24 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { MicrosoftGraphClient } from './client';
export type { MicrosoftGraphProviderConfig } from './config';
export { readMicrosoftGraphConfig } from './config';
export { readMicrosoftGraphOrg } from './read';
export {
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
MICROSOFT_GRAPH_USER_ID_ANNOTATION,
} from './constants';
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.test.ts
-347
View File
@@ -1,347 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import merge from 'lodash/merge';
import { RecursivePartial } from '../../../util';
import { GroupMember, MicrosoftGraphClient } from './client';
import {
normalizeEntityName,
readMicrosoftGraphGroups,
readMicrosoftGraphOrganization,
readMicrosoftGraphUsers,
resolveRelations,
} from './read';
function user(data: RecursivePartial<UserEntity>): UserEntity {
return merge(
{},
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name: 'name' },
spec: { profile: {}, memberOf: [] },
} as UserEntity,
data,
);
}
function group(data: RecursivePartial<GroupEntity>): GroupEntity {
return merge(
{},
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: 'name',
},
spec: {
children: [],
type: 'team',
},
} as GroupEntity,
data,
);
}
describe('read microsoft graph', () => {
const client: jest.Mocked<MicrosoftGraphClient> = {
getUsers: jest.fn(),
getGroups: jest.fn(),
getGroupMembers: jest.fn(),
getUserPhotoWithSizeLimit: jest.fn(),
getGroupPhotoWithSizeLimit: jest.fn(),
getOrganization: jest.fn(),
} as any;
afterEach(() => jest.resetAllMocks());
describe('normalizeEntityName', () => {
it('should normalize name to valid entity name', () => {
expect(normalizeEntityName('User Name')).toBe('user_name');
});
it('should normalize e-mail to valid entity name', () => {
expect(normalizeEntityName('user.name@example.com')).toBe(
'user.name_example.com',
);
});
});
describe('readMicrosoftGraphUsers', () => {
it('should read users', async () => {
async function* getExampleUsers() {
yield {
id: 'userid',
displayName: 'User Name',
mail: 'user.name@example.com',
};
}
client.getUsers.mockImplementation(getExampleUsers);
client.getUserPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
const { users } = await readMicrosoftGraphUsers(client, {
userFilter: 'accountEnabled eq true',
});
expect(users).toEqual([
user({
metadata: {
annotations: {
'graph.microsoft.com/user-id': 'userid',
},
name: 'user.name_example.com',
},
spec: {
profile: {
displayName: 'User Name',
email: 'user.name@example.com',
picture: 'data:image/jpeg;base64,...',
},
},
}),
]);
expect(client.getUsers).toBeCalledTimes(1);
expect(client.getUsers).toBeCalledWith({
filter: 'accountEnabled eq true',
select: ['id', 'displayName', 'mail'],
});
expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1);
expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120);
});
});
describe('readMicrosoftGraphOrganization', () => {
it('should read organization', async () => {
client.getOrganization.mockResolvedValue({
id: 'tenantid',
displayName: 'Organization Name',
});
const { rootGroup } = await readMicrosoftGraphOrganization(
client,
'tenantid',
);
expect(rootGroup).toEqual(
group({
metadata: {
annotations: {
'graph.microsoft.com/tenant-id': 'tenantid',
},
name: 'organization_name',
description: 'Organization Name',
},
spec: {
type: 'root',
profile: {
displayName: 'Organization Name',
},
},
}),
);
expect(client.getOrganization).toBeCalledTimes(1);
expect(client.getOrganization).toBeCalledWith('tenantid');
});
});
describe('readMicrosoftGraphGroups', () => {
it('should read groups', async () => {
async function* getExampleGroups() {
yield {
id: 'groupid',
displayName: 'Group Name',
description: 'Group Description',
mail: 'group@example.com',
};
}
async function* getExampleGroupMembers(): AsyncIterable<GroupMember> {
yield {
'@odata.type': '#microsoft.graph.group',
id: 'childgroupid',
};
yield {
'@odata.type': '#microsoft.graph.user',
id: 'userid',
};
}
client.getGroups.mockImplementation(getExampleGroups);
client.getGroupMembers.mockImplementation(getExampleGroupMembers);
client.getOrganization.mockResolvedValue({
id: 'tenantid',
displayName: 'Organization Name',
});
client.getGroupPhotoWithSizeLimit.mockResolvedValue(
'data:image/jpeg;base64,...',
);
const {
groups,
groupMember,
groupMemberOf,
rootGroup,
} = await readMicrosoftGraphGroups(client, 'tenantid', {
groupFilter: 'securityEnabled eq false',
});
const expectedRootGroup = group({
metadata: {
annotations: {
'graph.microsoft.com/tenant-id': 'tenantid',
},
name: 'organization_name',
description: 'Organization Name',
},
spec: {
type: 'root',
profile: {
displayName: 'Organization Name',
},
},
});
expect(groups).toEqual([
expectedRootGroup,
group({
metadata: {
annotations: {
'graph.microsoft.com/group-id': 'groupid',
},
name: 'group_name',
description: 'Group Description',
},
spec: {
type: 'team',
profile: {
displayName: 'Group Name',
email: 'group@example.com',
// TODO: Loading groups doesn't work right now as Microsoft Graph
// doesn't allows this yet
/* picture: 'data:image/jpeg;base64,...',*/
},
},
}),
]);
expect(rootGroup).toEqual(expectedRootGroup);
expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid']));
expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid']));
expect(groupMember.get('organization_name')).toEqual(new Set());
expect(client.getGroups).toBeCalledTimes(1);
expect(client.getGroups).toBeCalledWith({
filter: 'securityEnabled eq false',
select: ['id', 'displayName', 'description', 'mail', 'mailNickname'],
});
expect(client.getGroupMembers).toBeCalledTimes(1);
expect(client.getGroupMembers).toBeCalledWith('groupid');
// TODO: Loading groups doesn't work right now as Microsoft Graph
// doesn't allows this yet
// expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1);
// expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120);
});
});
describe('resolveRelations', () => {
it('should resolve relations', async () => {
const rootGroup = group({
metadata: {
annotations: {
'graph.microsoft.com/tenant-id': 'tenant-id-root',
},
name: 'root',
},
spec: {
type: 'root',
},
});
const groupA = group({
metadata: {
annotations: {
'graph.microsoft.com/group-id': 'group-id-a',
},
name: 'a',
},
});
const groupB = group({
metadata: {
annotations: {
'graph.microsoft.com/group-id': 'group-id-b',
},
name: 'b',
},
});
const groupC = group({
metadata: {
annotations: {
'graph.microsoft.com/group-id': 'group-id-c',
},
name: 'c',
},
});
const user1 = user({
metadata: {
annotations: {
'graph.microsoft.com/user-id': 'user-id-1',
},
name: 'user1',
},
});
const user2 = user({
metadata: {
annotations: {
'graph.microsoft.com/user-id': 'user-id-2',
},
name: 'user2',
},
});
const groups = [rootGroup, groupA, groupB, groupC];
const users = [user1, user2];
const groupMember = new Map<string, Set<string>>();
groupMember.set('group-id-b', new Set(['group-id-c']));
const groupMemberOf = new Map<string, Set<string>>();
groupMemberOf.set('user-id-1', new Set(['group-id-a']));
groupMemberOf.set('user-id-2', new Set(['group-id-c']));
// We have a root groups
// We have three groups: a, b, c. c is child of b
// we have two users: u1, u2. u1 is member of a, u2 is member of c
resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
expect(rootGroup.spec.parent).toBeUndefined();
expect(rootGroup.spec.children).toEqual(
expect.arrayContaining(['a', 'b']),
);
expect(groupA.spec.parent).toEqual('root');
expect(groupA.spec.children).toEqual(expect.arrayContaining([]));
expect(groupB.spec.parent).toEqual('root');
expect(groupB.spec.children).toEqual(expect.arrayContaining(['c']));
expect(groupC.spec.parent).toEqual('b');
expect(groupC.spec.children).toEqual(expect.arrayContaining([]));
expect(user1.spec.memberOf).toEqual(expect.arrayContaining(['a']));
expect(user2.spec.memberOf).toEqual(expect.arrayContaining(['b', 'c']));
});
});
});
plugins/catalog-backend/src/ingestion/processors/microsoftGraph/read.ts
-363
View File
@@ -1,363 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import limiterFactory from 'p-limit';
import { buildMemberOf, buildOrgHierarchy } from '../util/org';
import { MicrosoftGraphClient } from './client';
import {
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
MICROSOFT_GRAPH_USER_ID_ANNOTATION,
} from './constants';
export function normalizeEntityName(name: string): string {
return name
.trim()
.toLocaleLowerCase()
.replace(/[^a-zA-Z0-9_\-\.]/g, '_');
}
export async function readMicrosoftGraphUsers(
client: MicrosoftGraphClient,
options?: { userFilter?: string },
): Promise<{
users: UserEntity[]; // With all relations empty
}> {
const entities: UserEntity[] = [];
const promises: Promise<void>[] = [];
const limiter = limiterFactory(10);
for await (const user of client.getUsers({
filter: options?.userFilter,
select: ['id', 'displayName', 'mail'],
})) {
if (!user.id || !user.displayName || !user.mail) {
continue;
}
const name = normalizeEntityName(user.mail);
const entity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name,
annotations: {
[MICROSOFT_GRAPH_USER_ID_ANNOTATION]: user.id!,
},
},
spec: {
profile: {
displayName: user.displayName!,
email: user.mail!,
// TODO: Additional fields?
// jobTitle: user.jobTitle || undefined,
// officeLocation: user.officeLocation || undefined,
// mobilePhone: user.mobilePhone || undefined,
},
memberOf: [],
},
};
// Download the photos in parallel, otherwise it can take quite some time
const loadPhoto = limiter(async () => {
entity.spec.profile!.picture = await client.getUserPhotoWithSizeLimit(
user.id!,
// We are limiting the photo size, as users with full resolution photos
// can make the Backstage API slow
120,
);
});
promises.push(loadPhoto);
entities.push(entity);
}
// Wait for all photos to be downloaded
await Promise.all(promises);
return { users: entities };
}
export async function readMicrosoftGraphOrganization(
client: MicrosoftGraphClient,
tenantId: string,
): Promise<{
rootGroup: GroupEntity; // With all relations empty
}> {
// For now we expect a single root organization
const organization = await client.getOrganization(tenantId);
const name = normalizeEntityName(organization.displayName!);
const rootGroup: GroupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: name,
description: organization.displayName!,
annotations: {
[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]: organization.id!,
},
},
spec: {
type: 'root',
profile: {
displayName: organization.displayName!,
},
children: [],
},
};
return { rootGroup };
}
export async function readMicrosoftGraphGroups(
client: MicrosoftGraphClient,
tenantId: string,
options?: { groupFilter?: string },
): Promise<{
groups: GroupEntity[]; // With all relations empty
rootGroup: GroupEntity | undefined; // With all relations empty
groupMember: Map<string, Set<string>>;
groupMemberOf: Map<string, Set<string>>;
}> {
const groups: GroupEntity[] = [];
const groupMember: Map<string, Set<string>> = new Map();
const groupMemberOf: Map<string, Set<string>> = new Map();
const limiter = limiterFactory(10);
const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId);
groupMember.set(rootGroup.metadata.name, new Set<string>());
groups.push(rootGroup);
const promises: Promise<void>[] = [];
for await (const group of client.getGroups({
filter: options?.groupFilter,
select: ['id', 'displayName', 'description', 'mail', 'mailNickname'],
})) {
if (!group.id || !group.displayName) {
continue;
}
const name = normalizeEntityName(group.mailNickname || group.displayName);
const entity: GroupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: name,
annotations: {
[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]: group.id,
},
},
spec: {
type: 'team',
profile: {},
children: [],
},
};
if (group.description) {
entity.metadata.description = group.description;
}
if (group.displayName) {
entity.spec.profile!.displayName = group.displayName;
}
if (group.mail) {
entity.spec.profile!.email = group.mail;
}
// Download the members in parallel, otherwise it can take quite some time
const loadGroupMembers = limiter(async () => {
for await (const member of client.getGroupMembers(group.id!)) {
if (!member.id) {
continue;
}
if (member['@odata.type'] === '#microsoft.graph.user') {
ensureItem(groupMemberOf, member.id, group.id!);
}
if (member['@odata.type'] === '#microsoft.graph.group') {
ensureItem(groupMember, group.id!, member.id);
}
}
});
// TODO: Loading groups doesn't work right now as Microsoft Graph doesn't
// allows this yet: https://microsoftgraph.uservoice.com/forums/920506-microsoft-graph-feature-requests/suggestions/37884922-allow-application-to-set-or-update-a-group-s-photo
/*/ / Download the photos in parallel, otherwise it can take quite some time
const loadPhoto = limiter(async () => {
entity.spec.profile!.picture = await client.getGroupPhotoWithSizeLimit(
group.id!,
// We are limiting the photo size, as groups with full resolution photos
// can make the Backstage API slow
120,
);
});
promises.push(loadPhoto);*/
promises.push(loadGroupMembers);
groups.push(entity);
}
// Wait for all group members and photos to be loaded
await Promise.all(promises);
return {
groups,
rootGroup,
groupMember,
groupMemberOf,
};
}
export function resolveRelations(
rootGroup: GroupEntity | undefined,
groups: GroupEntity[],
users: UserEntity[],
groupMember: Map<string, Set<string>>,
groupMemberOf: Map<string, Set<string>>,
) {
// Build reference lookup tables, we reference them by the id the the graph
const groupMap: Map<string, GroupEntity> = new Map(); // by group-id or tenant-id
for (const group of groups) {
if (group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION]) {
groupMap.set(
group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION],
group,
);
}
if (group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION]) {
groupMap.set(
group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION],
group,
);
}
}
// Resolve all member relationships into the reverse direction
const parentGroups = new Map<string, Set<string>>();
groupMember.forEach((members, groupId) =>
members.forEach(m => ensureItem(parentGroups, m, groupId)),
);
// Make sure every group (except root) has at least one parent. If the parent is missing, add the root.
if (rootGroup) {
const tenantId = rootGroup.metadata.annotations![
MICROSOFT_GRAPH_TENANT_ID_ANNOTATION
];
groups.forEach(group => {
const groupId = group.metadata.annotations![
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION
];
if (!groupId) {
return;
}
if (retrieveItems(parentGroups, groupId).size === 0) {
ensureItem(parentGroups, groupId, tenantId);
ensureItem(groupMember, tenantId, groupId);
}
});
}
groups.forEach(group => {
const id =
group.metadata.annotations![MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ??
group.metadata.annotations![MICROSOFT_GRAPH_TENANT_ID_ANNOTATION];
retrieveItems(groupMember, id).forEach(m => {
const childGroup = groupMap.get(m);
if (childGroup) {
group.spec.children.push(childGroup.metadata.name);
}
});
retrieveItems(parentGroups, id).forEach(p => {
const parentGroup = groupMap.get(p);
if (parentGroup) {
// TODO: Only having a single parent group might not match every companies model, but fine for now.
group.spec.parent = parentGroup.metadata.name;
}
});
});
// Make sure that all groups have proper parents and children
buildOrgHierarchy(groups);
// Set relations for all users
users.forEach(user => {
const id = user.metadata.annotations![MICROSOFT_GRAPH_USER_ID_ANNOTATION];
retrieveItems(groupMemberOf, id).forEach(p => {
const parentGroup = groupMap.get(p);
if (parentGroup) {
user.spec.memberOf.push(parentGroup.metadata.name);
}
});
});
// Make sure all transitive memberships are available
buildMemberOf(groups, users);
}
export async function readMicrosoftGraphOrg(
client: MicrosoftGraphClient,
tenantId: string,
options?: { userFilter?: string; groupFilter?: string },
): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> {
const { users } = await readMicrosoftGraphUsers(client, {
userFilter: options?.userFilter,
});
const {
groups,
rootGroup,
groupMember,
groupMemberOf,
} = await readMicrosoftGraphGroups(client, tenantId, {
groupFilter: options?.groupFilter,
});
resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf);
users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
return { users, groups };
}
function ensureItem(
target: Map<string, Set<string>>,
key: string,
value: string,
) {
let set = target.get(key);
if (!set) {
set = new Set();
target.set(key, set);
}
set!.add(value);
}
function retrieveItems(
target: Map<string, Set<string>>,
key: string,
): Set<string> {
return target.get(key) ?? new Set();
}
plugins/catalog-backend/src/next/NextCatalogBuilder.ts
-2
View File
@@ -50,7 +50,6 @@ import {
GithubDiscoveryProcessor,
GithubOrgReaderProcessor,
LdapOrgReaderProcessor,
MicrosoftGraphOrgReaderProcessor,
PlaceholderProcessor,
PlaceholderResolver,
UrlReaderProcessor,
@@ -373,7 +372,6 @@ export class NextCatalogBuilder {
GithubDiscoveryProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
new AnnotateLocationEntityProcessor({ integrations }),
plugins/catalog-backend/src/service/CatalogBuilder.ts
-2
View File
@@ -51,7 +51,6 @@ import {
LdapOrgReaderProcessor,
LocationEntityProcessor,
LocationReaders,
MicrosoftGraphOrgReaderProcessor,
PlaceholderProcessor,
PlaceholderResolver,
StaticLocationProcessor,
@@ -319,7 +318,6 @@ export class CatalogBuilder {
GithubDiscoveryProcessor.fromConfig(config, { logger }),
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
MicrosoftGraphOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
CodeOwnersProcessor.fromConfig(config, { logger, reader }),
new LocationEntityProcessor({ integrations }),