paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

feat(catalog): add an ldap processor

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-10-02 13:08:54 +02:00
parent 2d73c037a0
commit b9c8062c08
22 changed files with 1782 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+17
View File
@@ -108,6 +108,23 @@ catalog:
# apiBaseUrl: https://ghe.example.net/api/v3
# token:
# $env: GHE_PRIVATE_TOKEN
ldapOrg:
### Example for how to add your enterprise LDAP server
# providers:
# - target: ldaps://ds.example.net
# bind:
# dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net
# secret: { $secret: { env: LDAP_SECRET } }
# users:
# dn: ou=people,ou=example,dc=example,dc=net
# options:
# filter: (uid=*)
# map:
# description: l
# groups:
# dn: ou=access,ou=groups,ou=example,dc=example,dc=net
# options:
# filter: (&(objectClass=some-group-class)(!(groupType=email)))
locations:
# Backstage example components
docs/features/software-catalog/well-known-annotations.md
+15
View File
@@ -179,6 +179,21 @@ fallback (`rollbar.organization` followed by `organization.name`).
Specifying this annotation may enable Rollbar related features in Backstage for
that entity.
### backstage.io/ldap-rdn, backstage.io/ldap-uuid, backstage.io/ldap-dn
```yaml
# Example:
metadata:
annotations:
backstage.io/ldap-rdn: my-team
backstage.io/ldap-uuid: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21
backstage.io/ldap-dn: cn=my-team,ou=access,ou=groups,ou=spotify,dc=spotify,dc=net
```
The value of these annotations are the corresponding attributes that were found
when ingestion the entity from LDAP. Not all of them may be present, depending
on what attributes that the server presented at ingestion time.
## Deprecated Annotations
The following annotations are deprecated, and only listed here to aid in
packages/catalog-model/src/entity/ref.test.ts
+21
View File
@@ -15,6 +15,7 @@
*/
import { ENTITY_DEFAULT_NAMESPACE } from './constants';
import { Entity } from './Entity';
import { parseEntityName, parseEntityRef, serializeEntityRef } from './ref';
describe('ref', () => {
@@ -333,6 +334,26 @@ describe('ref', () => {
expect(serializeEntityRef({ name: 'c' })).toEqual('c');
});
it('handles entities', () => {
const entityWithNamespace: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
namespace: 'd',
},
};
const entityWithoutNamespace: Entity = {
apiVersion: 'a',
kind: 'b',
metadata: {
name: 'c',
},
};
expect(serializeEntityRef(entityWithNamespace)).toEqual('b:d/c');
expect(serializeEntityRef(entityWithoutNamespace)).toEqual('b:c');
});
it('picks the least complex form', () => {
expect(
serializeEntityRef({ kind: 'a', namespace: 'b', name: 'c' }),
packages/catalog-model/src/entity/ref.ts
+23 -6
View File
@@ -160,12 +160,29 @@ export function parseEntityRef(
* @param ref The reference to serialize
* @returns The same reference on either string or compound form
*/
export function serializeEntityRef(ref: {
kind?: string;
namespace?: string;
name: string;
}): EntityRef {
const { kind, namespace, name } = ref;
export function serializeEntityRef(
ref:
| Entity
| {
kind?: string;
namespace?: string;
name: string;
},
): EntityRef {
let kind;
let namespace;
let name;
if ('apiVersion' in ref) {
kind = ref.kind;
namespace = ref.metadata.namespace;
name = ref.metadata.name;
} else {
kind = ref.kind;
namespace = ref.namespace;
name = ref.name;
}
if (
kind?.includes(':') ||
kind?.includes('/') ||
plugins/catalog-backend/package.json
+2
View File
@@ -33,6 +33,7 @@
"fs-extra": "^9.0.0",
"git-url-parse": "^11.2.0",
"knex": "^0.21.1",
"ldapjs": "^2.2.0",
"lodash": "^4.17.15",
"morgan": "^1.10.0",
"node-fetch": "^2.6.0",
@@ -47,6 +48,7 @@
"@backstage/cli": "^0.1.1-alpha.23",
"@types/core-js": "^2.5.4",
"@types/git-url-parse": "^9.0.0",
"@types/ldapjs": "^1.0.9",
"@types/lodash": "^4.14.151",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
+13 -14
View File
@@ -21,6 +21,7 @@ import {
getEntityName,
Location,
LocationSpec,
serializeEntityRef,
} from '@backstage/catalog-model';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
@@ -159,28 +160,23 @@ export class HigherOrderOperations implements HigherOrderOperation {
);
}
this.logger.info(
`Read ${readerOutput.entities.length} entities from location ${location.type} ${location.target}`,
);
const startTimestamp = Date.now();
for (const item of readerOutput.entities) {
const { entity } = item;
this.logger.debug(
`Read entity kind="${entity.kind}" namespace="${
entity.metadata.namespace || ''
}" name="${entity.metadata.name}"`,
);
try {
const previous = await this.entitiesCatalog.entityByName(
getEntityName(entity),
);
if (!previous) {
this.logger.debug(`No such entity found, adding`);
await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
} else if (entityHasChanges(previous, entity)) {
this.logger.debug(`Different from existing entity, updating`);
await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
} else {
this.logger.debug(`Equal to existing entity, skipping update`);
}
await this.locationsCatalog.logUpdateSuccess(
@@ -188,10 +184,8 @@ export class HigherOrderOperations implements HigherOrderOperation {
entity.metadata.name,
);
} catch (error) {
this.logger.debug(
`Failed refresh of entity kind="${entity.kind}" name="${
entity.metadata.name
}" namespace="${entity.metadata.namespace || ''}", ${error}`,
this.logger.info(
`Failed refresh of entity ${serializeEntityRef(entity)}, ${error}`,
);
await this.locationsCatalog.logUpdateFailure(
@@ -201,5 +195,10 @@ export class HigherOrderOperations implements HigherOrderOperation {
);
}
}
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.info(
`Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${duration} seconds`,
);
}
}
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+4 -14
View File
@@ -29,15 +29,16 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn
import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor';
import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor';
import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
import { FileReaderProcessor } from './processors/FileReaderProcessor';
import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
import { LdapOrgReaderProcessor } from './processors/LdapOrgReaderProcessor';
import { LocationRefProcessor } from './processors/LocationEntityProcessor';
import { PlaceholderProcessor } from './processors/PlaceholderProcessor';
import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor';
import * as result from './processors/results';
import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
import {
@@ -120,7 +121,8 @@ export class LocationReaders implements LocationReader {
StaticLocationProcessor.fromConfig(config),
new FileReaderProcessor(),
...oldProcessors,
GithubOrgReaderProcessor.fromConfig(config),
GithubOrgReaderProcessor.fromConfig(config, logger),
LdapOrgReaderProcessor.fromConfig(config, logger),
new UrlReaderProcessor(options),
new YamlProcessor(),
PlaceholderProcessor.default(),
@@ -199,10 +201,6 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorLocationResult,
emit: LocationProcessorEmit,
) {
this.logger.debug(
`Reading location ${item.location.type} ${item.location.target} optional=${item.optional}`,
);
for (const processor of this.processors) {
if (processor.readLocation) {
try {
@@ -228,10 +226,6 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorDataResult,
emit: LocationProcessorEmit,
) {
this.logger.debug(
`Parsing data from location ${item.location.type} ${item.location.target} (${item.data.byteLength} bytes)`,
);
for (const processor of this.processors) {
if (processor.parseData) {
try {
@@ -254,10 +248,6 @@ export class LocationReaders implements LocationReader {
item: LocationProcessorEntityResult,
emit: LocationProcessorEmit,
): Promise<Entity> {
this.logger.debug(
`Got entity at location ${item.location.type} ${item.location.target}, ${item.entity.apiVersion} ${item.entity.kind}`,
);
let current = item.entity;
for (const processor of this.processors) {
plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts
+19 -6
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { LocationSpec } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import {
@@ -94,9 +95,15 @@ describe('GithubOrgReaderProcessor', () => {
describe('implementation', () => {
it('rejects unknown types', async () => {
const processor = new GithubOrgReaderProcessor([
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
]);
const processor = new GithubOrgReaderProcessor(
[
{
target: 'https://github.com',
apiBaseUrl: 'https://api.github.com',
},
],
getVoidLogger(),
);
const location: LocationSpec = {
type: 'not-github-org',
target: 'https://github.com',
@@ -107,9 +114,15 @@ describe('GithubOrgReaderProcessor', () => {
});
it('rejects unknown targets', async () => {
const processor = new GithubOrgReaderProcessor([
{ target: 'https://github.com', apiBaseUrl: 'https://api.github.com' },
]);
const processor = new GithubOrgReaderProcessor(
[
{
target: 'https://github.com',
apiBaseUrl: 'https://api.github.com',
},
],
getVoidLogger(),
);
const location: LocationSpec = {
type: 'github-org',
target: 'https://not.github.com/apa',
plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts
+29 -4
View File
@@ -17,6 +17,7 @@
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { graphql } from '@octokit/graphql';
import { Logger } from 'winston';
import * as results from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { getOrganizationTeams, getOrganizationUsers } from './util/github';
@@ -26,11 +27,14 @@ import { buildOrgHierarchy } from './util/org';
* Extracts teams and users out of a GitHub org.
*/
export class GithubOrgReaderProcessor implements LocationProcessor {
static fromConfig(config: Config) {
return new GithubOrgReaderProcessor(readConfig(config));
static fromConfig(config: Config, logger: Logger) {
return new GithubOrgReaderProcessor(readConfig(config), logger);
}
constructor(private readonly providers: ProviderConfig[]) {}
constructor(
private readonly providers: ProviderConfig[],
private readonly logger: Logger,
) {}
async readLocation(
location: LocationSpec,
@@ -59,13 +63,34 @@ export class GithubOrgReaderProcessor implements LocationProcessor {
},
});
// Read out all of the raw data
const startTimestamp = Date.now();
this.logger.info('Reading GitHub users and groups');
const { users } = await getOrganizationUsers(client, org);
const { groups, groupMemberUsers } = await getOrganizationTeams(
client,
org,
);
buildOrgHierarchy(groups, users, groupMemberUsers);
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.debug(
`Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`,
);
// Fill out the hierarchy
const usersByName = new Map(users.map(u => [u.metadata.name, u]));
for (const [groupName, userNames] of groupMemberUsers.entries()) {
for (const userName of userNames) {
const user = usersByName.get(userName);
if (user && !user.spec.memberOf.includes(groupName)) {
user.spec.memberOf.push(groupName);
}
}
}
buildOrgHierarchy(groups);
// Done!
for (const group of groups) {
emit(results.entity(location, group));
}
plugins/catalog-backend/src/ingestion/processors/LdapOrgReaderProcessor.ts
+88
View File
@@ -0,0 +1,88 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import {
LdapClient,
LdapProviderConfig,
readLdapConfig,
readLdapOrg,
} from './ldap';
import * as results from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
/**
* Extracts teams and users out of an LDAP server.
*/
export class LdapOrgReaderProcessor implements LocationProcessor {
static fromConfig(config: Config, logger: Logger) {
const c = config.getOptionalConfig('catalog.processors.ldapOrg');
return new LdapOrgReaderProcessor(c ? readLdapConfig(c) : [], logger);
}
constructor(
private readonly providers: LdapProviderConfig[],
private readonly logger: Logger,
) {}
async readLocation(
location: LocationSpec,
_optional: boolean,
emit: LocationProcessorEmit,
): Promise<boolean> {
if (location.type !== 'ldap-org') {
return false;
}
const provider = this.providers.find(p => location.target === p.target);
if (!provider) {
throw new Error(
`There is no LDAP Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.ldapOrg.providers.`,
);
}
// Read out all of the raw data
const startTimestamp = Date.now();
this.logger.info('Reading LDAP users and groups');
// Be lazy and create the client each time; even though it's pretty
// inefficient, we usually only do this once per entire refresh loop and
// don't have to worry about timeouts and reconnects etc.
const client = await LdapClient.create(provider.target, provider.bind);
const { users, groups } = await readLdapOrg(
client,
provider.users,
provider.groups,
);
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.debug(
`Read ${users.length} LDAP users and ${groups.length} LDAP groups in ${duration} seconds`,
);
// Done!
for (const group of groups) {
emit(results.entity(location, group));
}
for (const user of users) {
emit(results.entity(location, user));
}
return true;
}
}
plugins/catalog-backend/src/ingestion/processors/ldap/client.ts
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import ldap, { Client, SearchEntry, SearchOptions } from 'ldapjs';
import { BindConfig } from './config';
import { errorString } from './util';
/**
* Basic wrapper for the ldapjs library.
*
* Helps out with promisifying calls, paging, binding etc.
*/
export class LdapClient {
static create(target: string, bind?: BindConfig): Promise<LdapClient> {
return new Promise<LdapClient>((resolve, reject) => {
const client = ldap.createClient({ url: target });
if (!bind) {
resolve(new LdapClient(client));
return;
}
const { dn, secret } = bind;
client.bind(dn, secret, err => {
if (err) {
reject(`LDAP bind failed for ${dn}, ${errorString(err)}`);
} else {
resolve(new LdapClient(client));
}
});
});
}
constructor(private readonly client: Client) {}
/**
* Performs an LDAP search operation.
*
* @param dn The fully qualified base DN to search within
* @param options The search options
*/
async search(dn: string, options: SearchOptions): Promise<SearchEntry[]> {
try {
return await new Promise<SearchEntry[]>((resolve, reject) => {
const output: SearchEntry[] = [];
this.client.search(dn, options, (err, res) => {
if (err) {
reject(errorString(err));
return;
}
res.on('searchReference', () => {
reject('Unable to handle referral');
});
res.on('searchEntry', entry => {
output.push(entry);
});
res.on('error', e => {
reject(errorString(e));
});
res.on('end', r => {
if (!r) {
reject('Null response');
} else if (r.status !== 0) {
reject(`Got status ${r.status}: ${r.errorMessage}`);
} else {
resolve(output);
}
});
});
});
} catch (e) {
throw new Error(`LDAP search at ${dn} failed, ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/processors/ldap/config.test.ts
+172
View File
@@ -0,0 +1,172 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readLdapConfig } from './config';
describe('readLdapConfig', () => {
it('applies all of the defaults', () => {
const config = {
providers: [
{
target: 'target',
users: {
dn: 'udn',
},
groups: {
dn: 'gdn',
},
},
],
};
const actual = readLdapConfig(
ConfigReader.fromConfigs([{ context: '', data: config }]),
);
const expected = [
{
target: 'target',
bind: undefined,
users: {
dn: 'udn',
options: {
scope: 'one',
attributes: ['*', '+'],
},
set: undefined,
map: {
rdn: 'uid',
name: 'uid',
displayName: 'cn',
email: 'mail',
memberOf: 'memberOf',
},
},
groups: {
dn: 'gdn',
options: {
scope: 'one',
attributes: ['*', '+'],
},
set: undefined,
map: {
rdn: 'cn',
name: 'cn',
description: 'description',
type: 'groupType',
memberOf: 'memberOf',
members: 'member',
},
},
},
];
expect(actual).toEqual(expected);
});
it('reads all the values', () => {
const config = {
providers: [
{
target: 'target',
bind: { dn: 'bdn', secret: 's' },
users: {
dn: 'udn',
options: {
scope: 'base',
attributes: ['*'],
filter: 'f',
paged: true,
},
set: [{ path: 'p', value: 'v' }],
map: {
rdn: 'u',
name: 'v',
description: 'd',
displayName: 'c',
email: 'm',
picture: 'p',
memberOf: 'm',
},
},
groups: {
dn: 'gdn',
options: {
scope: 'base',
attributes: ['*'],
filter: 'f',
paged: true,
},
set: [{ path: 'p', value: 'v' }],
map: {
rdn: 'u',
name: 'v',
description: 'd',
type: 't',
memberOf: 'm',
members: 'n',
},
},
},
],
};
const actual = readLdapConfig(
ConfigReader.fromConfigs([{ context: '', data: config }]),
);
const expected = [
{
target: 'target',
bind: { dn: 'bdn', secret: 's' },
users: {
dn: 'udn',
options: {
scope: 'base',
attributes: ['*'],
filter: 'f',
paged: true,
},
set: [{ path: 'p', value: 'v' }],
map: {
rdn: 'u',
name: 'v',
description: 'd',
displayName: 'c',
email: 'm',
picture: 'p',
memberOf: 'm',
},
},
groups: {
dn: 'gdn',
options: {
scope: 'base',
attributes: ['*'],
filter: 'f',
paged: true,
},
set: [{ path: 'p', value: 'v' }],
map: {
rdn: 'u',
name: 'v',
description: 'd',
type: 't',
memberOf: 'm',
members: 'n',
},
},
},
];
expect(actual).toEqual(expected);
});
});
plugins/catalog-backend/src/ingestion/processors/ldap/config.ts
+264
View File
@@ -0,0 +1,264 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config, JsonValue } from '@backstage/config';
import { SearchOptions } from 'ldapjs';
import mergeWith from 'lodash/mergeWith';
import { RecursivePartial } from './util';
/**
* The configuration parameters for a single LDAP provider.
*/
export type LdapProviderConfig = {
// The prefix of the target that this matches on, e.g.
// "ldaps://ds.example.net", with no trailing slash.
target: string;
// The settings to use for the bind command. If none are specified, the bind
// command is not issued.
bind?: BindConfig;
// The settings that govern the reading and interpretation of users
users: UserConfig;
// The settings that govern the reading and interpretation of groups
groups: GroupConfig;
};
/**
* The settings to use for the a command.
*/
export type BindConfig = {
// The DN of the user to auth as, e.g.
// uid=ldap-robot,ou=robots,ou=example,dc=example,dc=net
dn: string;
// The secret of the user to auth as (its password)
secret: string;
};
/**
* The settings that govern the reading and interpretation of users.
*/
export type UserConfig = {
// The DN under which users are stored.
dn: string;
// The search options to use.
// Only the scope, filter, attributes, and paged fields are supported. The
// default is scope "one" and attributes "*" and "+".
options: SearchOptions;
// JSON paths (on a.b.c form) and hard coded values to set on those paths
set?: { path: string; value: JsonValue }[];
// Mappings from well known entity fields, to LDAP attribute names
map: {
// The name of the attribute that holds the relative distinguished name of
// each entry. Defaults to "uid".
rdn: string;
// The name of the attribute that shall be used for the value of the
// metadata.name field of the entity. Defaults to "uid".
name: string;
// The name of the attribute that shall be used for the value of the
// metadata.description field of the entity.
description?: string;
// The name of the attribute that shall be used for the value of the
// spec.profile.displayName field of the entity. Defaults to "cn".
displayName: string;
// The name of the attribute that shall be used for the value of the
// spec.profile.email field of the entity. Defaults to "mail".
email: string;
// The name of the attribute that shall be used for the value of the
// spec.profile.picture field of the entity.
picture?: string;
// The name of the attribute that shall be used for the values of the
// spec.memberOf field of the entity. Defaults to "memberOf".
memberOf: string;
};
};
/**
* The settings that govern the reading and interpretation of groups.
*/
export type GroupConfig = {
// The DN under which groups are stored.
dn: string;
// The search options to use.
// Only the scope, filter, attributes, and paged fields are supported.
options: SearchOptions;
// JSON paths (on a.b.c form) and hard coded values to set on those paths
set?: { path: string; value: JsonValue }[];
// Mappings from well known entity fields, to LDAP attribute names
map: {
// The name of the attribute that holds the relative distinguished name of
// each entry. Defaults to "cn".
rdn: string;
// The name of the attribute that shall be used for the value of the
// metadata.name field of the entity. Defaults to "cn".
name: string;
// The name of the attribute that shall be used for the value of the
// metadata.description field of the entity. Defaults to "description".
description: string;
// The name of the attribute that shall be used for the value of the
// spec.type field of the entity. Defaults to "groupType".
type: string;
// The name of the attribute that shall be used for the values of the
// spec.parent field of the entity. Defaults to "memberOf".
memberOf: string;
// The name of the attribute that shall be used for the values of the
// spec.children field of the entity. Defaults to "member".
members: string;
};
};
const defaultConfig = {
users: {
options: {
scope: 'one',
attributes: ['*', '+'],
},
map: {
rdn: 'uid',
name: 'uid',
displayName: 'cn',
email: 'mail',
memberOf: 'memberOf',
},
},
groups: {
options: {
scope: 'one',
attributes: ['*', '+'],
},
map: {
rdn: 'cn',
name: 'cn',
description: 'description',
type: 'groupType',
memberOf: 'memberOf',
members: 'member',
},
},
};
/**
* Parses configuration.
*
* @param config The root of the LDAP config hierarchy
*/
export function readLdapConfig(config: Config): LdapProviderConfig[] {
function readBindConfig(
c: Config | undefined,
): LdapProviderConfig['bind'] | undefined {
if (!c) {
return undefined;
}
return {
dn: c.getString('dn'),
secret: c.getString('secret'),
};
}
function readOptionsConfig(c: Config | undefined): SearchOptions {
if (!c) {
return {};
}
return {
scope: c.getOptionalString('scope') as SearchOptions['scope'],
filter: c.getOptionalString('filter'),
attributes: c.getOptionalStringArray('attributes'),
paged: c.getOptionalBoolean('paged'),
};
}
function readSetConfig(
c: Config[] | undefined,
): { path: string; value: JsonValue }[] | undefined {
if (!c) {
return undefined;
}
return c.map(entry => ({
path: entry.getString('path'),
value: entry.get('value'),
}));
}
function readUserMapConfig(
c: Config | undefined,
): Partial<LdapProviderConfig['users']['map']> {
if (!c) {
return {};
}
return {
rdn: c.getOptionalString('rdn'),
name: c.getOptionalString('name'),
description: c.getOptionalString('description'),
displayName: c.getOptionalString('displayName'),
email: c.getOptionalString('email'),
picture: c.getOptionalString('picture'),
memberOf: c.getOptionalString('memberOf'),
};
}
function readGroupMapConfig(
c: Config | undefined,
): Partial<LdapProviderConfig['groups']['map']> {
if (!c) {
return {};
}
return {
rdn: c.getOptionalString('rdn'),
name: c.getOptionalString('name'),
description: c.getOptionalString('description'),
type: c.getOptionalString('type'),
memberOf: c.getOptionalString('memberOf'),
members: c.getOptionalString('members'),
};
}
function readUserConfig(
c: Config,
): RecursivePartial<LdapProviderConfig['users']> {
return {
dn: c.getString('dn'),
options: readOptionsConfig(c.getOptionalConfig('options')),
set: readSetConfig(c.getOptionalConfigArray('set')),
map: readUserMapConfig(c.getOptionalConfig('map')),
};
}
function readGroupConfig(
c: Config,
): RecursivePartial<LdapProviderConfig['groups']> {
return {
dn: c.getString('dn'),
options: readOptionsConfig(c.getOptionalConfig('options')),
set: readSetConfig(c.getOptionalConfigArray('set')),
map: readGroupMapConfig(c.getOptionalConfig('map')),
};
}
const providerConfigs = config.getOptionalConfigArray('providers') ?? [];
return providerConfigs.map(c => {
const newConfig = {
target: c.getString('target').replace(/\/+$/, ''),
bind: readBindConfig(c.getOptionalConfig('bind')),
users: readUserConfig(c.getConfig('users')),
groups: readGroupConfig(c.getConfig('groups')),
};
const merged = mergeWith({}, defaultConfig, newConfig, (_into, from) => {
// Replace arrays instead of merging, otherwise default behavior
return Array.isArray(from) ? from : undefined;
});
return merged as LdapProviderConfig;
});
}
plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* The name of an entity annotation, that references the RDN of the LDAP object
* it was ingested from.
*
* The RDN is the name of the leftmost attribute that identifies the item; for
* example, for an item with the fully qualified DN
* uid=john,ou=people,ou=spotify,dc=spotify,dc=net the generated entity would
* have this attribute, with the value "john".
*/
export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn';
/**
* The name of an entity annotation, that references the DN of the LDAP object
* it was ingested from.
*
* The DN is the fully qualified name that identifies the item; for example,
* for an item with the DN uid=john,ou=people,ou=spotify,dc=spotify,dc=net the
* generated entity would have this attribute, with that full string as its
* value.
*/
export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn';
/**
* The name of an entity annotation, that references the UUID of the LDAP
* object it was ingested from.
*
* The UUID is the globally unique ID that identifies the item; for example,
* for an item with the UUID 76ef928a-b251-1037-9840-d78227f36a7e, the
* generated entity would have this attribute, with that full string as its
* value.
*/
export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid';
plugins/catalog-backend/src/ingestion/processors/ldap/index.ts
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { LdapClient } from './client';
export { readLdapConfig } from './config';
export type { LdapProviderConfig } from './config';
export {
LDAP_DN_ANNOTATION,
LDAP_RDN_ANNOTATION,
LDAP_UUID_ANNOTATION,
} from './constants';
export { readLdapOrg } from './read';
plugins/catalog-backend/src/ingestion/processors/ldap/read.test.ts
+393
View File
@@ -0,0 +1,393 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { SearchEntry } from 'ldapjs';
import merge from 'lodash/merge';
import { LdapClient } from './client';
import { GroupConfig, UserConfig } from './config';
import {
LDAP_DN_ANNOTATION,
LDAP_RDN_ANNOTATION,
LDAP_UUID_ANNOTATION,
} from './constants';
import { readLdapGroups, readLdapUsers, resolveRelations } from './read';
import { RecursivePartial } from './util';
function user(data: RecursivePartial<UserEntity>): UserEntity {
return merge(
{},
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name: 'name' },
spec: { profile: {}, memberOf: [] },
} as UserEntity,
data,
);
}
function group(data: RecursivePartial<GroupEntity>): GroupEntity {
return merge(
{},
{
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: { name: 'name' },
spec: { type: 'type', ancestors: [], children: [], descendants: [] },
} as GroupEntity,
data,
);
}
function searchEntry(attributes: Record<string, string[]>): SearchEntry {
return {
attributes: Object.entries(attributes).map(([k, vs]) => ({
json: {
type: k,
vals: vs,
},
})),
} as any;
}
describe('readLdapUsers', () => {
const client: jest.Mocked<LdapClient> = {
search: jest.fn(),
} as any;
afterEach(() => jest.resetAllMocks());
it('transfers all attributes', async () => {
client.search.mockResolvedValue([
searchEntry({
uid: ['uid-value'],
description: ['description-value'],
cn: ['cn-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
]);
const config: UserConfig = {
dn: 'ddd',
options: {},
map: {
rdn: 'uid',
name: 'uid',
description: 'description',
displayName: 'cn',
email: 'mail',
picture: 'avatarUrl',
memberOf: 'memberOf',
},
};
const { users, userMemberOf } = await readLdapUsers(client, config);
expect(users).toEqual([
expect.objectContaining({
metadata: {
name: 'uid-value',
description: 'description-value',
annotations: {
[LDAP_DN_ANNOTATION]: 'dn-value',
[LDAP_RDN_ANNOTATION]: 'uid-value',
[LDAP_UUID_ANNOTATION]: 'uuid-value',
},
},
spec: {
profile: {
displayName: 'cn-value',
email: 'mail-value',
picture: 'avatarUrl-value',
},
memberOf: [],
},
}),
]);
expect(userMemberOf).toEqual(
new Map([['dn-value', new Set(['x', 'y', 'z'])]]),
);
});
});
describe('readLdapGroups', () => {
const client: jest.Mocked<LdapClient> = {
search: jest.fn(),
} as any;
afterEach(() => jest.resetAllMocks());
it('transfers all attributes', async () => {
client.search.mockResolvedValue([
searchEntry({
cn: ['cn-value'],
description: ['description-value'],
tt: ['type-value'],
memberOf: ['x', 'y', 'z'],
member: ['e', 'f', 'g'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
]);
const config: GroupConfig = {
dn: 'ddd',
options: {},
map: {
rdn: 'cn',
name: 'cn',
description: 'description',
type: 'tt',
memberOf: 'memberOf',
members: 'member',
},
};
const { groups, groupMember, groupMemberOf } = await readLdapGroups(
client,
config,
);
expect(groups).toEqual([
expect.objectContaining({
metadata: {
name: 'cn-value',
description: 'description-value',
annotations: {
[LDAP_DN_ANNOTATION]: 'dn-value',
[LDAP_RDN_ANNOTATION]: 'cn-value',
[LDAP_UUID_ANNOTATION]: 'uuid-value',
},
},
spec: {
type: 'type-value',
ancestors: [],
children: [],
descendants: [],
},
}),
]);
expect(groupMember).toEqual(
new Map([['dn-value', new Set(['e', 'f', 'g'])]]),
);
expect(groupMemberOf).toEqual(
new Map([['dn-value', new Set(['x', 'y', 'z'])]]),
);
});
});
describe('resolveRelations', () => {
describe('lookup', () => {
it('matches by DN', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_DN_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_DN_ANNOTATION]: 'ca' },
},
});
const groupMember = new Map<string, Set<string>>([
['pa', new Set(['ca'])],
]);
resolveRelations([parent, child], [], new Map(), new Map(), groupMember);
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
});
it('matches by UUID', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_UUID_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_UUID_ANNOTATION]: 'ca' },
},
});
const groupMember = new Map<string, Set<string>>([
['pa', new Set(['ca'])],
]);
resolveRelations([parent, child], [], new Map(), new Map(), groupMember);
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
});
});
describe('userMemberOf', () => {
it('populates relations by dn', () => {
const host = group({
metadata: { name: 'host', annotations: { [LDAP_DN_ANNOTATION]: 'ha' } },
});
const member = user({
metadata: {
name: 'member',
annotations: { [LDAP_DN_ANNOTATION]: 'ma' },
},
});
const userMemberOf = new Map<string, Set<string>>([
['ma', new Set(['ha'])],
]);
resolveRelations([host], [member], userMemberOf, new Map(), new Map());
expect(member.spec.memberOf).toEqual(['host']);
});
it('populates relations by uuid', () => {
const host = group({
metadata: {
name: 'host',
annotations: { [LDAP_UUID_ANNOTATION]: 'ha' },
},
});
const member = user({
metadata: {
name: 'member',
annotations: { [LDAP_DN_ANNOTATION]: 'ma' },
},
});
const userMemberOf = new Map<string, Set<string>>([
['ma', new Set(['ha'])],
]);
resolveRelations([host], [member], userMemberOf, new Map(), new Map());
expect(member.spec.memberOf).toEqual(['host']);
});
});
describe('groupMemberOf', () => {
it('populates relations by dn', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_DN_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_DN_ANNOTATION]: 'ca' },
},
});
const groupMemberOf = new Map<string, Set<string>>([
['ca', new Set(['pa'])],
]);
resolveRelations(
[parent, child],
[],
new Map(),
groupMemberOf,
new Map(),
);
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
});
});
it('populates relations by uuid', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_UUID_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_UUID_ANNOTATION]: 'ca' },
},
});
const groupMemberOf = new Map<string, Set<string>>([
['ca', new Set(['pa'])],
]);
resolveRelations([parent, child], [], new Map(), groupMemberOf, new Map());
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
});
describe('groupMember', () => {
it('populates relations by dn', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_DN_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_DN_ANNOTATION]: 'ca' },
},
});
const member = user({
metadata: {
name: 'member',
annotations: { [LDAP_DN_ANNOTATION]: 'ma' },
},
});
const groupMember = new Map<string, Set<string>>([
['pa', new Set(['ca', 'ma'])],
]);
resolveRelations(
[parent, child],
[member],
new Map(),
new Map(),
groupMember,
);
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
expect(member.spec.memberOf).toEqual(['parent']);
});
it('populates relations by uuid', () => {
const parent = group({
metadata: {
name: 'parent',
annotations: { [LDAP_UUID_ANNOTATION]: 'pa' },
},
});
const child = group({
metadata: {
name: 'child',
annotations: { [LDAP_UUID_ANNOTATION]: 'ca' },
},
});
const member = user({
metadata: {
name: 'member',
annotations: { [LDAP_UUID_ANNOTATION]: 'ma' },
},
});
const groupMember = new Map<string, Set<string>>([
['pa', new Set(['ca', 'ma'])],
]);
resolveRelations(
[parent, child],
[member],
new Map(),
new Map(),
groupMember,
);
expect(parent.spec.children).toEqual(['child']);
expect(child.spec.parent).toEqual('parent');
expect(member.spec.memberOf).toEqual(['parent']);
});
});
});
plugins/catalog-backend/src/ingestion/processors/ldap/read.ts
+422
View File
@@ -0,0 +1,422 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import lodashSet from 'lodash/set';
import { buildOrgHierarchy } from '../util/org';
import { LdapClient } from './client';
import { GroupConfig, UserConfig } from './config';
import {
LDAP_DN_ANNOTATION,
LDAP_RDN_ANNOTATION,
LDAP_UUID_ANNOTATION,
} from './constants';
// NOTE(freben): These attribute names are assumed to be the same across all
// compliant LDAP implementations
const DN_ATTRIBUTE = 'entryDN';
const UUID_ATTRIBUTE = 'entryUUID';
/**
* Reads groups out of an LDAP provider.
*
* @param client The LDAP client
* @param config The user data configuration
*/
export async function readLdapUsers(
client: LdapClient,
config: UserConfig,
): Promise<{
users: UserEntity[]; // With all relations empty
userMemberOf: Map<string, Set<string>>; // DN -> DN or UUID of groups
}> {
const { dn, options, set, map } = config;
const entries = await client.search(dn, options);
const entities: UserEntity[] = [];
const userMemberOf: Map<string, Set<string>> = new Map();
for (const entry of entries) {
const attributes = new Map(
entry.attributes.map(attr => {
const data = attr.json;
return [data.type, data.vals];
}),
);
const entity: UserEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
name: '',
annotations: {},
},
spec: {
profile: {},
memberOf: [],
},
};
if (set) {
for (const { path, value } of set) {
lodashSet(entity, path, value);
}
}
mapStringAttr(attributes, map.name, v => {
entity.metadata.name = v;
});
mapStringAttr(attributes, map.description, v => {
entity.metadata.description = v;
});
mapStringAttr(attributes, map.rdn, v => {
entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
});
mapStringAttr(attributes, UUID_ATTRIBUTE, v => {
entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
});
mapStringAttr(attributes, DN_ATTRIBUTE, v => {
entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
});
mapStringAttr(attributes, map.displayName, v => {
entity.spec.profile!.displayName = v;
});
mapStringAttr(attributes, map.email, v => {
entity.spec.profile!.email = v;
});
mapStringAttr(attributes, map.picture, v => {
entity.spec.profile!.picture = v;
});
mapReferencesAttr(attributes, map.memberOf, (myDn, vs) => {
ensureItems(userMemberOf, myDn, vs);
});
entities.push(entity);
}
return { users: entities, userMemberOf };
}
/**
* Reads groups out of an LDAP provider.
*
* @param client The LDAP client
* @param config The group data configuration
*/
export async function readLdapGroups(
client: LdapClient,
config: GroupConfig,
): Promise<{
groups: GroupEntity[]; // With all relations empty
groupMemberOf: Map<string, Set<string>>; // DN -> DN or UUID of groups
groupMember: Map<string, Set<string>>; // DN -> DN or UUID of groups & users
}> {
const { dn, options, set, map } = config;
const entries = await client.search(dn, options);
const groups: GroupEntity[] = [];
const groupMemberOf: Map<string, Set<string>> = new Map();
const groupMember: Map<string, Set<string>> = new Map();
for (const entry of entries) {
const attributes = new Map(
entry.attributes.map(attr => {
const data = attr.json;
return [data.type, data.vals];
}),
);
const entity: GroupEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
name: '',
annotations: {},
},
spec: {
type: 'unknown',
ancestors: [],
children: [],
descendants: [],
},
};
if (set) {
for (const { path, value } of set) {
lodashSet(entity, path, value);
}
}
mapStringAttr(attributes, map.name, v => {
entity.metadata.name = v;
});
mapStringAttr(attributes, map.description, v => {
entity.metadata.description = v;
});
mapStringAttr(attributes, map.rdn, v => {
entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
});
mapStringAttr(attributes, UUID_ATTRIBUTE, v => {
entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
});
mapStringAttr(attributes, DN_ATTRIBUTE, v => {
entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
});
mapStringAttr(attributes, map.type, v => {
entity.spec.type = v;
});
mapReferencesAttr(attributes, map.memberOf, (myDn, vs) => {
ensureItems(groupMemberOf, myDn, vs);
});
mapReferencesAttr(attributes, map.members, (myDn, vs) => {
ensureItems(groupMember, myDn, vs);
});
groups.push(entity);
}
return {
groups,
groupMemberOf,
groupMember,
};
}
/**
* Reads users and groups out of an LDAP provider.
*
* Invokes the above "raw" read functions and stitches together the results
* with all relations etc filled in.
*
* @param client The LDAP client
* @param logger A logger instance
* @param userConfig The user data configuration
* @param groupConfig The group data configuration
*/
export async function readLdapOrg(
client: LdapClient,
userConfig: UserConfig,
groupConfig: GroupConfig,
): Promise<{
users: UserEntity[];
groups: GroupEntity[];
}> {
const { users, userMemberOf } = await readLdapUsers(client, userConfig);
const { groups, groupMemberOf, groupMember } = await readLdapGroups(
client,
groupConfig,
);
resolveRelations(groups, users, userMemberOf, groupMemberOf, groupMember);
users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
return { users, groups };
}
//
// Helpers
//
// Maps a single-valued attribute to a consumer
function mapStringAttr(
attributes: Map<string, string[]>,
attributeName: string | undefined,
setter: (value: string) => void,
) {
if (attributeName) {
const values = attributes.get(attributeName);
if (values && values.length === 1) {
setter(values[0]);
}
}
}
// Maps a multi-valued attribute of references to other objects, to a consumer
function mapReferencesAttr(
attributes: Map<string, string[]>,
attributeName: string | undefined,
setter: (sourceDn: string, targets: string[]) => void,
) {
if (attributeName) {
const values = attributes.get(attributeName);
const dn = attributes.get(DN_ATTRIBUTE);
if (values && dn && dn.length === 1) {
setter(dn[0], values);
}
}
}
// Inserts a number of values in a key-values mapping
function ensureItems(
target: Map<string, Set<string>>,
key: string,
values: string[],
) {
if (key) {
let set = target.get(key);
if (!set) {
set = new Set();
target.set(key, set);
}
for (const value of values) {
if (value) {
set!.add(value);
}
}
}
}
/**
* Takes groups and entities with empty relations, and fills in the various
* relations that were returned by the readers, and forms the org hierarchy.
*
* @param groups Group entities with empty relations; modified in place
* @param users User entities with empty relations; modified in place
* @param userMemberOf For a user DN, the set of group DNs or UUIDs that the
* user is a member of
* @param groupMemberOf For a group DN, the set of group DNs or UUIDs that the
* group is a member of (parents in the hierarchy)
* @param groupMember For a group DN, the set of group DNs or UUIDs that are
* members of the group (children in the hierarchy)
*/
export function resolveRelations(
groups: GroupEntity[],
users: UserEntity[],
userMemberOf: Map<string, Set<string>>,
groupMemberOf: Map<string, Set<string>>,
groupMember: Map<string, Set<string>>,
) {
// Build reference lookup tables - all of the relations that are output from
// the above calls can be expressed as either DNs or UUIDs so we need to be
// able to find by both, as well as the name. Note that we expect them to not
// collide here - this is a reasonable assumption as long as the fields are
// the supported forms.
const userMap: Map<string, UserEntity> = new Map(); // by name, dn, uuid
const groupMap: Map<string, GroupEntity> = new Map(); // by name, dn, uuid
for (const user of users) {
userMap.set(user.metadata.name, user);
userMap.set(user.metadata.annotations![LDAP_DN_ANNOTATION], user);
userMap.set(user.metadata.annotations![LDAP_UUID_ANNOTATION], user);
}
for (const group of groups) {
groupMap.set(group.metadata.name, group);
groupMap.set(group.metadata.annotations![LDAP_DN_ANNOTATION], group);
groupMap.set(group.metadata.annotations![LDAP_UUID_ANNOTATION], group);
}
// This can happen e.g. if entryUUID wasn't returned by the server
userMap.delete('');
groupMap.delete('');
userMap.delete(undefined!);
groupMap.delete(undefined!);
// Fill in all of the immediate relations, now keyed on metadata.name. We
// keep all parents at this point, whether the target model can support more
// than one or not (it gets filtered farther down). And group children are
// only groups in here.
const newUserMemberOf: Map<string, Set<string>> = new Map();
const newGroupParents: Map<string, Set<string>> = new Map();
const newGroupChildren: Map<string, Set<string>> = new Map();
// Resolve and store in the intermediaries. It may seem redundant that the
// input data has both parent and children directions, as well as both
// user->group and group->user - the reason is that different LDAP schemas
// express relations in different directions. Some may have a user memberOf
// overlay, some don't, for example.
for (const [userN, groupsN] of userMemberOf.entries()) {
const user = userMap.get(userN);
if (user) {
for (const groupN of groupsN) {
const group = groupMap.get(groupN);
if (group) {
ensureItems(newUserMemberOf, user.metadata.name, [
group.metadata.name,
]);
}
}
}
}
for (const [groupN, parentsN] of groupMemberOf.entries()) {
const group = groupMap.get(groupN);
if (group) {
for (const parentN of parentsN) {
const parentGroup = groupMap.get(parentN);
if (parentGroup) {
ensureItems(newGroupParents, group.metadata.name, [
parentGroup.metadata.name,
]);
ensureItems(newGroupChildren, parentGroup.metadata.name, [
group.metadata.name,
]);
}
}
}
}
for (const [groupN, membersN] of groupMember.entries()) {
const group = groupMap.get(groupN);
if (group) {
for (const memberN of membersN) {
// Group members can be both users and groups in the input model, so
// try both
const memberUser = userMap.get(memberN);
if (memberUser) {
ensureItems(newUserMemberOf, memberUser.metadata.name, [
group.metadata.name,
]);
} else {
const memberGroup = groupMap.get(memberN);
if (memberGroup) {
ensureItems(newGroupChildren, group.metadata.name, [
memberGroup.metadata.name,
]);
ensureItems(newGroupParents, memberGroup.metadata.name, [
group.metadata.name,
]);
}
}
}
}
}
// Write down the relations again into the actual entities
for (const [userN, groupsN] of newUserMemberOf.entries()) {
const user = userMap.get(userN);
if (user) {
user.spec.memberOf = Array.from(groupsN).sort();
}
}
for (const [groupN, parentsN] of newGroupParents.entries()) {
if (parentsN.size === 1) {
const group = groupMap.get(groupN);
if (group) {
group.spec.parent = parentsN.values().next().value;
}
}
}
for (const [groupN, childrenN] of newGroupChildren.entries()) {
const group = groupMap.get(groupN);
if (group) {
group.spec.children = Array.from(childrenN).sort();
}
}
// Fill out the rest of the hierarchy
buildOrgHierarchy(groups);
}
plugins/catalog-backend/src/ingestion/processors/ldap/util.test.ts
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { errorString, RecursivePartial } from './util';
describe('errorString', () => {
it('formats', () => {
const e = { code: 1, name: 'n', message: 'm' };
expect(errorString(e)).toEqual('1 n: m');
});
});
describe('RecursivePartial', () => {
it('is recursive', () => {
type X = {
required: {
required: string;
};
};
const x: RecursivePartial<X> = {
required: {},
};
expect(x).toEqual({ required: {} });
});
});
plugins/catalog-backend/src/ingestion/processors/ldap/util.ts
+37
View File
@@ -0,0 +1,37 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Error as LDAPError } from 'ldapjs';
/**
* Builds a string form of an LDAP Error structure.
*
* @param error The error
*/
export function errorString(error: LDAPError) {
return `${error.code} ${error.name}: ${error.message}`;
}
/**
* Makes all keys of an entire hierarchy optional.
*/
export type RecursivePartial<T> = {
[P in keyof T]?: T[P] extends (infer U)[]
? RecursivePartial<U>[]
: T[P] extends object
? RecursivePartial<T[P]>
: T[P];
};
plugins/catalog-backend/src/ingestion/processors/util/org.test.ts
+4 -27
View File
@@ -14,18 +14,9 @@
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { GroupEntity } from '@backstage/catalog-model';
import { buildOrgHierarchy } from './org';
function u(name: string): UserEntity {
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: { name },
spec: { memberOf: [] },
};
}
function g(
name: string,
parent: string | undefined,
@@ -40,26 +31,12 @@ function g(
}
describe('buildOrgHierarchy', () => {
it('puts users in the respective groups', () => {
const a = g('a', undefined, []);
const b = g('b', undefined, []);
const x = u('x');
const y = u('y');
const groupMemberUsers: Map<string, string[]> = new Map([
['a', ['x', 'y']],
['b', ['y']],
]);
buildOrgHierarchy([a, b], [x, y], groupMemberUsers);
expect(x.spec.memberOf).toEqual(['a']);
expect(y.spec.memberOf).toEqual(['a', 'b']);
});
it('adds groups to their parent.children', () => {
const a = g('a', undefined, []);
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
buildOrgHierarchy([a, b, c, d]);
expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd']));
expect(b.spec.children).toEqual(expect.arrayContaining(['c']));
expect(c.spec.children).toEqual([]);
@@ -71,7 +48,7 @@ describe('buildOrgHierarchy', () => {
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
buildOrgHierarchy([a, b, c, d]);
expect(a.spec.descendants).toEqual(expect.arrayContaining(['b', 'c', 'd']));
expect(b.spec.descendants).toEqual(expect.arrayContaining(['c']));
expect(c.spec.descendants).toEqual([]);
@@ -83,7 +60,7 @@ describe('buildOrgHierarchy', () => {
const b = g('b', 'a', []);
const c = g('c', 'b', []);
const d = g('d', 'a', []);
buildOrgHierarchy([a, b, c, d], [], new Map());
buildOrgHierarchy([a, b, c, d]);
expect(a.spec.ancestors).toEqual([]);
expect(b.spec.ancestors).toEqual(expect.arrayContaining(['a']));
expect(c.spec.ancestors).toEqual(expect.arrayContaining(['a', 'b']));
plugins/catalog-backend/src/ingestion/processors/util/org.ts
+2 -20
View File
@@ -14,28 +14,10 @@
* limitations under the License.
*/
import { GroupEntity, UserEntity } from '@backstage/catalog-model';
import { GroupEntity } from '@backstage/catalog-model';
export function buildOrgHierarchy(
groups: GroupEntity[],
users: UserEntity[],
groupMemberUsers: Map<string, string[]>,
) {
export function buildOrgHierarchy(groups: GroupEntity[]) {
const groupsByName = new Map(groups.map(g => [g.metadata.name, g]));
const usersByName = new Map(users.map(u => [u.metadata.name, u]));
//
// Make sure that u.memberOf contain all g
//
for (const [groupName, userNames] of groupMemberUsers.entries()) {
for (const userName of userNames) {
const user = usersByName.get(userName);
if (user && !user.spec.memberOf.includes(groupName)) {
user.spec.memberOf.push(groupName);
}
}
}
//
// Make sure that g.parent.children contain g
yarn.lock
+54 -2
View File
@@ -5121,6 +5121,13 @@
"@types/koa-compose" "*"
"@types/node" "*"
"@types/ldapjs@^1.0.9":
version "1.0.9"
resolved "https://registry.npmjs.org/@types/ldapjs/-/ldapjs-1.0.9.tgz#1224192d14cc5ab5218fcea72ebb04489c52cb95"
integrity sha512-3PvY7Drp1zoLbcGlothCAkoc5o6Jp9KvUPwHadlHyKp3yPvyeIh7w2zQc9UXMzgDRkoeGXUEODtbEs5XCh9ZyA==
dependencies:
"@types/node" "*"
"@types/lodash@^4.14.151":
version "4.14.161"
resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.161.tgz#a21ca0777dabc6e4f44f3d07f37b765f54188b18"
@@ -6017,6 +6024,11 @@ abbrev@1:
resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
abstract-logging@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.0.tgz#08a85814946c98ef06f4256ad470aba1886d4490"
integrity sha512-/oA9z7JszpIioo6J6dB79LVUgJ3eD3cxkAmdCkvWWS+Y9tPtALs1rLqOekLUXUbYqM2fB9TTK0ibAyZJJOP/CA==
accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7:
version "1.3.7"
resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd"
@@ -6701,7 +6713,7 @@ asn1.js@^4.0.0:
inherits "^2.0.1"
minimalistic-assert "^1.0.0"
asn1@~0.2.0, asn1@~0.2.3:
asn1@^0.2.4, asn1@~0.2.0, asn1@~0.2.3:
version "0.2.4"
resolved "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136"
integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==
@@ -7254,6 +7266,13 @@ backo2@^1.0.2:
resolved "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947"
integrity sha1-MasayLEpNjRj41s+u2n038+6eUc=
backoff@^2.5.0:
version "2.5.0"
resolved "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz#f616eda9d3e4b66b8ca7fca79f695722c5f8e26f"
integrity sha1-9hbtqdPktmuMp/ynn2lXIsX44m8=
dependencies:
precond "0.2"
bail@^1.0.0:
version "1.0.5"
resolved "https://registry.npmjs.org/bail/-/bail-1.0.5.tgz#b6fa133404a392cbc1f8c4bf63f5953351e7a776"
@@ -14948,6 +14967,27 @@ lcid@^1.0.0:
dependencies:
invert-kv "^1.0.0"
ldap-filter@^0.3.3:
version "0.3.3"
resolved "https://registry.npmjs.org/ldap-filter/-/ldap-filter-0.3.3.tgz#2b14c68a2a9d4104dbdbc910a1ca85fd189e9797"
integrity sha1-KxTGiiqdQQTb28kQocqF/Riel5c=
dependencies:
assert-plus "^1.0.0"
ldapjs@^2.2.0:
version "2.2.0"
resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.2.0.tgz#c23846d35bf215b4ba4721ff22f11d3df6391e2f"
integrity sha512-9+ekbj97nxRYQMRgEm/HYFhFLWSRKah2PnReUfhfM5f62XBeUSFolB+AQ2Jij5tqowpksTnrdNCZLP6C+V3uag==
dependencies:
abstract-logging "^2.0.0"
asn1 "^0.2.4"
assert-plus "^1.0.0"
backoff "^2.5.0"
ldap-filter "^0.3.3"
once "^1.4.0"
vasync "^2.2.0"
verror "^1.8.1"
left-pad@^1.3.0:
version "1.3.0"
resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e"
@@ -18390,6 +18430,11 @@ postgres-interval@^1.1.0:
dependencies:
xtend "^4.0.0"
precond@0.2:
version "0.2.3"
resolved "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz#aa9591bcaa24923f1e0f4849d240f47efc1075ac"
integrity sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw=
preferred-pm@^3.0.0:
version "3.0.2"
resolved "https://registry.npmjs.org/preferred-pm/-/preferred-pm-3.0.2.tgz#bbdbef1014e34a7490349bf70d6d244b8d57a5e1"
@@ -23077,12 +23122,19 @@ vary@^1, vary@~1.1.2:
resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=
vasync@^2.2.0:
version "2.2.0"
resolved "https://registry.npmjs.org/vasync/-/vasync-2.2.0.tgz#cfde751860a15822db3b132bc59b116a4adaf01b"
integrity sha1-z951GGChWCLbOxMrxZsRakra8Bs=
dependencies:
verror "1.10.0"
vendors@^1.0.0:
version "1.0.4"
resolved "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz#e2b800a53e7a29b93506c3cf41100d16c4c4ad8e"
integrity sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w==
verror@1.10.0:
verror@1.10.0, verror@^1.8.1:
version "1.10.0"
resolved "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400"
integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=