From b9c8062c0833d973c8f06554266cb8f7ee32746a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 2 Oct 2020 13:08:54 +0200 Subject: [PATCH] feat(catalog): add an ldap processor --- app-config.yaml | 17 + .../well-known-annotations.md | 15 + packages/catalog-model/src/entity/ref.test.ts | 21 + packages/catalog-model/src/entity/ref.ts | 29 +- plugins/catalog-backend/package.json | 2 + .../src/ingestion/HigherOrderOperations.ts | 27 +- .../src/ingestion/LocationReaders.ts | 18 +- .../GithubOrgReaderProcessor.test.ts | 25 +- .../processors/GithubOrgReaderProcessor.ts | 33 +- .../processors/LdapOrgReaderProcessor.ts | 88 ++++ .../src/ingestion/processors/ldap/client.ts | 92 ++++ .../ingestion/processors/ldap/config.test.ts | 172 +++++++ .../src/ingestion/processors/ldap/config.ts | 264 +++++++++++ .../ingestion/processors/ldap/constants.ts | 48 ++ .../src/ingestion/processors/ldap/index.ts | 25 ++ .../ingestion/processors/ldap/read.test.ts | 393 ++++++++++++++++ .../src/ingestion/processors/ldap/read.ts | 422 ++++++++++++++++++ .../ingestion/processors/ldap/util.test.ts | 38 ++ .../src/ingestion/processors/ldap/util.ts | 37 ++ .../src/ingestion/processors/util/org.test.ts | 31 +- .../src/ingestion/processors/util/org.ts | 22 +- yarn.lock | 56 ++- 22 files changed, 1782 insertions(+), 93 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/LdapOrgReaderProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/config.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/config.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/read.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/read.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/util.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/ldap/util.ts diff --git a/app-config.yaml b/app-config.yaml index 227e188ec1..d670771a31 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -108,6 +108,23 @@ catalog: # apiBaseUrl: https://ghe.example.net/api/v3 # token: # $env: GHE_PRIVATE_TOKEN + ldapOrg: + ### Example for how to add your enterprise LDAP server + # providers: + # - target: ldaps://ds.example.net + # bind: + # dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net + # secret: { $secret: { env: LDAP_SECRET } } + # users: + # dn: ou=people,ou=example,dc=example,dc=net + # options: + # filter: (uid=*) + # map: + # description: l + # groups: + # dn: ou=access,ou=groups,ou=example,dc=example,dc=net + # options: + # filter: (&(objectClass=some-group-class)(!(groupType=email))) locations: # Backstage example components diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md index 1bd5528f22..757e0461a9 100644 --- a/docs/features/software-catalog/well-known-annotations.md +++ b/docs/features/software-catalog/well-known-annotations.md @@ -179,6 +179,21 @@ fallback (`rollbar.organization` followed by `organization.name`). Specifying this annotation may enable Rollbar related features in Backstage for that entity. +### backstage.io/ldap-rdn, backstage.io/ldap-uuid, backstage.io/ldap-dn + +```yaml +# Example: +metadata: + annotations: + backstage.io/ldap-rdn: my-team + backstage.io/ldap-uuid: c57e8ba2-6cc4-1039-9ebc-d5f241a7ca21 + backstage.io/ldap-dn: cn=my-team,ou=access,ou=groups,ou=spotify,dc=spotify,dc=net +``` + +The value of these annotations are the corresponding attributes that were found +when ingestion the entity from LDAP. Not all of them may be present, depending +on what attributes that the server presented at ingestion time. + ## Deprecated Annotations The following annotations are deprecated, and only listed here to aid in diff --git a/packages/catalog-model/src/entity/ref.test.ts b/packages/catalog-model/src/entity/ref.test.ts index d9787fe675..bd7cc9477d 100644 --- a/packages/catalog-model/src/entity/ref.test.ts +++ b/packages/catalog-model/src/entity/ref.test.ts @@ -15,6 +15,7 @@ */ import { ENTITY_DEFAULT_NAMESPACE } from './constants'; +import { Entity } from './Entity'; import { parseEntityName, parseEntityRef, serializeEntityRef } from './ref'; describe('ref', () => { @@ -333,6 +334,26 @@ describe('ref', () => { expect(serializeEntityRef({ name: 'c' })).toEqual('c'); }); + it('handles entities', () => { + const entityWithNamespace: Entity = { + apiVersion: 'a', + kind: 'b', + metadata: { + name: 'c', + namespace: 'd', + }, + }; + const entityWithoutNamespace: Entity = { + apiVersion: 'a', + kind: 'b', + metadata: { + name: 'c', + }, + }; + expect(serializeEntityRef(entityWithNamespace)).toEqual('b:d/c'); + expect(serializeEntityRef(entityWithoutNamespace)).toEqual('b:c'); + }); + it('picks the least complex form', () => { expect( serializeEntityRef({ kind: 'a', namespace: 'b', name: 'c' }), diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index c4c86b176c..cc3276ded5 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -160,12 +160,29 @@ export function parseEntityRef( * @param ref The reference to serialize * @returns The same reference on either string or compound form */ -export function serializeEntityRef(ref: { - kind?: string; - namespace?: string; - name: string; -}): EntityRef { - const { kind, namespace, name } = ref; +export function serializeEntityRef( + ref: + | Entity + | { + kind?: string; + namespace?: string; + name: string; + }, +): EntityRef { + let kind; + let namespace; + let name; + + if ('apiVersion' in ref) { + kind = ref.kind; + namespace = ref.metadata.namespace; + name = ref.metadata.name; + } else { + kind = ref.kind; + namespace = ref.namespace; + name = ref.name; + } + if ( kind?.includes(':') || kind?.includes('/') || diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 4eb437b871..a056766428 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -33,6 +33,7 @@ "fs-extra": "^9.0.0", "git-url-parse": "^11.2.0", "knex": "^0.21.1", + "ldapjs": "^2.2.0", "lodash": "^4.17.15", "morgan": "^1.10.0", "node-fetch": "^2.6.0", @@ -47,6 +48,7 @@ "@backstage/cli": "^0.1.1-alpha.23", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", + "@types/ldapjs": "^1.0.9", "@types/lodash": "^4.14.151", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts index 1b60666c8e..3263b887ed 100644 --- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts +++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts @@ -21,6 +21,7 @@ import { getEntityName, Location, LocationSpec, + serializeEntityRef, } from '@backstage/catalog-model'; import { v4 as uuidv4 } from 'uuid'; import { Logger } from 'winston'; @@ -159,28 +160,23 @@ export class HigherOrderOperations implements HigherOrderOperation { ); } + this.logger.info( + `Read ${readerOutput.entities.length} entities from location ${location.type} ${location.target}`, + ); + + const startTimestamp = Date.now(); for (const item of readerOutput.entities) { const { entity } = item; - this.logger.debug( - `Read entity kind="${entity.kind}" namespace="${ - entity.metadata.namespace || '' - }" name="${entity.metadata.name}"`, - ); - try { const previous = await this.entitiesCatalog.entityByName( getEntityName(entity), ); if (!previous) { - this.logger.debug(`No such entity found, adding`); await this.entitiesCatalog.addOrUpdateEntity(entity, location.id); } else if (entityHasChanges(previous, entity)) { - this.logger.debug(`Different from existing entity, updating`); await this.entitiesCatalog.addOrUpdateEntity(entity, location.id); - } else { - this.logger.debug(`Equal to existing entity, skipping update`); } await this.locationsCatalog.logUpdateSuccess( @@ -188,10 +184,8 @@ export class HigherOrderOperations implements HigherOrderOperation { entity.metadata.name, ); } catch (error) { - this.logger.debug( - `Failed refresh of entity kind="${entity.kind}" name="${ - entity.metadata.name - }" namespace="${entity.metadata.namespace || ''}", ${error}`, + this.logger.info( + `Failed refresh of entity ${serializeEntityRef(entity)}, ${error}`, ); await this.locationsCatalog.logUpdateFailure( @@ -201,5 +195,10 @@ export class HigherOrderOperations implements HigherOrderOperation { ); } } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.info( + `Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${duration} seconds`, + ); } } diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 96ffaf398c..3667fc58cb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -29,15 +29,16 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor'; import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor'; import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; +import { LdapOrgReaderProcessor } from './processors/LdapOrgReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { PlaceholderProcessor } from './processors/PlaceholderProcessor'; -import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor'; import * as result from './processors/results'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; import { @@ -120,7 +121,8 @@ export class LocationReaders implements LocationReader { StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), ...oldProcessors, - GithubOrgReaderProcessor.fromConfig(config), + GithubOrgReaderProcessor.fromConfig(config, logger), + LdapOrgReaderProcessor.fromConfig(config, logger), new UrlReaderProcessor(options), new YamlProcessor(), PlaceholderProcessor.default(), @@ -199,10 +201,6 @@ export class LocationReaders implements LocationReader { item: LocationProcessorLocationResult, emit: LocationProcessorEmit, ) { - this.logger.debug( - `Reading location ${item.location.type} ${item.location.target} optional=${item.optional}`, - ); - for (const processor of this.processors) { if (processor.readLocation) { try { @@ -228,10 +226,6 @@ export class LocationReaders implements LocationReader { item: LocationProcessorDataResult, emit: LocationProcessorEmit, ) { - this.logger.debug( - `Parsing data from location ${item.location.type} ${item.location.target} (${item.data.byteLength} bytes)`, - ); - for (const processor of this.processors) { if (processor.parseData) { try { @@ -254,10 +248,6 @@ export class LocationReaders implements LocationReader { item: LocationProcessorEntityResult, emit: LocationProcessorEmit, ): Promise { - this.logger.debug( - `Got entity at location ${item.location.type} ${item.location.target}, ${item.entity.apiVersion} ${item.entity.kind}`, - ); - let current = item.entity; for (const processor of this.processors) { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts index e0d35f78aa..6368fbe654 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { @@ -94,9 +95,15 @@ describe('GithubOrgReaderProcessor', () => { describe('implementation', () => { it('rejects unknown types', async () => { - const processor = new GithubOrgReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); + const processor = new GithubOrgReaderProcessor( + [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + getVoidLogger(), + ); const location: LocationSpec = { type: 'not-github-org', target: 'https://github.com', @@ -107,9 +114,15 @@ describe('GithubOrgReaderProcessor', () => { }); it('rejects unknown targets', async () => { - const processor = new GithubOrgReaderProcessor([ - { target: 'https://github.com', apiBaseUrl: 'https://api.github.com' }, - ]); + const processor = new GithubOrgReaderProcessor( + [ + { + target: 'https://github.com', + apiBaseUrl: 'https://api.github.com', + }, + ], + getVoidLogger(), + ); const location: LocationSpec = { type: 'github-org', target: 'https://not.github.com/apa', diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 5084d4c16f..f1d2dc2c32 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -17,6 +17,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { graphql } from '@octokit/graphql'; +import { Logger } from 'winston'; import * as results from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; import { getOrganizationTeams, getOrganizationUsers } from './util/github'; @@ -26,11 +27,14 @@ import { buildOrgHierarchy } from './util/org'; * Extracts teams and users out of a GitHub org. */ export class GithubOrgReaderProcessor implements LocationProcessor { - static fromConfig(config: Config) { - return new GithubOrgReaderProcessor(readConfig(config)); + static fromConfig(config: Config, logger: Logger) { + return new GithubOrgReaderProcessor(readConfig(config), logger); } - constructor(private readonly providers: ProviderConfig[]) {} + constructor( + private readonly providers: ProviderConfig[], + private readonly logger: Logger, + ) {} async readLocation( location: LocationSpec, @@ -59,13 +63,34 @@ export class GithubOrgReaderProcessor implements LocationProcessor { }, }); + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading GitHub users and groups'); + const { users } = await getOrganizationUsers(client, org); const { groups, groupMemberUsers } = await getOrganizationTeams( client, org, ); - buildOrgHierarchy(groups, users, groupMemberUsers); + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${users.length} GitHub users and ${groups.length} GitHub groups in ${duration} seconds`, + ); + + // Fill out the hierarchy + const usersByName = new Map(users.map(u => [u.metadata.name, u])); + for (const [groupName, userNames] of groupMemberUsers.entries()) { + for (const userName of userNames) { + const user = usersByName.get(userName); + if (user && !user.spec.memberOf.includes(groupName)) { + user.spec.memberOf.push(groupName); + } + } + } + buildOrgHierarchy(groups); + + // Done! for (const group of groups) { emit(results.entity(location, group)); } diff --git a/plugins/catalog-backend/src/ingestion/processors/LdapOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/LdapOrgReaderProcessor.ts new file mode 100644 index 0000000000..9e083889f2 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/LdapOrgReaderProcessor.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { Logger } from 'winston'; +import { + LdapClient, + LdapProviderConfig, + readLdapConfig, + readLdapOrg, +} from './ldap'; +import * as results from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +/** + * Extracts teams and users out of an LDAP server. + */ +export class LdapOrgReaderProcessor implements LocationProcessor { + static fromConfig(config: Config, logger: Logger) { + const c = config.getOptionalConfig('catalog.processors.ldapOrg'); + return new LdapOrgReaderProcessor(c ? readLdapConfig(c) : [], logger); + } + + constructor( + private readonly providers: LdapProviderConfig[], + private readonly logger: Logger, + ) {} + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'ldap-org') { + return false; + } + + const provider = this.providers.find(p => location.target === p.target); + if (!provider) { + throw new Error( + `There is no LDAP Org provider that matches ${location.target}. Please add a configuration entry for it under catalog.processors.ldapOrg.providers.`, + ); + } + + // Read out all of the raw data + const startTimestamp = Date.now(); + this.logger.info('Reading LDAP users and groups'); + + // Be lazy and create the client each time; even though it's pretty + // inefficient, we usually only do this once per entire refresh loop and + // don't have to worry about timeouts and reconnects etc. + const client = await LdapClient.create(provider.target, provider.bind); + const { users, groups } = await readLdapOrg( + client, + provider.users, + provider.groups, + ); + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${users.length} LDAP users and ${groups.length} LDAP groups in ${duration} seconds`, + ); + + // Done! + for (const group of groups) { + emit(results.entity(location, group)); + } + for (const user of users) { + emit(results.entity(location, user)); + } + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/client.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/client.ts new file mode 100644 index 0000000000..7c1fc629c6 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/client.ts @@ -0,0 +1,92 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ldap, { Client, SearchEntry, SearchOptions } from 'ldapjs'; +import { BindConfig } from './config'; +import { errorString } from './util'; + +/** + * Basic wrapper for the ldapjs library. + * + * Helps out with promisifying calls, paging, binding etc. + */ +export class LdapClient { + static create(target: string, bind?: BindConfig): Promise { + return new Promise((resolve, reject) => { + const client = ldap.createClient({ url: target }); + if (!bind) { + resolve(new LdapClient(client)); + return; + } + + const { dn, secret } = bind; + client.bind(dn, secret, err => { + if (err) { + reject(`LDAP bind failed for ${dn}, ${errorString(err)}`); + } else { + resolve(new LdapClient(client)); + } + }); + }); + } + + constructor(private readonly client: Client) {} + + /** + * Performs an LDAP search operation. + * + * @param dn The fully qualified base DN to search within + * @param options The search options + */ + async search(dn: string, options: SearchOptions): Promise { + try { + return await new Promise((resolve, reject) => { + const output: SearchEntry[] = []; + + this.client.search(dn, options, (err, res) => { + if (err) { + reject(errorString(err)); + return; + } + + res.on('searchReference', () => { + reject('Unable to handle referral'); + }); + + res.on('searchEntry', entry => { + output.push(entry); + }); + + res.on('error', e => { + reject(errorString(e)); + }); + + res.on('end', r => { + if (!r) { + reject('Null response'); + } else if (r.status !== 0) { + reject(`Got status ${r.status}: ${r.errorMessage}`); + } else { + resolve(output); + } + }); + }); + }); + } catch (e) { + throw new Error(`LDAP search at ${dn} failed, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/config.test.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/config.test.ts new file mode 100644 index 0000000000..7e6036668f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/config.test.ts @@ -0,0 +1,172 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readLdapConfig } from './config'; + +describe('readLdapConfig', () => { + it('applies all of the defaults', () => { + const config = { + providers: [ + { + target: 'target', + users: { + dn: 'udn', + }, + groups: { + dn: 'gdn', + }, + }, + ], + }; + const actual = readLdapConfig( + ConfigReader.fromConfigs([{ context: '', data: config }]), + ); + const expected = [ + { + target: 'target', + bind: undefined, + users: { + dn: 'udn', + options: { + scope: 'one', + attributes: ['*', '+'], + }, + set: undefined, + map: { + rdn: 'uid', + name: 'uid', + displayName: 'cn', + email: 'mail', + memberOf: 'memberOf', + }, + }, + groups: { + dn: 'gdn', + options: { + scope: 'one', + attributes: ['*', '+'], + }, + set: undefined, + map: { + rdn: 'cn', + name: 'cn', + description: 'description', + type: 'groupType', + memberOf: 'memberOf', + members: 'member', + }, + }, + }, + ]; + expect(actual).toEqual(expected); + }); + + it('reads all the values', () => { + const config = { + providers: [ + { + target: 'target', + bind: { dn: 'bdn', secret: 's' }, + users: { + dn: 'udn', + options: { + scope: 'base', + attributes: ['*'], + filter: 'f', + paged: true, + }, + set: [{ path: 'p', value: 'v' }], + map: { + rdn: 'u', + name: 'v', + description: 'd', + displayName: 'c', + email: 'm', + picture: 'p', + memberOf: 'm', + }, + }, + groups: { + dn: 'gdn', + options: { + scope: 'base', + attributes: ['*'], + filter: 'f', + paged: true, + }, + set: [{ path: 'p', value: 'v' }], + map: { + rdn: 'u', + name: 'v', + description: 'd', + type: 't', + memberOf: 'm', + members: 'n', + }, + }, + }, + ], + }; + const actual = readLdapConfig( + ConfigReader.fromConfigs([{ context: '', data: config }]), + ); + const expected = [ + { + target: 'target', + bind: { dn: 'bdn', secret: 's' }, + users: { + dn: 'udn', + options: { + scope: 'base', + attributes: ['*'], + filter: 'f', + paged: true, + }, + set: [{ path: 'p', value: 'v' }], + map: { + rdn: 'u', + name: 'v', + description: 'd', + displayName: 'c', + email: 'm', + picture: 'p', + memberOf: 'm', + }, + }, + groups: { + dn: 'gdn', + options: { + scope: 'base', + attributes: ['*'], + filter: 'f', + paged: true, + }, + set: [{ path: 'p', value: 'v' }], + map: { + rdn: 'u', + name: 'v', + description: 'd', + type: 't', + memberOf: 'm', + members: 'n', + }, + }, + }, + ]; + expect(actual).toEqual(expected); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/config.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/config.ts new file mode 100644 index 0000000000..74c02e3f84 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/config.ts @@ -0,0 +1,264 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config, JsonValue } from '@backstage/config'; +import { SearchOptions } from 'ldapjs'; +import mergeWith from 'lodash/mergeWith'; +import { RecursivePartial } from './util'; + +/** + * The configuration parameters for a single LDAP provider. + */ +export type LdapProviderConfig = { + // The prefix of the target that this matches on, e.g. + // "ldaps://ds.example.net", with no trailing slash. + target: string; + // The settings to use for the bind command. If none are specified, the bind + // command is not issued. + bind?: BindConfig; + // The settings that govern the reading and interpretation of users + users: UserConfig; + // The settings that govern the reading and interpretation of groups + groups: GroupConfig; +}; + +/** + * The settings to use for the a command. + */ +export type BindConfig = { + // The DN of the user to auth as, e.g. + // uid=ldap-robot,ou=robots,ou=example,dc=example,dc=net + dn: string; + // The secret of the user to auth as (its password) + secret: string; +}; + +/** + * The settings that govern the reading and interpretation of users. + */ +export type UserConfig = { + // The DN under which users are stored. + dn: string; + // The search options to use. + // Only the scope, filter, attributes, and paged fields are supported. The + // default is scope "one" and attributes "*" and "+". + options: SearchOptions; + // JSON paths (on a.b.c form) and hard coded values to set on those paths + set?: { path: string; value: JsonValue }[]; + // Mappings from well known entity fields, to LDAP attribute names + map: { + // The name of the attribute that holds the relative distinguished name of + // each entry. Defaults to "uid". + rdn: string; + // The name of the attribute that shall be used for the value of the + // metadata.name field of the entity. Defaults to "uid". + name: string; + // The name of the attribute that shall be used for the value of the + // metadata.description field of the entity. + description?: string; + // The name of the attribute that shall be used for the value of the + // spec.profile.displayName field of the entity. Defaults to "cn". + displayName: string; + // The name of the attribute that shall be used for the value of the + // spec.profile.email field of the entity. Defaults to "mail". + email: string; + // The name of the attribute that shall be used for the value of the + // spec.profile.picture field of the entity. + picture?: string; + // The name of the attribute that shall be used for the values of the + // spec.memberOf field of the entity. Defaults to "memberOf". + memberOf: string; + }; +}; + +/** + * The settings that govern the reading and interpretation of groups. + */ +export type GroupConfig = { + // The DN under which groups are stored. + dn: string; + // The search options to use. + // Only the scope, filter, attributes, and paged fields are supported. + options: SearchOptions; + // JSON paths (on a.b.c form) and hard coded values to set on those paths + set?: { path: string; value: JsonValue }[]; + // Mappings from well known entity fields, to LDAP attribute names + map: { + // The name of the attribute that holds the relative distinguished name of + // each entry. Defaults to "cn". + rdn: string; + // The name of the attribute that shall be used for the value of the + // metadata.name field of the entity. Defaults to "cn". + name: string; + // The name of the attribute that shall be used for the value of the + // metadata.description field of the entity. Defaults to "description". + description: string; + // The name of the attribute that shall be used for the value of the + // spec.type field of the entity. Defaults to "groupType". + type: string; + // The name of the attribute that shall be used for the values of the + // spec.parent field of the entity. Defaults to "memberOf". + memberOf: string; + // The name of the attribute that shall be used for the values of the + // spec.children field of the entity. Defaults to "member". + members: string; + }; +}; + +const defaultConfig = { + users: { + options: { + scope: 'one', + attributes: ['*', '+'], + }, + map: { + rdn: 'uid', + name: 'uid', + displayName: 'cn', + email: 'mail', + memberOf: 'memberOf', + }, + }, + groups: { + options: { + scope: 'one', + attributes: ['*', '+'], + }, + map: { + rdn: 'cn', + name: 'cn', + description: 'description', + type: 'groupType', + memberOf: 'memberOf', + members: 'member', + }, + }, +}; + +/** + * Parses configuration. + * + * @param config The root of the LDAP config hierarchy + */ +export function readLdapConfig(config: Config): LdapProviderConfig[] { + function readBindConfig( + c: Config | undefined, + ): LdapProviderConfig['bind'] | undefined { + if (!c) { + return undefined; + } + return { + dn: c.getString('dn'), + secret: c.getString('secret'), + }; + } + + function readOptionsConfig(c: Config | undefined): SearchOptions { + if (!c) { + return {}; + } + return { + scope: c.getOptionalString('scope') as SearchOptions['scope'], + filter: c.getOptionalString('filter'), + attributes: c.getOptionalStringArray('attributes'), + paged: c.getOptionalBoolean('paged'), + }; + } + + function readSetConfig( + c: Config[] | undefined, + ): { path: string; value: JsonValue }[] | undefined { + if (!c) { + return undefined; + } + return c.map(entry => ({ + path: entry.getString('path'), + value: entry.get('value'), + })); + } + + function readUserMapConfig( + c: Config | undefined, + ): Partial { + if (!c) { + return {}; + } + + return { + rdn: c.getOptionalString('rdn'), + name: c.getOptionalString('name'), + description: c.getOptionalString('description'), + displayName: c.getOptionalString('displayName'), + email: c.getOptionalString('email'), + picture: c.getOptionalString('picture'), + memberOf: c.getOptionalString('memberOf'), + }; + } + + function readGroupMapConfig( + c: Config | undefined, + ): Partial { + if (!c) { + return {}; + } + + return { + rdn: c.getOptionalString('rdn'), + name: c.getOptionalString('name'), + description: c.getOptionalString('description'), + type: c.getOptionalString('type'), + memberOf: c.getOptionalString('memberOf'), + members: c.getOptionalString('members'), + }; + } + + function readUserConfig( + c: Config, + ): RecursivePartial { + return { + dn: c.getString('dn'), + options: readOptionsConfig(c.getOptionalConfig('options')), + set: readSetConfig(c.getOptionalConfigArray('set')), + map: readUserMapConfig(c.getOptionalConfig('map')), + }; + } + + function readGroupConfig( + c: Config, + ): RecursivePartial { + return { + dn: c.getString('dn'), + options: readOptionsConfig(c.getOptionalConfig('options')), + set: readSetConfig(c.getOptionalConfigArray('set')), + map: readGroupMapConfig(c.getOptionalConfig('map')), + }; + } + + const providerConfigs = config.getOptionalConfigArray('providers') ?? []; + return providerConfigs.map(c => { + const newConfig = { + target: c.getString('target').replace(/\/+$/, ''), + bind: readBindConfig(c.getOptionalConfig('bind')), + users: readUserConfig(c.getConfig('users')), + groups: readGroupConfig(c.getConfig('groups')), + }; + const merged = mergeWith({}, defaultConfig, newConfig, (_into, from) => { + // Replace arrays instead of merging, otherwise default behavior + return Array.isArray(from) ? from : undefined; + }); + return merged as LdapProviderConfig; + }); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts new file mode 100644 index 0000000000..3f32a34da4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/constants.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The name of an entity annotation, that references the RDN of the LDAP object + * it was ingested from. + * + * The RDN is the name of the leftmost attribute that identifies the item; for + * example, for an item with the fully qualified DN + * uid=john,ou=people,ou=spotify,dc=spotify,dc=net the generated entity would + * have this attribute, with the value "john". + */ +export const LDAP_RDN_ANNOTATION = 'backstage.io/ldap-rdn'; + +/** + * The name of an entity annotation, that references the DN of the LDAP object + * it was ingested from. + * + * The DN is the fully qualified name that identifies the item; for example, + * for an item with the DN uid=john,ou=people,ou=spotify,dc=spotify,dc=net the + * generated entity would have this attribute, with that full string as its + * value. + */ +export const LDAP_DN_ANNOTATION = 'backstage.io/ldap-dn'; + +/** + * The name of an entity annotation, that references the UUID of the LDAP + * object it was ingested from. + * + * The UUID is the globally unique ID that identifies the item; for example, + * for an item with the UUID 76ef928a-b251-1037-9840-d78227f36a7e, the + * generated entity would have this attribute, with that full string as its + * value. + */ +export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid'; diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/index.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/index.ts new file mode 100644 index 0000000000..2f8dae7486 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { LdapClient } from './client'; +export { readLdapConfig } from './config'; +export type { LdapProviderConfig } from './config'; +export { + LDAP_DN_ANNOTATION, + LDAP_RDN_ANNOTATION, + LDAP_UUID_ANNOTATION, +} from './constants'; +export { readLdapOrg } from './read'; diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/read.test.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/read.test.ts new file mode 100644 index 0000000000..e7a7d1c567 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/read.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { SearchEntry } from 'ldapjs'; +import merge from 'lodash/merge'; +import { LdapClient } from './client'; +import { GroupConfig, UserConfig } from './config'; +import { + LDAP_DN_ANNOTATION, + LDAP_RDN_ANNOTATION, + LDAP_UUID_ANNOTATION, +} from './constants'; +import { readLdapGroups, readLdapUsers, resolveRelations } from './read'; +import { RecursivePartial } from './util'; + +function user(data: RecursivePartial): UserEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'name' }, + spec: { profile: {}, memberOf: [] }, + } as UserEntity, + data, + ); +} + +function group(data: RecursivePartial): GroupEntity { + return merge( + {}, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { name: 'name' }, + spec: { type: 'type', ancestors: [], children: [], descendants: [] }, + } as GroupEntity, + data, + ); +} + +function searchEntry(attributes: Record): SearchEntry { + return { + attributes: Object.entries(attributes).map(([k, vs]) => ({ + json: { + type: k, + vals: vs, + }, + })), + } as any; +} + +describe('readLdapUsers', () => { + const client: jest.Mocked = { + search: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + it('transfers all attributes', async () => { + client.search.mockResolvedValue([ + searchEntry({ + uid: ['uid-value'], + description: ['description-value'], + cn: ['cn-value'], + mail: ['mail-value'], + avatarUrl: ['avatarUrl-value'], + memberOf: ['x', 'y', 'z'], + entryDN: ['dn-value'], + entryUUID: ['uuid-value'], + }), + ]); + const config: UserConfig = { + dn: 'ddd', + options: {}, + map: { + rdn: 'uid', + name: 'uid', + description: 'description', + displayName: 'cn', + email: 'mail', + picture: 'avatarUrl', + memberOf: 'memberOf', + }, + }; + const { users, userMemberOf } = await readLdapUsers(client, config); + expect(users).toEqual([ + expect.objectContaining({ + metadata: { + name: 'uid-value', + description: 'description-value', + annotations: { + [LDAP_DN_ANNOTATION]: 'dn-value', + [LDAP_RDN_ANNOTATION]: 'uid-value', + [LDAP_UUID_ANNOTATION]: 'uuid-value', + }, + }, + spec: { + profile: { + displayName: 'cn-value', + email: 'mail-value', + picture: 'avatarUrl-value', + }, + memberOf: [], + }, + }), + ]); + expect(userMemberOf).toEqual( + new Map([['dn-value', new Set(['x', 'y', 'z'])]]), + ); + }); +}); + +describe('readLdapGroups', () => { + const client: jest.Mocked = { + search: jest.fn(), + } as any; + + afterEach(() => jest.resetAllMocks()); + + it('transfers all attributes', async () => { + client.search.mockResolvedValue([ + searchEntry({ + cn: ['cn-value'], + description: ['description-value'], + tt: ['type-value'], + memberOf: ['x', 'y', 'z'], + member: ['e', 'f', 'g'], + entryDN: ['dn-value'], + entryUUID: ['uuid-value'], + }), + ]); + const config: GroupConfig = { + dn: 'ddd', + options: {}, + map: { + rdn: 'cn', + name: 'cn', + description: 'description', + type: 'tt', + memberOf: 'memberOf', + members: 'member', + }, + }; + const { groups, groupMember, groupMemberOf } = await readLdapGroups( + client, + config, + ); + expect(groups).toEqual([ + expect.objectContaining({ + metadata: { + name: 'cn-value', + description: 'description-value', + annotations: { + [LDAP_DN_ANNOTATION]: 'dn-value', + [LDAP_RDN_ANNOTATION]: 'cn-value', + [LDAP_UUID_ANNOTATION]: 'uuid-value', + }, + }, + spec: { + type: 'type-value', + ancestors: [], + children: [], + descendants: [], + }, + }), + ]); + expect(groupMember).toEqual( + new Map([['dn-value', new Set(['e', 'f', 'g'])]]), + ); + expect(groupMemberOf).toEqual( + new Map([['dn-value', new Set(['x', 'y', 'z'])]]), + ); + }); +}); + +describe('resolveRelations', () => { + describe('lookup', () => { + it('matches by DN', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_DN_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_DN_ANNOTATION]: 'ca' }, + }, + }); + const groupMember = new Map>([ + ['pa', new Set(['ca'])], + ]); + resolveRelations([parent, child], [], new Map(), new Map(), groupMember); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + }); + it('matches by UUID', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_UUID_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_UUID_ANNOTATION]: 'ca' }, + }, + }); + const groupMember = new Map>([ + ['pa', new Set(['ca'])], + ]); + resolveRelations([parent, child], [], new Map(), new Map(), groupMember); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + }); + }); + + describe('userMemberOf', () => { + it('populates relations by dn', () => { + const host = group({ + metadata: { name: 'host', annotations: { [LDAP_DN_ANNOTATION]: 'ha' } }, + }); + const member = user({ + metadata: { + name: 'member', + annotations: { [LDAP_DN_ANNOTATION]: 'ma' }, + }, + }); + const userMemberOf = new Map>([ + ['ma', new Set(['ha'])], + ]); + resolveRelations([host], [member], userMemberOf, new Map(), new Map()); + expect(member.spec.memberOf).toEqual(['host']); + }); + + it('populates relations by uuid', () => { + const host = group({ + metadata: { + name: 'host', + annotations: { [LDAP_UUID_ANNOTATION]: 'ha' }, + }, + }); + const member = user({ + metadata: { + name: 'member', + annotations: { [LDAP_DN_ANNOTATION]: 'ma' }, + }, + }); + const userMemberOf = new Map>([ + ['ma', new Set(['ha'])], + ]); + resolveRelations([host], [member], userMemberOf, new Map(), new Map()); + expect(member.spec.memberOf).toEqual(['host']); + }); + }); + + describe('groupMemberOf', () => { + it('populates relations by dn', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_DN_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_DN_ANNOTATION]: 'ca' }, + }, + }); + const groupMemberOf = new Map>([ + ['ca', new Set(['pa'])], + ]); + resolveRelations( + [parent, child], + [], + new Map(), + groupMemberOf, + new Map(), + ); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + }); + }); + + it('populates relations by uuid', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_UUID_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_UUID_ANNOTATION]: 'ca' }, + }, + }); + const groupMemberOf = new Map>([ + ['ca', new Set(['pa'])], + ]); + resolveRelations([parent, child], [], new Map(), groupMemberOf, new Map()); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + }); + + describe('groupMember', () => { + it('populates relations by dn', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_DN_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_DN_ANNOTATION]: 'ca' }, + }, + }); + const member = user({ + metadata: { + name: 'member', + annotations: { [LDAP_DN_ANNOTATION]: 'ma' }, + }, + }); + const groupMember = new Map>([ + ['pa', new Set(['ca', 'ma'])], + ]); + resolveRelations( + [parent, child], + [member], + new Map(), + new Map(), + groupMember, + ); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + expect(member.spec.memberOf).toEqual(['parent']); + }); + + it('populates relations by uuid', () => { + const parent = group({ + metadata: { + name: 'parent', + annotations: { [LDAP_UUID_ANNOTATION]: 'pa' }, + }, + }); + const child = group({ + metadata: { + name: 'child', + annotations: { [LDAP_UUID_ANNOTATION]: 'ca' }, + }, + }); + const member = user({ + metadata: { + name: 'member', + annotations: { [LDAP_UUID_ANNOTATION]: 'ma' }, + }, + }); + const groupMember = new Map>([ + ['pa', new Set(['ca', 'ma'])], + ]); + resolveRelations( + [parent, child], + [member], + new Map(), + new Map(), + groupMember, + ); + expect(parent.spec.children).toEqual(['child']); + expect(child.spec.parent).toEqual('parent'); + expect(member.spec.memberOf).toEqual(['parent']); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/read.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/read.ts new file mode 100644 index 0000000000..63fce00bd8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/read.ts @@ -0,0 +1,422 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import lodashSet from 'lodash/set'; +import { buildOrgHierarchy } from '../util/org'; +import { LdapClient } from './client'; +import { GroupConfig, UserConfig } from './config'; +import { + LDAP_DN_ANNOTATION, + LDAP_RDN_ANNOTATION, + LDAP_UUID_ANNOTATION, +} from './constants'; + +// NOTE(freben): These attribute names are assumed to be the same across all +// compliant LDAP implementations +const DN_ATTRIBUTE = 'entryDN'; +const UUID_ATTRIBUTE = 'entryUUID'; + +/** + * Reads groups out of an LDAP provider. + * + * @param client The LDAP client + * @param config The user data configuration + */ +export async function readLdapUsers( + client: LdapClient, + config: UserConfig, +): Promise<{ + users: UserEntity[]; // With all relations empty + userMemberOf: Map>; // DN -> DN or UUID of groups +}> { + const { dn, options, set, map } = config; + + const entries = await client.search(dn, options); + + const entities: UserEntity[] = []; + const userMemberOf: Map> = new Map(); + + for (const entry of entries) { + const attributes = new Map( + entry.attributes.map(attr => { + const data = attr.json; + return [data.type, data.vals]; + }), + ); + + const entity: UserEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { + name: '', + annotations: {}, + }, + spec: { + profile: {}, + memberOf: [], + }, + }; + + if (set) { + for (const { path, value } of set) { + lodashSet(entity, path, value); + } + } + + mapStringAttr(attributes, map.name, v => { + entity.metadata.name = v; + }); + mapStringAttr(attributes, map.description, v => { + entity.metadata.description = v; + }); + mapStringAttr(attributes, map.rdn, v => { + entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; + }); + mapStringAttr(attributes, UUID_ATTRIBUTE, v => { + entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; + }); + mapStringAttr(attributes, DN_ATTRIBUTE, v => { + entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; + }); + mapStringAttr(attributes, map.displayName, v => { + entity.spec.profile!.displayName = v; + }); + mapStringAttr(attributes, map.email, v => { + entity.spec.profile!.email = v; + }); + mapStringAttr(attributes, map.picture, v => { + entity.spec.profile!.picture = v; + }); + + mapReferencesAttr(attributes, map.memberOf, (myDn, vs) => { + ensureItems(userMemberOf, myDn, vs); + }); + + entities.push(entity); + } + + return { users: entities, userMemberOf }; +} + +/** + * Reads groups out of an LDAP provider. + * + * @param client The LDAP client + * @param config The group data configuration + */ +export async function readLdapGroups( + client: LdapClient, + config: GroupConfig, +): Promise<{ + groups: GroupEntity[]; // With all relations empty + groupMemberOf: Map>; // DN -> DN or UUID of groups + groupMember: Map>; // DN -> DN or UUID of groups & users +}> { + const { dn, options, set, map } = config; + const entries = await client.search(dn, options); + + const groups: GroupEntity[] = []; + const groupMemberOf: Map> = new Map(); + const groupMember: Map> = new Map(); + + for (const entry of entries) { + const attributes = new Map( + entry.attributes.map(attr => { + const data = attr.json; + return [data.type, data.vals]; + }), + ); + + const entity: GroupEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: '', + annotations: {}, + }, + spec: { + type: 'unknown', + ancestors: [], + children: [], + descendants: [], + }, + }; + + if (set) { + for (const { path, value } of set) { + lodashSet(entity, path, value); + } + } + + mapStringAttr(attributes, map.name, v => { + entity.metadata.name = v; + }); + mapStringAttr(attributes, map.description, v => { + entity.metadata.description = v; + }); + mapStringAttr(attributes, map.rdn, v => { + entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v; + }); + mapStringAttr(attributes, UUID_ATTRIBUTE, v => { + entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v; + }); + mapStringAttr(attributes, DN_ATTRIBUTE, v => { + entity.metadata.annotations![LDAP_DN_ANNOTATION] = v; + }); + mapStringAttr(attributes, map.type, v => { + entity.spec.type = v; + }); + + mapReferencesAttr(attributes, map.memberOf, (myDn, vs) => { + ensureItems(groupMemberOf, myDn, vs); + }); + mapReferencesAttr(attributes, map.members, (myDn, vs) => { + ensureItems(groupMember, myDn, vs); + }); + + groups.push(entity); + } + + return { + groups, + groupMemberOf, + groupMember, + }; +} + +/** + * Reads users and groups out of an LDAP provider. + * + * Invokes the above "raw" read functions and stitches together the results + * with all relations etc filled in. + * + * @param client The LDAP client + * @param logger A logger instance + * @param userConfig The user data configuration + * @param groupConfig The group data configuration + */ +export async function readLdapOrg( + client: LdapClient, + userConfig: UserConfig, + groupConfig: GroupConfig, +): Promise<{ + users: UserEntity[]; + groups: GroupEntity[]; +}> { + const { users, userMemberOf } = await readLdapUsers(client, userConfig); + const { groups, groupMemberOf, groupMember } = await readLdapGroups( + client, + groupConfig, + ); + + resolveRelations(groups, users, userMemberOf, groupMemberOf, groupMember); + users.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + groups.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + + return { users, groups }; +} + +// +// Helpers +// + +// Maps a single-valued attribute to a consumer +function mapStringAttr( + attributes: Map, + attributeName: string | undefined, + setter: (value: string) => void, +) { + if (attributeName) { + const values = attributes.get(attributeName); + if (values && values.length === 1) { + setter(values[0]); + } + } +} + +// Maps a multi-valued attribute of references to other objects, to a consumer +function mapReferencesAttr( + attributes: Map, + attributeName: string | undefined, + setter: (sourceDn: string, targets: string[]) => void, +) { + if (attributeName) { + const values = attributes.get(attributeName); + const dn = attributes.get(DN_ATTRIBUTE); + if (values && dn && dn.length === 1) { + setter(dn[0], values); + } + } +} + +// Inserts a number of values in a key-values mapping +function ensureItems( + target: Map>, + key: string, + values: string[], +) { + if (key) { + let set = target.get(key); + if (!set) { + set = new Set(); + target.set(key, set); + } + for (const value of values) { + if (value) { + set!.add(value); + } + } + } +} + +/** + * Takes groups and entities with empty relations, and fills in the various + * relations that were returned by the readers, and forms the org hierarchy. + * + * @param groups Group entities with empty relations; modified in place + * @param users User entities with empty relations; modified in place + * @param userMemberOf For a user DN, the set of group DNs or UUIDs that the + * user is a member of + * @param groupMemberOf For a group DN, the set of group DNs or UUIDs that the + * group is a member of (parents in the hierarchy) + * @param groupMember For a group DN, the set of group DNs or UUIDs that are + * members of the group (children in the hierarchy) + */ +export function resolveRelations( + groups: GroupEntity[], + users: UserEntity[], + userMemberOf: Map>, + groupMemberOf: Map>, + groupMember: Map>, +) { + // Build reference lookup tables - all of the relations that are output from + // the above calls can be expressed as either DNs or UUIDs so we need to be + // able to find by both, as well as the name. Note that we expect them to not + // collide here - this is a reasonable assumption as long as the fields are + // the supported forms. + const userMap: Map = new Map(); // by name, dn, uuid + const groupMap: Map = new Map(); // by name, dn, uuid + for (const user of users) { + userMap.set(user.metadata.name, user); + userMap.set(user.metadata.annotations![LDAP_DN_ANNOTATION], user); + userMap.set(user.metadata.annotations![LDAP_UUID_ANNOTATION], user); + } + for (const group of groups) { + groupMap.set(group.metadata.name, group); + groupMap.set(group.metadata.annotations![LDAP_DN_ANNOTATION], group); + groupMap.set(group.metadata.annotations![LDAP_UUID_ANNOTATION], group); + } + + // This can happen e.g. if entryUUID wasn't returned by the server + userMap.delete(''); + groupMap.delete(''); + userMap.delete(undefined!); + groupMap.delete(undefined!); + + // Fill in all of the immediate relations, now keyed on metadata.name. We + // keep all parents at this point, whether the target model can support more + // than one or not (it gets filtered farther down). And group children are + // only groups in here. + const newUserMemberOf: Map> = new Map(); + const newGroupParents: Map> = new Map(); + const newGroupChildren: Map> = new Map(); + + // Resolve and store in the intermediaries. It may seem redundant that the + // input data has both parent and children directions, as well as both + // user->group and group->user - the reason is that different LDAP schemas + // express relations in different directions. Some may have a user memberOf + // overlay, some don't, for example. + for (const [userN, groupsN] of userMemberOf.entries()) { + const user = userMap.get(userN); + if (user) { + for (const groupN of groupsN) { + const group = groupMap.get(groupN); + if (group) { + ensureItems(newUserMemberOf, user.metadata.name, [ + group.metadata.name, + ]); + } + } + } + } + for (const [groupN, parentsN] of groupMemberOf.entries()) { + const group = groupMap.get(groupN); + if (group) { + for (const parentN of parentsN) { + const parentGroup = groupMap.get(parentN); + if (parentGroup) { + ensureItems(newGroupParents, group.metadata.name, [ + parentGroup.metadata.name, + ]); + ensureItems(newGroupChildren, parentGroup.metadata.name, [ + group.metadata.name, + ]); + } + } + } + } + for (const [groupN, membersN] of groupMember.entries()) { + const group = groupMap.get(groupN); + if (group) { + for (const memberN of membersN) { + // Group members can be both users and groups in the input model, so + // try both + const memberUser = userMap.get(memberN); + if (memberUser) { + ensureItems(newUserMemberOf, memberUser.metadata.name, [ + group.metadata.name, + ]); + } else { + const memberGroup = groupMap.get(memberN); + if (memberGroup) { + ensureItems(newGroupChildren, group.metadata.name, [ + memberGroup.metadata.name, + ]); + ensureItems(newGroupParents, memberGroup.metadata.name, [ + group.metadata.name, + ]); + } + } + } + } + } + + // Write down the relations again into the actual entities + for (const [userN, groupsN] of newUserMemberOf.entries()) { + const user = userMap.get(userN); + if (user) { + user.spec.memberOf = Array.from(groupsN).sort(); + } + } + for (const [groupN, parentsN] of newGroupParents.entries()) { + if (parentsN.size === 1) { + const group = groupMap.get(groupN); + if (group) { + group.spec.parent = parentsN.values().next().value; + } + } + } + for (const [groupN, childrenN] of newGroupChildren.entries()) { + const group = groupMap.get(groupN); + if (group) { + group.spec.children = Array.from(childrenN).sort(); + } + } + + // Fill out the rest of the hierarchy + buildOrgHierarchy(groups); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/util.test.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/util.test.ts new file mode 100644 index 0000000000..2def6ad960 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/util.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { errorString, RecursivePartial } from './util'; + +describe('errorString', () => { + it('formats', () => { + const e = { code: 1, name: 'n', message: 'm' }; + expect(errorString(e)).toEqual('1 n: m'); + }); +}); + +describe('RecursivePartial', () => { + it('is recursive', () => { + type X = { + required: { + required: string; + }; + }; + const x: RecursivePartial = { + required: {}, + }; + expect(x).toEqual({ required: {} }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/ldap/util.ts b/plugins/catalog-backend/src/ingestion/processors/ldap/util.ts new file mode 100644 index 0000000000..29159dc219 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/ldap/util.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Error as LDAPError } from 'ldapjs'; + +/** + * Builds a string form of an LDAP Error structure. + * + * @param error The error + */ +export function errorString(error: LDAPError) { + return `${error.code} ${error.name}: ${error.message}`; +} + +/** + * Makes all keys of an entire hierarchy optional. + */ +export type RecursivePartial = { + [P in keyof T]?: T[P] extends (infer U)[] + ? RecursivePartial[] + : T[P] extends object + ? RecursivePartial + : T[P]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts index e0120735fe..e9dddc8226 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts @@ -14,18 +14,9 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { GroupEntity } from '@backstage/catalog-model'; import { buildOrgHierarchy } from './org'; -function u(name: string): UserEntity { - return { - apiVersion: 'backstage.io/v1alpha1', - kind: 'User', - metadata: { name }, - spec: { memberOf: [] }, - }; -} - function g( name: string, parent: string | undefined, @@ -40,26 +31,12 @@ function g( } describe('buildOrgHierarchy', () => { - it('puts users in the respective groups', () => { - const a = g('a', undefined, []); - const b = g('b', undefined, []); - const x = u('x'); - const y = u('y'); - const groupMemberUsers: Map = new Map([ - ['a', ['x', 'y']], - ['b', ['y']], - ]); - buildOrgHierarchy([a, b], [x, y], groupMemberUsers); - expect(x.spec.memberOf).toEqual(['a']); - expect(y.spec.memberOf).toEqual(['a', 'b']); - }); - it('adds groups to their parent.children', () => { const a = g('a', undefined, []); const b = g('b', 'a', []); const c = g('c', 'b', []); const d = g('d', 'a', []); - buildOrgHierarchy([a, b, c, d], [], new Map()); + buildOrgHierarchy([a, b, c, d]); expect(a.spec.children).toEqual(expect.arrayContaining(['b', 'd'])); expect(b.spec.children).toEqual(expect.arrayContaining(['c'])); expect(c.spec.children).toEqual([]); @@ -71,7 +48,7 @@ describe('buildOrgHierarchy', () => { const b = g('b', 'a', []); const c = g('c', 'b', []); const d = g('d', 'a', []); - buildOrgHierarchy([a, b, c, d], [], new Map()); + buildOrgHierarchy([a, b, c, d]); expect(a.spec.descendants).toEqual(expect.arrayContaining(['b', 'c', 'd'])); expect(b.spec.descendants).toEqual(expect.arrayContaining(['c'])); expect(c.spec.descendants).toEqual([]); @@ -83,7 +60,7 @@ describe('buildOrgHierarchy', () => { const b = g('b', 'a', []); const c = g('c', 'b', []); const d = g('d', 'a', []); - buildOrgHierarchy([a, b, c, d], [], new Map()); + buildOrgHierarchy([a, b, c, d]); expect(a.spec.ancestors).toEqual([]); expect(b.spec.ancestors).toEqual(expect.arrayContaining(['a'])); expect(c.spec.ancestors).toEqual(expect.arrayContaining(['a', 'b'])); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/ingestion/processors/util/org.ts index bcc4c7b5e6..a280a265a8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/org.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/org.ts @@ -14,28 +14,10 @@ * limitations under the License. */ -import { GroupEntity, UserEntity } from '@backstage/catalog-model'; +import { GroupEntity } from '@backstage/catalog-model'; -export function buildOrgHierarchy( - groups: GroupEntity[], - users: UserEntity[], - groupMemberUsers: Map, -) { +export function buildOrgHierarchy(groups: GroupEntity[]) { const groupsByName = new Map(groups.map(g => [g.metadata.name, g])); - const usersByName = new Map(users.map(u => [u.metadata.name, u])); - - // - // Make sure that u.memberOf contain all g - // - - for (const [groupName, userNames] of groupMemberUsers.entries()) { - for (const userName of userNames) { - const user = usersByName.get(userName); - if (user && !user.spec.memberOf.includes(groupName)) { - user.spec.memberOf.push(groupName); - } - } - } // // Make sure that g.parent.children contain g diff --git a/yarn.lock b/yarn.lock index fe36f17fd2..a3f62e6e47 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5121,6 +5121,13 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/ldapjs@^1.0.9": + version "1.0.9" + resolved "https://registry.npmjs.org/@types/ldapjs/-/ldapjs-1.0.9.tgz#1224192d14cc5ab5218fcea72ebb04489c52cb95" + integrity sha512-3PvY7Drp1zoLbcGlothCAkoc5o6Jp9KvUPwHadlHyKp3yPvyeIh7w2zQc9UXMzgDRkoeGXUEODtbEs5XCh9ZyA== + dependencies: + "@types/node" "*" + "@types/lodash@^4.14.151": version "4.14.161" resolved "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.161.tgz#a21ca0777dabc6e4f44f3d07f37b765f54188b18" @@ -6017,6 +6024,11 @@ abbrev@1: resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abstract-logging@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.0.tgz#08a85814946c98ef06f4256ad470aba1886d4490" + integrity sha512-/oA9z7JszpIioo6J6dB79LVUgJ3eD3cxkAmdCkvWWS+Y9tPtALs1rLqOekLUXUbYqM2fB9TTK0ibAyZJJOP/CA== + accepts@^1.3.5, accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: version "1.3.7" resolved "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" @@ -6701,7 +6713,7 @@ asn1.js@^4.0.0: inherits "^2.0.1" minimalistic-assert "^1.0.0" -asn1@~0.2.0, asn1@~0.2.3: +asn1@^0.2.4, asn1@~0.2.0, asn1@~0.2.3: version "0.2.4" resolved "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== @@ -7254,6 +7266,13 @@ backo2@^1.0.2: resolved "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947" integrity sha1-MasayLEpNjRj41s+u2n038+6eUc= +backoff@^2.5.0: + version "2.5.0" + resolved "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz#f616eda9d3e4b66b8ca7fca79f695722c5f8e26f" + integrity sha1-9hbtqdPktmuMp/ynn2lXIsX44m8= + dependencies: + precond "0.2" + bail@^1.0.0: version "1.0.5" resolved "https://registry.npmjs.org/bail/-/bail-1.0.5.tgz#b6fa133404a392cbc1f8c4bf63f5953351e7a776" @@ -14948,6 +14967,27 @@ lcid@^1.0.0: dependencies: invert-kv "^1.0.0" +ldap-filter@^0.3.3: + version "0.3.3" + resolved "https://registry.npmjs.org/ldap-filter/-/ldap-filter-0.3.3.tgz#2b14c68a2a9d4104dbdbc910a1ca85fd189e9797" + integrity sha1-KxTGiiqdQQTb28kQocqF/Riel5c= + dependencies: + assert-plus "^1.0.0" + +ldapjs@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.2.0.tgz#c23846d35bf215b4ba4721ff22f11d3df6391e2f" + integrity sha512-9+ekbj97nxRYQMRgEm/HYFhFLWSRKah2PnReUfhfM5f62XBeUSFolB+AQ2Jij5tqowpksTnrdNCZLP6C+V3uag== + dependencies: + abstract-logging "^2.0.0" + asn1 "^0.2.4" + assert-plus "^1.0.0" + backoff "^2.5.0" + ldap-filter "^0.3.3" + once "^1.4.0" + vasync "^2.2.0" + verror "^1.8.1" + left-pad@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz#5b8a3a7765dfe001261dde915589e782f8c94d1e" @@ -18390,6 +18430,11 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" +precond@0.2: + version "0.2.3" + resolved "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz#aa9591bcaa24923f1e0f4849d240f47efc1075ac" + integrity sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw= + preferred-pm@^3.0.0: version "3.0.2" resolved "https://registry.npmjs.org/preferred-pm/-/preferred-pm-3.0.2.tgz#bbdbef1014e34a7490349bf70d6d244b8d57a5e1" @@ -23077,12 +23122,19 @@ vary@^1, vary@~1.1.2: resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= +vasync@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/vasync/-/vasync-2.2.0.tgz#cfde751860a15822db3b132bc59b116a4adaf01b" + integrity sha1-z951GGChWCLbOxMrxZsRakra8Bs= + dependencies: + verror "1.10.0" + vendors@^1.0.0: version "1.0.4" resolved "https://registry.npmjs.org/vendors/-/vendors-1.0.4.tgz#e2b800a53e7a29b93506c3cf41100d16c4c4ad8e" integrity sha512-/juG65kTL4Cy2su4P8HjtkTxk6VmJDiOPBufWniqQ6wknac6jNiXS9vU+hO3wgusiyqWlzTbVHi0dyJqRONg3w== -verror@1.10.0: +verror@1.10.0, verror@^1.8.1: version "1.10.0" resolved "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=