Add extension points for the LDAP processor for modifying group and user entities

Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>
2021-07-05 15:09:31 +00:00
parent 9b743a33b8
commit b055ef88aa
8 changed files with 325 additions and 128 deletions
@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-catalog-backend-module-ldap': minor
+---
+
+Add extension points to the `LdapOrgReaderProcessor` to make it possible to do more advanced modifications
+of the ingested users and groups.
@@ -122,8 +122,8 @@ The DN under which users are stored, e.g.
 #### users.options

 The search options to use when sending the query to the server, when reading all
-users. All of the options are shown below, with their default values, but they
-are all optional.
+users. All the options are shown below, with their default values, but they are
+all optional.

 ```yaml
 options:
@@ -158,8 +158,8 @@ set:

 Mappings from well known entity fields, to LDAP attribute names. This is where
 you are able to define how to interpret the attributes of each LDAP result item,
-and to move them into the corresponding entity fields. All of the options are
-shown below, with their default values, but they are all optional.
+and to move them into the corresponding entity fields. All the options are shown
+below, with their default values, but they are all optional.

 If you leave out an optional mapping, it will still be copied using that default
 value. For example, even if you do not put in the field `displayName` in your
@@ -204,8 +204,8 @@ The DN under which groups are stored, e.g.
 #### groups.options

 The search options to use when sending the query to the server, when reading all
-groups. All of the options are shown below, with their default values, but they
-are all optional.
+groups. All the options are shown below, with their default values, but they are
+all optional.

 ```yaml
 options:
@@ -282,3 +282,35 @@ map:
  # the spec.children field of the entity.
  members: member
 ```
+
+## Customize the Processor
+
+In case you want to customize the ingested entities, the
+`LdapOrgReaderProcessor` allows to pass transformers for users and groups.
+
+1. Create a transformer:
+
+```ts
+export async function myGroupTransformer(
+  vendor: LdapVendor,
+  config: GroupConfig,
+  group: SearchEntry,
+): Promise<GroupEntity | undefined> {
+  // Transformations may change namespace, change entity naming pattern, fill
+  // profile with more or other details...
+
+  // Create the group entity on your own, or wrap the default transformer
+  return await defaultGroupTransformer(vendor, config, group);
+}
+```
+
+2. Configure the processor with the transformer:
+
+```ts
+builder.addProcessor(
+  LdapOrgReaderProcessor.fromConfig(config, {
+    logger,
+    groupTransformer: myGroupTransformer,
+  }),
+);
+```
@@ -16,6 +16,15 @@ import { SearchEntry } from 'ldapjs';
 import { SearchOptions } from 'ldapjs';
 import { UserEntity } from '@backstage/catalog-model';

+// @public (undocumented)
+export function defaultGroupTransformer(vendor: LdapVendor, config: GroupConfig, entry: SearchEntry): Promise<GroupEntity | undefined>;
+
+// @public (undocumented)
+export function defaultUserTransformer(vendor: LdapVendor, config: UserConfig, entry: SearchEntry): Promise<UserEntity | undefined>;
+
+// @public
+export type GroupTransformer = (vendor: LdapVendor, config: GroupConfig, group: SearchEntry) => Promise<GroupEntity | undefined>;
+
 // @public
 export const LDAP_DN_ANNOTATION = "backstage.io/ldap-dn";

@@ -40,14 +49,18 @@ export class LdapOrgReaderProcessor implements CatalogProcessor {
    constructor(options: {
        providers: LdapProviderConfig[];
        logger: Logger;
+        groupTransformer?: GroupTransformer;
+        userTransformer?: UserTransformer;
    });
    // (undocumented)
    static fromConfig(config: Config, options: {
        logger: Logger;
+        groupTransformer?: GroupTransformer;
+        userTransformer?: UserTransformer;
    }): LdapOrgReaderProcessor;
    // (undocumented)
    readLocation(location: LocationSpec, _optional: boolean, emit: CatalogProcessorEmit): Promise<boolean>;
-}
+    }

 // @public
 export type LdapProviderConfig = {
@@ -57,15 +70,25 @@ export type LdapProviderConfig = {
    groups: GroupConfig;
 };

+// @public
+export function mapStringAttr(entry: SearchEntry, vendor: LdapVendor, attributeName: string | undefined, setter: (value: string) => void): void;
+
 // @public
 export function readLdapConfig(config: Config): LdapProviderConfig[];

 // @public
-export function readLdapOrg(client: LdapClient, userConfig: UserConfig, groupConfig: GroupConfig): Promise<{
+export function readLdapOrg(client: LdapClient, userConfig: UserConfig, groupConfig: GroupConfig, options: {
+    groupTransformer?: GroupTransformer;
+    userTransformer?: UserTransformer;
+    logger: Logger;
+}): Promise<{
    users: UserEntity[];
    groups: GroupEntity[];
 }>;

+// @public
+export type UserTransformer = (vendor: LdapVendor, config: UserConfig, user: SearchEntry) => Promise<UserEntity | undefined>;
+

 // (No @packageDocumentation comment for this package)

@@ -15,6 +15,7 @@
 */

 export { LdapClient } from './client';
+export { mapStringAttr } from './util';
 export { readLdapConfig } from './config';
 export type { LdapProviderConfig } from './config';
 export {
@@ -22,4 +23,9 @@ export {
  LDAP_RDN_ANNOTATION,
  LDAP_UUID_ANNOTATION,
 } from './constants';
-export { readLdapOrg } from './read';
+export {
+  defaultGroupTransformer,
+  defaultUserTransformer,
+  readLdapOrg,
+} from './read';
+export type { GroupTransformer, UserTransformer } from './types';
@@ -26,74 +26,97 @@ import {
  LDAP_UUID_ANNOTATION,
 } from './constants';
 import { LdapVendor } from './vendors';
+import { Logger } from 'winston';
+import { GroupTransformer, UserTransformer } from './types';
+import { mapStringAttr } from './util';
+
+export async function defaultUserTransformer(
+  vendor: LdapVendor,
+  config: UserConfig,
+  entry: SearchEntry,
+): Promise<UserEntity | undefined> {
+  const { set, map } = config;
+
+  const entity: UserEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'User',
+    metadata: {
+      name: '',
+      annotations: {},
+    },
+    spec: {
+      profile: {},
+      memberOf: [],
+    },
+  };
+
+  if (set) {
+    for (const [path, value] of Object.entries(set)) {
+      lodashSet(entity, path, value);
+    }
+  }
+
+  mapStringAttr(entry, vendor, map.name, v => {
+    entity.metadata.name = v;
+  });
+  mapStringAttr(entry, vendor, map.description, v => {
+    entity.metadata.description = v;
+  });
+  mapStringAttr(entry, vendor, map.rdn, v => {
+    entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => {
+    entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, vendor.dnAttributeName, v => {
+    entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, map.displayName, v => {
+    entity.spec.profile!.displayName = v;
+  });
+  mapStringAttr(entry, vendor, map.email, v => {
+    entity.spec.profile!.email = v;
+  });
+  mapStringAttr(entry, vendor, map.picture, v => {
+    entity.spec.profile!.picture = v;
+  });
+
+  return entity;
+}

 /**
 * Reads users out of an LDAP provider.
 *
 * @param client The LDAP client
 * @param config The user data configuration
+ * @param opts
 */
 export async function readLdapUsers(
  client: LdapClient,
  config: UserConfig,
+  opts?: { transformer?: UserTransformer },
 ): Promise<{
  users: UserEntity[]; // With all relations empty
  userMemberOf: Map<string, Set<string>>; // DN -> DN or UUID of groups
 }> {
-  const { dn, options, set, map } = config;
+  const { dn, options, map } = config;
  const vendor = await client.getVendor();

-  const entries = await client.search(dn, options);
-
  const entities: UserEntity[] = [];
  const userMemberOf: Map<string, Set<string>> = new Map();

-  for (const entry of entries) {
-    const entity: UserEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'User',
-      metadata: {
-        name: '',
-        annotations: {},
-      },
-      spec: {
-        profile: {},
-        memberOf: [],
-      },
-    };
+  const transformer = opts?.transformer ?? defaultUserTransformer;

-    if (set) {
-      for (const [path, value] of Object.entries(set)) {
-        lodashSet(entity, path, value);
-      }
+  const entries = await client.search(dn, options);
+
+  for (const user of entries) {
+    const entity = await transformer(vendor, config, user);
+
+    if (!entity) {
+      continue;
    }

-    mapStringAttr(entry, vendor, map.name, v => {
-      entity.metadata.name = v;
-    });
-    mapStringAttr(entry, vendor, map.description, v => {
-      entity.metadata.description = v;
-    });
-    mapStringAttr(entry, vendor, map.rdn, v => {
-      entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => {
-      entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, vendor.dnAttributeName, v => {
-      entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, map.displayName, v => {
-      entity.spec.profile!.displayName = v;
-    });
-    mapStringAttr(entry, vendor, map.email, v => {
-      entity.spec.profile!.email = v;
-    });
-    mapStringAttr(entry, vendor, map.picture, v => {
-      entity.spec.profile!.picture = v;
-    });
-
-    mapReferencesAttr(entry, vendor, map.memberOf, (myDn, vs) => {
+    mapReferencesAttr(user, vendor, map.memberOf, (myDn, vs) => {
      ensureItems(userMemberOf, myDn, vs);
    });

@@ -103,82 +126,103 @@ export async function readLdapUsers(
  return { users: entities, userMemberOf };
 }

+export async function defaultGroupTransformer(
+  vendor: LdapVendor,
+  config: GroupConfig,
+  entry: SearchEntry,
+): Promise<GroupEntity | undefined> {
+  const { set, map } = config;
+  const entity: GroupEntity = {
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Group',
+    metadata: {
+      name: '',
+      annotations: {},
+    },
+    spec: {
+      type: 'unknown',
+      profile: {},
+      children: [],
+    },
+  };
+
+  if (set) {
+    for (const [path, value] of Object.entries(set)) {
+      lodashSet(entity, path, value);
+    }
+  }
+
+  mapStringAttr(entry, vendor, map.name, v => {
+    entity.metadata.name = v;
+  });
+  mapStringAttr(entry, vendor, map.description, v => {
+    entity.metadata.description = v;
+  });
+  mapStringAttr(entry, vendor, map.rdn, v => {
+    entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => {
+    entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, vendor.dnAttributeName, v => {
+    entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
+  });
+  mapStringAttr(entry, vendor, map.type, v => {
+    entity.spec.type = v;
+  });
+  mapStringAttr(entry, vendor, map.displayName, v => {
+    entity.spec.profile!.displayName = v;
+  });
+  mapStringAttr(entry, vendor, map.email, v => {
+    entity.spec.profile!.email = v;
+  });
+  mapStringAttr(entry, vendor, map.picture, v => {
+    entity.spec.profile!.picture = v;
+  });
+
+  return entity;
+}
+
 /**
 * Reads groups out of an LDAP provider.
 *
 * @param client The LDAP client
 * @param config The group data configuration
+ * @param opts
 */
 export async function readLdapGroups(
  client: LdapClient,
  config: GroupConfig,
+  opts?: {
+    transformer?: GroupTransformer;
+  },
 ): Promise<{
  groups: GroupEntity[]; // With all relations empty
  groupMemberOf: Map<string, Set<string>>; // DN -> DN or UUID of groups
  groupMember: Map<string, Set<string>>; // DN -> DN or UUID of groups & users
 }> {
-  const { dn, options, set, map } = config;
-  const vendor = await client.getVendor();
-
-  const entries = await client.search(dn, options);
-
  const groups: GroupEntity[] = [];
  const groupMemberOf: Map<string, Set<string>> = new Map();
  const groupMember: Map<string, Set<string>> = new Map();

-  for (const entry of entries) {
-    const entity: GroupEntity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Group',
-      metadata: {
-        name: '',
-        annotations: {},
-      },
-      spec: {
-        type: 'unknown',
-        profile: {},
-        children: [],
-      },
-    };
+  const { dn, map, options } = config;
+  const vendor = await client.getVendor();

-    if (set) {
-      for (const [path, value] of Object.entries(set)) {
-        lodashSet(entity, path, value);
-      }
+  const transformer = opts?.transformer ?? defaultGroupTransformer;
+
+  const entries = await client.search(dn, options);
+
+  for (const group of entries) {
+    const entity = await transformer(vendor, config, group);
+
+    if (!entity) {
+      continue;
    }

-    mapStringAttr(entry, vendor, map.name, v => {
-      entity.metadata.name = v;
-    });
-    mapStringAttr(entry, vendor, map.description, v => {
-      entity.metadata.description = v;
-    });
-    mapStringAttr(entry, vendor, map.rdn, v => {
-      entity.metadata.annotations![LDAP_RDN_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, vendor.uuidAttributeName, v => {
-      entity.metadata.annotations![LDAP_UUID_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, vendor.dnAttributeName, v => {
-      entity.metadata.annotations![LDAP_DN_ANNOTATION] = v;
-    });
-    mapStringAttr(entry, vendor, map.type, v => {
-      entity.spec.type = v;
-    });
-    mapStringAttr(entry, vendor, map.displayName, v => {
-      entity.spec.profile!.displayName = v;
-    });
-    mapStringAttr(entry, vendor, map.email, v => {
-      entity.spec.profile!.email = v;
-    });
-    mapStringAttr(entry, vendor, map.picture, v => {
-      entity.spec.profile!.picture = v;
-    });
-
-    mapReferencesAttr(entry, vendor, map.memberOf, (myDn, vs) => {
+    mapReferencesAttr(group, vendor, map.memberOf, (myDn, vs) => {
      ensureItems(groupMemberOf, myDn, vs);
    });
-    mapReferencesAttr(entry, vendor, map.members, (myDn, vs) => {
+    mapReferencesAttr(group, vendor, map.members, (myDn, vs) => {
      ensureItems(groupMember, myDn, vs);
    });

@@ -199,22 +243,30 @@ export async function readLdapGroups(
 * with all relations etc filled in.
 *
 * @param client The LDAP client
- * @param logger A logger instance
 * @param userConfig The user data configuration
 * @param groupConfig The group data configuration
+ * @param options
 */
 export async function readLdapOrg(
  client: LdapClient,
  userConfig: UserConfig,
  groupConfig: GroupConfig,
+  options: {
+    groupTransformer?: GroupTransformer;
+    userTransformer?: UserTransformer;
+    logger: Logger;
+  },
 ): Promise<{
  users: UserEntity[];
  groups: GroupEntity[];
 }> {
-  const { users, userMemberOf } = await readLdapUsers(client, userConfig);
+  const { users, userMemberOf } = await readLdapUsers(client, userConfig, {
+    transformer: options?.userTransformer,
+  });
  const { groups, groupMemberOf, groupMember } = await readLdapGroups(
    client,
    groupConfig,
+    { transformer: options?.groupTransformer },
  );

  resolveRelations(groups, users, userMemberOf, groupMemberOf, groupMember);
@@ -228,21 +280,6 @@ export async function readLdapOrg(
 // Helpers
 //

-// Maps a single-valued attribute to a consumer
-function mapStringAttr(
-  entry: SearchEntry,
-  vendor: LdapVendor,
-  attributeName: string | undefined,
-  setter: (value: string) => void,
-) {
-  if (attributeName) {
-    const values = vendor.decodeStringAttribute(entry, attributeName);
-    if (values && values.length === 1) {
-      setter(values[0]);
-    }
-  }
-}
-
 // Maps a multi-valued attribute of references to other objects, to a consumer
 function mapReferencesAttr(
  entry: SearchEntry,
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { GroupEntity, UserEntity } from '@backstage/catalog-model';
+import { SearchEntry } from 'ldapjs';
+import { LdapVendor } from './vendors';
+import { GroupConfig, UserConfig } from './config';
+
+/**
+ * Customize the ingested User entity
+ *
+ * @param vendor The LDAP vendor that can be used to find and decode vendor specific attributes
+ * @param config The User specific config used by the default transformer.
+ * @param user The found LDAP entry in its source format. This is the entry that you want to transform
+ * @return A `UserEntity` or `undefined` if you want to ignore the found user for being ingested by the catalog
+ */
+export type UserTransformer = (
+  vendor: LdapVendor,
+  config: UserConfig,
+  user: SearchEntry,
+) => Promise<UserEntity | undefined>;
+
+/**
+ * Customize the ingested Group entity
+ *
+ * @param vendor The LDAP vendor that can be used to find and decode vendor specific attributes
+ * @param config The Group specific config used by the default transformer.
+ * @param group The found LDAP entry in its source format. This is the entry that you want to transform
+ * @return A `GroupEntity` or `undefined` if you want to ignore the found group for being ingested by the catalog
+ */
+export type GroupTransformer = (
+  vendor: LdapVendor,
+  config: GroupConfig,
+  group: SearchEntry,
+) => Promise<GroupEntity | undefined>;
@@ -14,7 +14,8 @@
 * limitations under the License.
 */

-import { Error as LDAPError } from 'ldapjs';
+import { Error as LDAPError, SearchEntry } from 'ldapjs';
+import { LdapVendor } from './vendors';

 /**
 * Builds a string form of an LDAP Error structure.
@@ -24,3 +25,25 @@ import { Error as LDAPError } from 'ldapjs';
 export function errorString(error: LDAPError) {
  return `${error.code} ${error.name}: ${error.message}`;
 }
+
+/**
+ * Maps a single-valued attribute to a consumer
+ *
+ * @param entry The LDAP source entry
+ * @param vendor The LDAP vendor
+ * @param attributeName The source attribute to map. If the attribute is undefined the mapping will be silently ignored.
+ * @param setter The function to be called with the decoded attribute from the source entry
+ */
+export function mapStringAttr(
+  entry: SearchEntry,
+  vendor: LdapVendor,
+  attributeName: string | undefined,
+  setter: (value: string) => void,
+) {
+  if (attributeName) {
+    const values = vendor.decodeStringAttribute(entry, attributeName);
+    if (values && values.length === 1) {
+      setter(values[0]);
+    }
+  }
+}
@@ -18,10 +18,12 @@ import { LocationSpec } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import { Logger } from 'winston';
 import {
+  GroupTransformer,
  LdapClient,
  LdapProviderConfig,
  readLdapConfig,
  readLdapOrg,
+  UserTransformer,
 } from '../ldap';
 import {
  CatalogProcessor,
@@ -35,8 +37,17 @@ import {
 export class LdapOrgReaderProcessor implements CatalogProcessor {
  private readonly providers: LdapProviderConfig[];
  private readonly logger: Logger;
+  private readonly groupTransformer?: GroupTransformer;
+  private readonly userTransformer?: UserTransformer;

-  static fromConfig(config: Config, options: { logger: Logger }) {
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger;
+      groupTransformer?: GroupTransformer;
+      userTransformer?: UserTransformer;
+    },
+  ) {
    const c = config.getOptionalConfig('catalog.processors.ldapOrg');
    return new LdapOrgReaderProcessor({
      ...options,
@@ -44,9 +55,16 @@ export class LdapOrgReaderProcessor implements CatalogProcessor {
    });
  }

-  constructor(options: { providers: LdapProviderConfig[]; logger: Logger }) {
+  constructor(options: {
+    providers: LdapProviderConfig[];
+    logger: Logger;
+    groupTransformer?: GroupTransformer;
+    userTransformer?: UserTransformer;
+  }) {
    this.providers = options.providers;
    this.logger = options.logger;
+    this.groupTransformer = options.groupTransformer;
+    this.userTransformer = options.userTransformer;
  }

  async readLocation(
@@ -81,6 +99,11 @@ export class LdapOrgReaderProcessor implements CatalogProcessor {
      client,
      provider.users,
      provider.groups,
+      {
+        groupTransformer: this.groupTransformer,
+        userTransformer: this.userTransformer,
+        logger: this.logger,
+      },
    );

    const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);