Introduce LdapOrgEntityProvider

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

.changeset/fifty-bats-itch.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Export the entity provider related types for external use.

.changeset/rich-paws-flow.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-ldap': minor
+---
+
+Introduce `LdapOrgEntityProvider` as an alternative to `LdapOrgReaderProcessor`. This also changes the `LdapClient` interface to require a logger.

plugins/catalog-backend-module-ldap/api-report.md

@@ -7,6 +7,8 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
 import { Client } from 'ldapjs';
 import { Config } from '@backstage/config';
+import { EntityProvider } from '@backstage/plugin-catalog-backend';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GroupEntity } from '@backstage/catalog-model';
 import { JsonValue } from '@backstage/config';
 import { LocationSpec } from '@backstage/catalog-model';
@@ -87,7 +89,7 @@ export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid';
 //
 // @public
 export class LdapClient {
-  constructor(client: Client);
+  constructor(client: Client, logger: Logger_2);
   // Warning: (ae-forgotten-export) The symbol "BindConfig" needs to be exported by the entry point index.d.ts
   //
   // (undocumented)
@@ -103,6 +105,36 @@ export class LdapClient {
   search(dn: string, options: SearchOptions): Promise<SearchEntry[]>;
 }
 
+// Warning: (ae-missing-release-tag) "LdapOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export class LdapOrgEntityProvider implements EntityProvider {
+  constructor(options: {
+    id: string;
+    provider: LdapProviderConfig;
+    logger: Logger_2;
+    userTransformer?: UserTransformer;
+    groupTransformer?: GroupTransformer;
+  });
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  static fromConfig(
+    configRoot: Config,
+    options: {
+      id: string;
+      target: string;
+      userTransformer?: UserTransformer;
+      groupTransformer?: GroupTransformer;
+      logger: Logger_2;
+    },
+  ): LdapOrgEntityProvider;
+  // (undocumented)
+  getProviderName(): string;
+  // (undocumented)
+  read(): Promise<void>;
+}
+
 // Warning: (ae-missing-release-tag) "LdapOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public

plugins/catalog-backend-module-ldap/src/ldap/client.ts

@@ -47,7 +47,7 @@ export class LdapClient {
     });
 
     if (!bind) {
-      return new LdapClient(client);
+      return new LdapClient(client, logger);
     }
 
     return new Promise<LdapClient>((resolve, reject) => {
@@ -56,13 +56,16 @@ export class LdapClient {
         if (err) {
           reject(`LDAP bind failed for ${dn}, ${errorString(err)}`);
         } else {
-          resolve(new LdapClient(client));
+          resolve(new LdapClient(client, logger));
         }
       });
     });
   }
 
-  constructor(private readonly client: Client) {}
+  constructor(
+    private readonly client: Client,
+    private readonly logger: Logger,
+  ) {}
 
   /**
    * Performs an LDAP search operation.
@@ -72,9 +75,13 @@ export class LdapClient {
    */
   async search(dn: string, options: SearchOptions): Promise<SearchEntry[]> {
     try {
-      return await new Promise<SearchEntry[]>((resolve, reject) => {
-        const output: SearchEntry[] = [];
+      const output: SearchEntry[] = [];
 
+      const logInterval = setInterval(() => {
+        this.logger.debug(`Read ${output.length} LDAP entries so far...`);
+      }, 5000);
+
+      const search = new Promise<SearchEntry[]>((resolve, reject) => {
         this.client.search(dn, options, (err, res) => {
           if (err) {
             reject(new Error(errorString(err)));
@@ -104,6 +111,10 @@ export class LdapClient {
           });
         });
       });
+
+      return await search.finally(() => {
+        clearInterval(logInterval);
+      });
     } catch (e) {
       throw new Error(`LDAP search at DN "${dn}" failed, ${e.message}`);
     }

plugins/catalog-backend-module-ldap/src/ldap/read.ts

@@ -38,7 +38,7 @@ export async function defaultUserTransformer(
   const { set, map } = config;
 
   const entity: UserEntity = {
-    apiVersion: 'backstage.io/v1alpha1',
+    apiVersion: 'backstage.io/v1beta1',
     kind: 'User',
     metadata: {
       name: '',
@@ -133,7 +133,7 @@ export async function defaultGroupTransformer(
 ): Promise<GroupEntity | undefined> {
   const { set, map } = config;
   const entity: GroupEntity = {
-    apiVersion: 'backstage.io/v1alpha1',
+    apiVersion: 'backstage.io/v1beta1',
     kind: 'Group',
     metadata: {
       name: '',

plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts

@@ -0,0 +1,204 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  Entity,
+  LOCATION_ANNOTATION,
+  ORIGIN_LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import {
+  EntityProvider,
+  EntityProviderConnection,
+} from '@backstage/plugin-catalog-backend';
+import { merge } from 'lodash';
+import { Logger } from 'winston';
+import {
+  GroupTransformer,
+  LdapClient,
+  LdapProviderConfig,
+  LDAP_DN_ANNOTATION,
+  readLdapConfig,
+  readLdapOrg,
+  UserTransformer,
+} from '../ldap';
+
+/**
+ * Reads user and group entries out of an LDAP service, and provides them as
+ * User and Group entities for the catalog.
+ */
+export class LdapOrgEntityProvider implements EntityProvider {
+  private connection?: EntityProviderConnection;
+
+  static fromConfig(
+    configRoot: Config,
+    options: {
+      /**
+       * A unique, stable identifier for this provider.
+       *
+       * @example "production"
+       */
+      id: string;
+      /**
+       * The target that this provider should consume.
+       *
+       * Should exactly match the "target" field of one of the "ldap.providers"
+       * configuration entries.
+       *
+       * @example "ldaps://ds-read.example.net"
+       */
+      target: string;
+      /**
+       * The function that transforms a user entry in LDAP to an entity.
+       */
+      userTransformer?: UserTransformer;
+      /**
+       * The function that transforms a group entry in LDAP to an entity.
+       */
+      groupTransformer?: GroupTransformer;
+      logger: Logger;
+    },
+  ): LdapOrgEntityProvider {
+    // TODO(freben): Deprecate the old catalog.processors.ldapOrg config
+    const config =
+      configRoot.getOptionalConfig('ldap') ||
+      configRoot.getOptionalConfig('catalog.processors.ldapOrg');
+    if (!config) {
+      throw new TypeError(
+        `There is no LDAP configuration. Please add it as "ldap.providers".`,
+      );
+    }
+
+    const providers = readLdapConfig(config);
+    const provider = providers.find(p => options.target === p.target);
+    if (!provider) {
+      throw new TypeError(
+        `There is no LDAP configuration that matches ${options.target}. Please add a configuration entry for it under "ldap.providers".`,
+      );
+    }
+
+    const logger = options.logger.child({
+      target: options.target,
+    });
+
+    return new LdapOrgEntityProvider({
+      id: options.id,
+      provider,
+      userTransformer: options.userTransformer,
+      groupTransformer: options.groupTransformer,
+      logger,
+    });
+  }
+
+  constructor(
+    private options: {
+      id: string;
+      provider: LdapProviderConfig;
+      logger: Logger;
+      userTransformer?: UserTransformer;
+      groupTransformer?: GroupTransformer;
+    },
+  ) {}
+
+  getProviderName() {
+    return `LdapOrgEntityProvider:${this.options.id}`;
+  }
+
+  async connect(connection: EntityProviderConnection) {
+    this.connection = connection;
+  }
+
+  async read() {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const { markReadComplete } = trackProgress(this.options.logger);
+
+    // Be lazy and create the client each time; even though it's pretty
+    // inefficient, we usually only do this once per entire refresh loop and
+    // don't have to worry about timeouts and reconnects etc.
+    const client = await LdapClient.create(
+      this.options.logger,
+      this.options.provider.target,
+      this.options.provider.bind,
+    );
+
+    const { users, groups } = await readLdapOrg(
+      client,
+      this.options.provider.users,
+      this.options.provider.groups,
+      {
+        groupTransformer: this.options.groupTransformer,
+        userTransformer: this.options.userTransformer,
+        logger: this.options.logger,
+      },
+    );
+
+    const { markCommitComplete } = markReadComplete({ users, groups });
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: [...users, ...groups].map(entity => ({
+        locationKey: `ldap-org-provider:${this.options.id}`,
+        entity: withLocations(this.options.id, entity),
+      })),
+    });
+
+    markCommitComplete();
+  }
+}
+
+// Helps wrap the timing and logging behaviors
+function trackProgress(logger: Logger) {
+  let timestamp = Date.now();
+  let summary: string;
+
+  logger.info('Reading LDAP users and groups');
+
+  function markReadComplete(read: { users: unknown[]; groups: unknown[] }) {
+    summary = `${read.users.length} LDAP users and ${read.groups.length} LDAP groups`;
+    const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    timestamp = Date.now();
+    logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`);
+    return { markCommitComplete };
+  }
+
+  function markCommitComplete() {
+    const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
+    logger.info(`Committed ${summary} in ${commitDuration} seconds.`);
+  }
+
+  return { markReadComplete };
+}
+
+// Makes sure that emitted entities have a proper location based on their DN
+function withLocations(providerId: string, entity: Entity): Entity {
+  const dn =
+    entity.metadata.annotations?.[LDAP_DN_ANNOTATION] || entity.metadata.name;
+  const location = `ldap://${providerId}/${encodeURIComponent(dn)}`;
+  return merge(
+    {
+      metadata: {
+        annotations: {
+          [LOCATION_ANNOTATION]: location,
+          [ORIGIN_LOCATION_ANNOTATION]: location,
+        },
+      },
+    },
+    entity,
+  ) as Entity;
+}

plugins/catalog-backend-module-ldap/src/processors/index.ts

@@ -13,4 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+export { LdapOrgEntityProvider } from './LdapOrgEntityProvider';
 export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor';

plugins/catalog-backend/api-report.md

@@ -699,6 +699,38 @@ export type EntityProcessingResult =
       errors: Error[];
     };
 
+// Warning: (ae-missing-release-tag) "EntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface EntityProvider {
+  // (undocumented)
+  connect(connection: EntityProviderConnection): Promise<void>;
+  // (undocumented)
+  getProviderName(): string;
+}
+
+// Warning: (ae-missing-release-tag) "EntityProviderConnection" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface EntityProviderConnection {
+  // (undocumented)
+  applyMutation(mutation: EntityProviderMutation): Promise<void>;
+}
+
+// Warning: (ae-missing-release-tag) "EntityProviderMutation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type EntityProviderMutation =
+  | {
+      type: 'full';
+      entities: DeferredEntity[];
+    }
+  | {
+      type: 'delta';
+      added: DeferredEntity[];
+      removed: DeferredEntity[];
+    };
+
 // Warning: (ae-missing-release-tag) "FileReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -899,7 +931,6 @@ export class NextCatalogBuilder {
   // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
   addEntityPolicy(...policies: EntityPolicy[]): NextCatalogBuilder;
   // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
-  // Warning: (ae-forgotten-export) The symbol "EntityProvider" needs to be exported by the entry point index.d.ts
   addEntityProvider(...providers: EntityProvider[]): NextCatalogBuilder;
   // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
   addProcessor(...processors: CatalogProcessor[]): NextCatalogBuilder;

plugins/catalog-backend/src/next/index.ts

@@ -17,6 +17,11 @@
 export { NextCatalogBuilder } from './NextCatalogBuilder';
 export { createNextRouter } from './NextRouter';
 export * from './processing';
-export * from './stitching';
-export type { RefreshIntervalFunction } from './refresh';
 export { createRandomRefreshInterval } from './refresh';
+export type { RefreshIntervalFunction } from './refresh';
+export * from './stitching';
+export type {
+  EntityProvider,
+  EntityProviderConnection,
+  EntityProviderMutation,
+} from './types';