From 54b441abe34542d502406990804171620f6a89c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 18 Aug 2021 15:44:02 +0200 Subject: [PATCH] Introduce LdapOrgEntityProvider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/fifty-bats-itch.md | 5 + .changeset/rich-paws-flow.md | 5 + .../catalog-backend-module-ldap/api-report.md | 34 ++- .../src/ldap/client.ts | 21 +- .../src/ldap/read.ts | 4 +- .../src/processors/LdapOrgEntityProvider.ts | 204 ++++++++++++++++++ .../src/processors/index.ts | 2 + plugins/catalog-backend/api-report.md | 33 ++- plugins/catalog-backend/src/next/index.ts | 9 +- 9 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 .changeset/fifty-bats-itch.md create mode 100644 .changeset/rich-paws-flow.md create mode 100644 plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts diff --git a/.changeset/fifty-bats-itch.md b/.changeset/fifty-bats-itch.md new file mode 100644 index 0000000000..c8793beb06 --- /dev/null +++ b/.changeset/fifty-bats-itch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Export the entity provider related types for external use. diff --git a/.changeset/rich-paws-flow.md b/.changeset/rich-paws-flow.md new file mode 100644 index 0000000000..73ce4fd34a --- /dev/null +++ b/.changeset/rich-paws-flow.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-ldap': minor +--- + +Introduce `LdapOrgEntityProvider` as an alternative to `LdapOrgReaderProcessor`. This also changes the `LdapClient` interface to require a logger. diff --git a/plugins/catalog-backend-module-ldap/api-report.md b/plugins/catalog-backend-module-ldap/api-report.md index 5e713eee09..35f647ec70 100644 --- a/plugins/catalog-backend-module-ldap/api-report.md +++ b/plugins/catalog-backend-module-ldap/api-report.md @@ -7,6 +7,8 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { Client } from 'ldapjs'; import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { GroupEntity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/config'; import { LocationSpec } from '@backstage/catalog-model'; @@ -87,7 +89,7 @@ export const LDAP_UUID_ANNOTATION = 'backstage.io/ldap-uuid'; // // @public export class LdapClient { - constructor(client: Client); + constructor(client: Client, logger: Logger_2); // Warning: (ae-forgotten-export) The symbol "BindConfig" needs to be exported by the entry point index.d.ts // // (undocumented) @@ -103,6 +105,36 @@ export class LdapClient { search(dn: string, options: SearchOptions): Promise; } +// Warning: (ae-missing-release-tag) "LdapOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class LdapOrgEntityProvider implements EntityProvider { + constructor(options: { + id: string; + provider: LdapProviderConfig; + logger: Logger_2; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + }); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + configRoot: Config, + options: { + id: string; + target: string; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + logger: Logger_2; + }, + ): LdapOrgEntityProvider; + // (undocumented) + getProviderName(): string; + // (undocumented) + read(): Promise; +} + // Warning: (ae-missing-release-tag) "LdapOrgReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog-backend-module-ldap/src/ldap/client.ts b/plugins/catalog-backend-module-ldap/src/ldap/client.ts index a99fc3846e..5418577381 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/client.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/client.ts @@ -47,7 +47,7 @@ export class LdapClient { }); if (!bind) { - return new LdapClient(client); + return new LdapClient(client, logger); } return new Promise((resolve, reject) => { @@ -56,13 +56,16 @@ export class LdapClient { if (err) { reject(`LDAP bind failed for ${dn}, ${errorString(err)}`); } else { - resolve(new LdapClient(client)); + resolve(new LdapClient(client, logger)); } }); }); } - constructor(private readonly client: Client) {} + constructor( + private readonly client: Client, + private readonly logger: Logger, + ) {} /** * Performs an LDAP search operation. @@ -72,9 +75,13 @@ export class LdapClient { */ async search(dn: string, options: SearchOptions): Promise { try { - return await new Promise((resolve, reject) => { - const output: SearchEntry[] = []; + const output: SearchEntry[] = []; + const logInterval = setInterval(() => { + this.logger.debug(`Read ${output.length} LDAP entries so far...`); + }, 5000); + + const search = new Promise((resolve, reject) => { this.client.search(dn, options, (err, res) => { if (err) { reject(new Error(errorString(err))); @@ -104,6 +111,10 @@ export class LdapClient { }); }); }); + + return await search.finally(() => { + clearInterval(logInterval); + }); } catch (e) { throw new Error(`LDAP search at DN "${dn}" failed, ${e.message}`); } diff --git a/plugins/catalog-backend-module-ldap/src/ldap/read.ts b/plugins/catalog-backend-module-ldap/src/ldap/read.ts index f3ee09d32d..77ca43a519 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/read.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/read.ts @@ -38,7 +38,7 @@ export async function defaultUserTransformer( const { set, map } = config; const entity: UserEntity = { - apiVersion: 'backstage.io/v1alpha1', + apiVersion: 'backstage.io/v1beta1', kind: 'User', metadata: { name: '', @@ -133,7 +133,7 @@ export async function defaultGroupTransformer( ): Promise { const { set, map } = config; const entity: GroupEntity = { - apiVersion: 'backstage.io/v1alpha1', + apiVersion: 'backstage.io/v1beta1', kind: 'Group', metadata: { name: '', diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts new file mode 100644 index 0000000000..c0cd663052 --- /dev/null +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts @@ -0,0 +1,204 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + LOCATION_ANNOTATION, + ORIGIN_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { merge } from 'lodash'; +import { Logger } from 'winston'; +import { + GroupTransformer, + LdapClient, + LdapProviderConfig, + LDAP_DN_ANNOTATION, + readLdapConfig, + readLdapOrg, + UserTransformer, +} from '../ldap'; + +/** + * Reads user and group entries out of an LDAP service, and provides them as + * User and Group entities for the catalog. + */ +export class LdapOrgEntityProvider implements EntityProvider { + private connection?: EntityProviderConnection; + + static fromConfig( + configRoot: Config, + options: { + /** + * A unique, stable identifier for this provider. + * + * @example "production" + */ + id: string; + /** + * The target that this provider should consume. + * + * Should exactly match the "target" field of one of the "ldap.providers" + * configuration entries. + * + * @example "ldaps://ds-read.example.net" + */ + target: string; + /** + * The function that transforms a user entry in LDAP to an entity. + */ + userTransformer?: UserTransformer; + /** + * The function that transforms a group entry in LDAP to an entity. + */ + groupTransformer?: GroupTransformer; + logger: Logger; + }, + ): LdapOrgEntityProvider { + // TODO(freben): Deprecate the old catalog.processors.ldapOrg config + const config = + configRoot.getOptionalConfig('ldap') || + configRoot.getOptionalConfig('catalog.processors.ldapOrg'); + if (!config) { + throw new TypeError( + `There is no LDAP configuration. Please add it as "ldap.providers".`, + ); + } + + const providers = readLdapConfig(config); + const provider = providers.find(p => options.target === p.target); + if (!provider) { + throw new TypeError( + `There is no LDAP configuration that matches ${options.target}. Please add a configuration entry for it under "ldap.providers".`, + ); + } + + const logger = options.logger.child({ + target: options.target, + }); + + return new LdapOrgEntityProvider({ + id: options.id, + provider, + userTransformer: options.userTransformer, + groupTransformer: options.groupTransformer, + logger, + }); + } + + constructor( + private options: { + id: string; + provider: LdapProviderConfig; + logger: Logger; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + }, + ) {} + + getProviderName() { + return `LdapOrgEntityProvider:${this.options.id}`; + } + + async connect(connection: EntityProviderConnection) { + this.connection = connection; + } + + async read() { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(this.options.logger); + + // Be lazy and create the client each time; even though it's pretty + // inefficient, we usually only do this once per entire refresh loop and + // don't have to worry about timeouts and reconnects etc. + const client = await LdapClient.create( + this.options.logger, + this.options.provider.target, + this.options.provider.bind, + ); + + const { users, groups } = await readLdapOrg( + client, + this.options.provider.users, + this.options.provider.groups, + { + groupTransformer: this.options.groupTransformer, + userTransformer: this.options.userTransformer, + logger: this.options.logger, + }, + ); + + const { markCommitComplete } = markReadComplete({ users, groups }); + + await this.connection.applyMutation({ + type: 'full', + entities: [...users, ...groups].map(entity => ({ + locationKey: `ldap-org-provider:${this.options.id}`, + entity: withLocations(this.options.id, entity), + })), + }); + + markCommitComplete(); + } +} + +// Helps wrap the timing and logging behaviors +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + let summary: string; + + logger.info('Reading LDAP users and groups'); + + function markReadComplete(read: { users: unknown[]; groups: unknown[] }) { + summary = `${read.users.length} LDAP users and ${read.groups.length} LDAP groups`; + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info(`Read ${summary} in ${readDuration} seconds. Committing...`); + return { markCommitComplete }; + } + + function markCommitComplete() { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info(`Committed ${summary} in ${commitDuration} seconds.`); + } + + return { markReadComplete }; +} + +// Makes sure that emitted entities have a proper location based on their DN +function withLocations(providerId: string, entity: Entity): Entity { + const dn = + entity.metadata.annotations?.[LDAP_DN_ANNOTATION] || entity.metadata.name; + const location = `ldap://${providerId}/${encodeURIComponent(dn)}`; + return merge( + { + metadata: { + annotations: { + [LOCATION_ANNOTATION]: location, + [ORIGIN_LOCATION_ANNOTATION]: location, + }, + }, + }, + entity, + ) as Entity; +} diff --git a/plugins/catalog-backend-module-ldap/src/processors/index.ts b/plugins/catalog-backend-module-ldap/src/processors/index.ts index 6e3f413085..96e1a49cbb 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/index.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/index.ts @@ -13,4 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +export { LdapOrgEntityProvider } from './LdapOrgEntityProvider'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a6e67215c9..723ea5c50c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -699,6 +699,38 @@ export type EntityProcessingResult = errors: Error[]; }; +// Warning: (ae-missing-release-tag) "EntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + getProviderName(): string; +} + +// Warning: (ae-missing-release-tag) "EntityProviderConnection" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface EntityProviderConnection { + // (undocumented) + applyMutation(mutation: EntityProviderMutation): Promise; +} + +// Warning: (ae-missing-release-tag) "EntityProviderMutation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type EntityProviderMutation = + | { + type: 'full'; + entities: DeferredEntity[]; + } + | { + type: 'delta'; + added: DeferredEntity[]; + removed: DeferredEntity[]; + }; + // Warning: (ae-missing-release-tag) "FileReaderProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -899,7 +931,6 @@ export class NextCatalogBuilder { // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen addEntityPolicy(...policies: EntityPolicy[]): NextCatalogBuilder; // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (ae-forgotten-export) The symbol "EntityProvider" needs to be exported by the entry point index.d.ts addEntityProvider(...providers: EntityProvider[]): NextCatalogBuilder; // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen addProcessor(...processors: CatalogProcessor[]): NextCatalogBuilder; diff --git a/plugins/catalog-backend/src/next/index.ts b/plugins/catalog-backend/src/next/index.ts index a27174c415..2afdff63fb 100644 --- a/plugins/catalog-backend/src/next/index.ts +++ b/plugins/catalog-backend/src/next/index.ts @@ -17,6 +17,11 @@ export { NextCatalogBuilder } from './NextCatalogBuilder'; export { createNextRouter } from './NextRouter'; export * from './processing'; -export * from './stitching'; -export type { RefreshIntervalFunction } from './refresh'; export { createRandomRefreshInterval } from './refresh'; +export type { RefreshIntervalFunction } from './refresh'; +export * from './stitching'; +export type { + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, +} from './types';