From e2e7cb207cfe07cd74642c4e4eed184a7f96d952 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 14:41:17 +0200 Subject: [PATCH 01/22] Update CODEOWNERS Signed-off-by: Tomas Dabasinskas --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 10f1ac188c..010af451af 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,6 +30,7 @@ yarn.lock @backstage/maintainers @back /plugins/catalog-backend-module-aws @backstage/maintainers @backstage/catalog-core @pjungermann /plugins/catalog-backend-module-bitbucket-cloud @backstage/maintainers @backstage/catalog-core @pjungermann /plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-core @pjungermann +/plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-core @tdabasinskas /plugins/catalog-graph @backstage/maintainers @backstage/catalog-core @backstage/sda-se-reviewers /plugins/circleci @backstage/maintainers @adamdmharvey /plugins/cloudbuild @backstage/maintainers @trivago/ebarrios From 0c48cb2137f4deb863677b2707df4427eca7888b Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 14:58:42 +0200 Subject: [PATCH 02/22] Generate new plugin Signed-off-by: Tomas Dabasinskas --- .../.eslintrc.js | 1 + .../catalog-backend-module-puppetdb/README.md | 14 +++++ .../package.json | 54 +++++++++++++++++++ .../src/setupTests.ts | 16 ++++++ yarn.lock | 16 ++++++ 5 files changed, 101 insertions(+) create mode 100644 plugins/catalog-backend-module-puppetdb/.eslintrc.js create mode 100644 plugins/catalog-backend-module-puppetdb/README.md create mode 100644 plugins/catalog-backend-module-puppetdb/package.json create mode 100644 plugins/catalog-backend-module-puppetdb/src/setupTests.ts diff --git a/plugins/catalog-backend-module-puppetdb/.eslintrc.js b/plugins/catalog-backend-module-puppetdb/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md new file mode 100644 index 0000000000..c66627fa8b --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -0,0 +1,14 @@ +# catalog-backend-module-puppetdb + +Welcome to the catalog-backend-module-puppetdb backend plugin! + +_This plugin was created through the Backstage CLI_ + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn +start` in the root directory, and then navigating to [/catalog-backend-module-puppetdb](http://localhost:3000/catalog-backend-module-puppetdb). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json new file mode 100644 index 0000000000..5f6a39f73d --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -0,0 +1,54 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-puppetdb-backend", + "description": "A Backstage catalog backend module that helps integrate towards PuppetDB", + "version": "0.0.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "catalog-backend-module-puppetdb" + }, + "keywords": [ + "backstage", + "puppetdb", + "puppet" + ], + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/config": "workspace:^", + "lodash": "^4.17.21", + "node-fetch": "^2.6.7", + "uuid": "^8.0.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "workspace:^", + "@types/lodash": "^4.14.151", + "msw": "^0.49.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-puppetdb/src/setupTests.ts b/plugins/catalog-backend-module-puppetdb/src/setupTests.ts new file mode 100644 index 0000000000..4b9026cde5 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/yarn.lock b/yarn.lock index 6973779ac5..33261fc7da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5206,6 +5206,22 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-puppetdb-backend@workspace:plugins/catalog-backend-module-puppetdb": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-puppetdb-backend@workspace:plugins/catalog-backend-module-puppetdb" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@types/lodash": ^4.14.151 + lodash: ^4.17.21 + msw: ^0.49.0 + node-fetch: ^2.6.7 + uuid: ^8.0.0 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend" From 9bb20d2b0ddd0cd1a4c041634d86ef97c8d6c0d4 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 19:49:08 +0200 Subject: [PATCH 03/22] Add initial code Signed-off-by: Tomas Dabasinskas --- .../config.d.ts | 58 +++++ .../package.json | 6 + .../src/index.ts | 24 ++ .../src/providers/PuppetDBEntityProvider.ts | 236 ++++++++++++++++++ .../providers/PuppetDBEntityProviderConfig.ts | 99 ++++++++ .../src/providers/constants.ts | 29 +++ .../src/providers/index.ts | 19 ++ .../src/puppet/constants.ts | 32 +++ .../src/puppet/index.ts | 24 ++ .../src/puppet/read.ts | 75 ++++++ .../src/puppet/transformers.ts | 65 +++++ .../src/puppet/types.ts | 102 ++++++++ yarn.lock | 6 + 13 files changed, 775 insertions(+) create mode 100644 plugins/catalog-backend-module-puppetdb/config.d.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/index.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/constants.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/index.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/index.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/read.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/types.ts diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts new file mode 100644 index 0000000000..6514629251 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TaskScheduleDefinition, + TaskScheduleDefinitionConfig, +} from '@backstage/backend-tasks'; + +/** + * Represents the configuration for the Backstage. + */ +export interface Config { + /** + * Configuration for the catalog. + */ + catalog?: { + /** + * Configuration for the providers. + */ + providers?: { + /** + * Puppet entity provider configuration. Uses "default" as default ID for the single config variant. + */ + puppet?: ProviderConfig | Record; + }; + }; +} + +/** + * Configuration of {@link PuppetDBEntityProvider}. + */ +interface ProviderConfig { + /** + * (Required) The host of PuppetDB API instance. + */ + host: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; +} diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json index 5f6a39f73d..fa7f08da1a 100644 --- a/plugins/catalog-backend-module-puppetdb/package.json +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -35,8 +35,14 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/types": "workspace:^", "lodash": "^4.17.21", + "luxon": "^3.0.0", "node-fetch": "^2.6.7", "uuid": "^8.0.0", "winston": "^3.2.1" diff --git a/plugins/catalog-backend-module-puppetdb/src/index.ts b/plugins/catalog-backend-module-puppetdb/src/index.ts new file mode 100644 index 0000000000..b5c5f9d209 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards PuppetDB + * + * @packageDocumentation + */ + +export * from './providers'; +export * from './puppet'; diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts new file mode 100644 index 0000000000..a887d4778c --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts @@ -0,0 +1,236 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { + PuppetDBEntityProviderConfig, + readProviderConfigs, +} from './PuppetDBEntityProviderConfig'; +import { Config } from '@backstage/config'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import * as uuid from 'uuid'; +import { ResourceTransformer, defaultResourceTransformer } from '../puppet'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + Entity, +} from '@backstage/catalog-model/'; +import { merge } from 'lodash'; +import { readPuppetNodes } from '../puppet/read'; +import { ENDPOINT_NODES } from '../puppet/constants'; + +/** + * Reads nodes from [PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html) + * based on the provided query and registers them as Resource entities in the catalog. + * + * @public + */ +export class PuppetDBEntityProvider implements EntityProvider { + private readonly config: PuppetDBEntityProviderConfig; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private readonly transformer: ResourceTransformer; + private connection?: EntityProviderConnection; + + /** + * Creates instances of {@link PuppetDBEntityProvider} from a configuration. + * + * @param config - The configuration to read provider information from. + * @param deps - The dependencies for {@link PuppetDBEntityProvider}. + * + * @returns A list of {@link PuppetDBEntityProvider} instances. + */ + static fromConfig( + config: Config, + deps: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + transformer?: ResourceTransformer; + }, + ): PuppetDBEntityProvider[] { + if (!deps.schedule && !deps.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return readProviderConfigs(config).map(providerConfig => { + if (!deps.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for puppet-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + deps.schedule ?? + deps.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + const transformer = deps.transformer ?? defaultResourceTransformer; + + return new PuppetDBEntityProvider( + providerConfig, + deps.logger, + taskRunner, + transformer, + ); + }); + } + + /** + * Creates an instance of {@link PuppetDBEntityProvider}. + * + * @param config - Configuration of the provider. + * @param logger - The instance of a {@link Logger}. + * @param taskRunner - The instance of {@link TaskRunner}. + * @param transformer - A {@link ResourceTransformer} function. + * + * @private + */ + private constructor( + config: PuppetDBEntityProviderConfig, + logger: Logger, + taskRunner: TaskRunner, + transformer: ResourceTransformer, + ) { + this.config = config; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(taskRunner); + this.transformer = transformer; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `puppetdb-provider:${this.config.id}`; + } + + /** + * Creates a function that can be used to schedule a refresh of the catalog. + * + * @param taskRunner - The instance of {@link TaskRunner}. + * + * @private + */ + private createScheduleFn(taskRunner: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: PuppetDBEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + try { + await this.refresh(logger); + } catch (error) { + logger.error(`${this.getProviderName()} refresh failed`, error); + } + }, + }); + }; + } + + /** + * Refreshes the catalog by reading nodes from PuppetDB and registering them as Resource Entities. + * + * @param logger - The instance of a {@link Logger}. + */ + async refresh(logger: Logger) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(logger); + const entities = await readPuppetNodes(this.config, { + logger, + transformer: this.transformer, + }); + const { markCommitComplete } = markReadComplete(entities); + + await this.connection.applyMutation({ + type: 'full', + entities: [...entities].map(entity => ({ + locationKey: this.getProviderName(), + entity: withLocations(this.config.host, entity), + })), + }); + markCommitComplete(entities); + } +} + +/** + * Ensures the entities have required annotation data. + * + * @param host - The host of the PuppetDB instance. + * @param entity - The entity to add the annotations to. + * + * @returns Entity with @{@link ANNOTATION_LOCATION} and @{@link ANNOTATION_ORIGIN_LOCATION} annotations. + */ +function withLocations(host: string, entity: Entity): Entity { + const location = new URL(host); + location.pathname = `${ENDPOINT_NODES}/${entity.metadata?.name}`; + + return merge( + { + metadata: { + annotations: { + [ANNOTATION_LOCATION]: `url:${location.toString()}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${location.toString()}`, + }, + }, + }, + entity, + ) as Entity; +} + +/** + * Tracks the progress of the PuppetDB read and commit operations. + * + * @param logger - The instance of a {@link Logger}. + */ +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + + function markReadComplete(entities: Entity[]) { + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info( + `Read ${entities?.length ?? 0} in ${readDuration} seconds. Committing...`, + ); + return { markCommitComplete }; + } + + function markCommitComplete(entities: Entity[]) { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info( + `Committed ${entities?.length ?? 0} in ${commitDuration} seconds.`, + ); + } + + return { markReadComplete }; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts new file mode 100644 index 0000000000..e96a4a11e9 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { DEFAULT_PROVIDER_ID } from './constants'; + +/** + * Configuration of {@link PuppetDBEntityProvider}. + * + * @public + */ +export type PuppetDBEntityProviderConfig = { + /** + * ID of the provider. + */ + id: string; + /** + * (Required) The host of PuppetDB API instance. + */ + host: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; +}; + +/** + * Reads the configuration of the PuppetDB Entity Providers. + * + * @param config - The application configuration. + * + * @returns PuppetDB Entity Provider configurations list. + */ +export function readProviderConfigs( + config: Config, +): PuppetDBEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig( + 'catalog.providers.puppetdb', + ); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('host')) { + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + return readProviderConfig(id, providersConfig.getConfig(id)); + }); +} + +/** + * Reads the configuration for the PuppetDB Entity Provider. + * + * @param id - ID of the provider. + * @param config - The application configuration. + * + * @returns The PuppetDB Entity Provider configuration. + */ +function readProviderConfig( + id: string, + config: Config, +): PuppetDBEntityProviderConfig { + const host = config.getString('host').replace(/\/+$/, ''); + const query = config.getOptionalString('query'); + + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + + return { + id, + host, + query, + schedule, + }; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts new file mode 100644 index 0000000000..02b333c613 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Name of the default provider when a using a simple configuration. + * + * @public + */ +export const DEFAULT_PROVIDER_ID = 'default'; + +/** + * Default owner for entities created by the PuppetDB provider. + * + * @public + */ +export const DEFAULT_OWNER = 'unknown'; diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts new file mode 100644 index 0000000000..4e220395ba --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { PuppetDBEntityProvider } from './PuppetDBEntityProvider'; +export type { PuppetDBEntityProviderConfig } from './PuppetDBEntityProviderConfig'; +export { DEFAULT_PROVIDER_ID, DEFAULT_OWNER } from './constants'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts new file mode 100644 index 0000000000..ef355f22d3 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Annotation for specifying the certificate name of a node in PuppetDB. + * + * @public + */ +export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname'; + +/** + * Path of PuppetDB FactSets endpoint. + */ +export const ENDPOINT_FACTSETS = '/pdb/query/v4/factsets'; + +/** + * Path of PuppetDB Nodes endpoint. + */ +export const ENDPOINT_NODES = '/pdb/query/v4/nodes'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts new file mode 100644 index 0000000000..644d111ac1 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ANNOTATION_PUPPET_CERTNAME } from './constants'; +export { defaultResourceTransformer } from './transformers'; +export type { + PuppetNode, + PuppetFactSet, + PuppetFact, + ResourceTransformer, +} from './types'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts new file mode 100644 index 0000000000..edd33bc76a --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PuppetDBEntityProviderConfig } from '../providers'; +import { PuppetNode, ResourceTransformer } from './types'; +import { ResourceEntity } from '@backstage/catalog-model/'; +import { defaultResourceTransformer } from './transformers'; +import { ResponseError } from '@backstage/errors'; +import { ENDPOINT_FACTSETS } from './constants'; +import { Logger } from 'winston'; + +/** + * Reads nodes and their facts from PuppetDB. + * + * @param config - The provider configuration. + * @param opts - Additional options. + */ +export async function readPuppetNodes( + config: PuppetDBEntityProviderConfig, + opts?: { + transformer?: ResourceTransformer; + logger?: Logger; + }, +): Promise { + const transformFn = opts?.transformer ?? defaultResourceTransformer; + const url = new URL(config.host); + url.pathname = ENDPOINT_FACTSETS; + + if (config.query) { + url.searchParams.set('query', config.query); + } + + if (opts?.logger) { + opts.logger + .child({ url: url.toString() }) + .debug('Reading nodes from PuppetDB'); + } + + const response = await fetch(url.toString(), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw await ResponseError.fromResponse(response); + } + + const nodes = (await response.json()) as PuppetNode[]; + const entities: ResourceEntity[] = []; + + for (const node of nodes) { + const entity = await transformFn(node, config); + if (entity) { + entities.push(entity); + } + } + + return entities; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts new file mode 100644 index 0000000000..ca0d2a4ccc --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ResourceTransformer } from './types'; +import { DEFAULT_NAMESPACE, ResourceEntity } from '@backstage/catalog-model'; +import { DEFAULT_OWNER } from '../providers'; +import { ANNOTATION_PUPPET_CERTNAME } from './constants'; + +/** + * A default implementation of the {@link ResourceTransformer}. + * + * @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform. + * @param _config - The configuration for the entity provider. + * + * @returns A `ResourceEntity`. + * + * @public + */ +export const defaultResourceTransformer: ResourceTransformer = async ( + node, + _config, +) => { + const certName = node.certname.toLowerCase(); + const type = node.facts?.data?.find(e => e.name === 'is_virtual')?.value + ? 'virtual-machine' + : 'physical-server'; + const kernel = node.facts?.data?.find(e => e.name === 'kernel')?.value; + + const entity: ResourceEntity = { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: certName, + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: certName, + }, + namespace: DEFAULT_NAMESPACE, + description: node.facts?.data + ?.find(e => e.name === 'ipaddress') + ?.value?.toString(), + tags: kernel ? [kernel.toString().toLowerCase()] : [], + }, + spec: { + type: type, + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }; + + return entity; +}; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts new file mode 100644 index 0000000000..7dc951f49f --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ResourceEntity } from '@backstage/catalog-model'; +import { JsonValue } from '@backstage/types'; +import { PuppetDBEntityProviderConfig } from '../providers/PuppetDBEntityProviderConfig'; + +/** + * Customize the ingested Resource entity. + * + * @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform. + * @param config - The configuration for the entity provider. + * + * @returns A `ResourceEntity` or `undefined` if you want to ignore the found group for being ingested by the catalog. + * + * @public + */ +export type ResourceTransformer = ( + node: PuppetNode, + config: PuppetDBEntityProviderConfig, +) => Promise; + +/** + * A node in PuppetDB. + * + * @public + */ +export type PuppetNode = { + /** + * The most recent time of fact submission from the associated certname. + */ + timestamp: string; + /** + * The certname associated with the factset. + */ + certname: string; + /** + * A hash of the factset's certname, environment, timestamp, facts, and producer_timestamp. + */ + hash: string; + /** + * The most recent time of fact submission for the relevant certname from the Puppet Server. + */ + producer_timestamp: string; + /** + * The certname of the Puppet Server that sent the factset to PuppetDB. + */ + producer: string; + /** + * The environment associated with the fact. + */ + environment: string; + /** + * The facts associated with the factset. + */ + facts: PuppetFactSet; +}; + +/** + * The set of all facts for a single certname in PuppetDB. + * + * @public + */ +export type PuppetFactSet = { + /** + * The array of facts. + */ + data: PuppetFact[]; + /** + * The URL to retrieve more information about the facts. + */ + href: string; +}; + +/** + * A fact in PuppetDB. + * + * @public + */ +export type PuppetFact = { + /** + * The name of the fact. + */ + name: string; + /** + * The value of the fact, in JSON format. + */ + value: JsonValue; +}; diff --git a/yarn.lock b/yarn.lock index 33261fc7da..eb17085b2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5211,10 +5211,16 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-puppetdb-backend@workspace:plugins/catalog-backend-module-puppetdb" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 lodash: ^4.17.21 + luxon: ^3.0.0 msw: ^0.49.0 node-fetch: ^2.6.7 uuid: ^8.0.0 From 30483f692dab8dc0de33dd108b1b82643e5b9ef3 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 19:49:23 +0200 Subject: [PATCH 04/22] Add tests Signed-off-by: Tomas Dabasinskas --- .../providers/PuppetDBEntityProvider.test.ts | 218 ++++++++++++++++ .../PuppetDBEntityProviderConfig.test.ts | 129 +++++++++ .../src/puppet/read.test.ts | 244 ++++++++++++++++++ .../src/puppet/transformers.test.ts | 85 ++++++ 4 files changed, 676 insertions(+) create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts create mode 100644 plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts new file mode 100644 index 0000000000..3b665b94b4 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PuppetDBEntityProvider } from './PuppetDBEntityProvider'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import * as p from '../puppet/read'; +import { DEFAULT_OWNER } from './constants'; +import { ANNOTATION_PUPPET_CERTNAME } from '../puppet'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model/'; +import { ENDPOINT_NODES } from '../puppet/constants'; + +const logger = getVoidLogger(); + +jest.mock('../puppet/read', () => ({ + __esModule: true, + default: jest.fn(), + readPuppetNodes: null, +})); + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +describe('PuppetEntityProvider', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + host: 'http://puppetdb:8080', + schedule: { + frequency: { + minutes: 10, + }, + timeout: { + minutes: 10, + }, + }, + }, + }, + }, + }); + + describe('where there are no nodes', () => { + beforeEach(() => { + // @ts-ignore + p.readPuppetNodes = jest.fn().mockResolvedValue([]); + }); + + it('creates no entities', async () => { + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const providers = PuppetDBEntityProvider.fromConfig(config, { + logger, + schedule: new PersistingTaskRunner(), + }); + + await providers[0].connect(connection); + await providers[0].refresh(logger); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }); + }); + + describe('where there are nodes', () => { + beforeEach(() => { + // @ts-ignore + p.readPuppetNodes = jest.fn().mockResolvedValue([ + { + api_version: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }, + tags: ['windows'], + description: 'Description 1', + spec: { + type: 'virtual-machine', + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + { + api_version: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node2', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + }, + tags: ['linux'], + description: 'Description 2', + spec: { + type: 'physical-server', + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + ]); + }); + + it('creates entities', async () => { + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const providers = PuppetDBEntityProvider.fromConfig(config, { + logger, + schedule: new PersistingTaskRunner(), + }); + + await providers[0].connect(connection); + await providers[0].refresh(logger); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + locationKey: providers[0].getProviderName(), + entity: { + api_version: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + [ANNOTATION_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.host', + )}${ENDPOINT_NODES}/node1`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.host', + )}${ENDPOINT_NODES}/node1`, + }, + tags: ['windows'], + description: 'Description 1', + spec: { + type: 'virtual-machine', + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + }, + { + locationKey: providers[0].getProviderName(), + entity: { + api_version: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node2', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + [ANNOTATION_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.host', + )}${ENDPOINT_NODES}/node2`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.host', + )}${ENDPOINT_NODES}/node2`, + }, + tags: ['linux'], + description: 'Description 2', + spec: { + type: 'physical-server', + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + }, + ], + }); + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts new file mode 100644 index 0000000000..07fe54bbca --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './PuppetDBEntityProviderConfig'; +import { Duration } from 'luxon'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + host: 'https://puppetdb', + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('default'); + expect(providerConfigs[0].host).toEqual('https://puppetdb'); + }); + + it('single specific provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + 'my-provider': { + host: 'https://puppetdb', + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('my-provider'); + expect(providerConfigs[0].host).toEqual('https://puppetdb'); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + 'my-provider': { + host: 'https://my-puppet/', + query: 'my-query', + }, + 'your-provider': { + host: 'https://your-puppet', + query: 'your-query', + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(2); + expect(providerConfigs[0]).toEqual({ + id: 'my-provider', + host: 'https://my-puppet', + query: 'my-query', + }); + expect(providerConfigs[1]).toEqual({ + id: 'your-provider', + host: 'https://your-puppet', + query: 'your-query', + }); + }); + + it('provider config with schedule', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + host: 'https://puppetdb', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 10, + }, + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].schedule).toEqual({ + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 10, + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts new file mode 100644 index 0000000000..573be365dd --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -0,0 +1,244 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readPuppetNodes } from './read'; +import { + DEFAULT_PROVIDER_ID, + PuppetDBEntityProviderConfig, +} from '../providers'; +import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import { ANNOTATION_PUPPET_CERTNAME, ENDPOINT_FACTSETS } from './constants'; + +jest.mock('node-fetch', () => { + const original = jest.requireActual('node-fetch'); + return { + __esModule: true, + default: jest.fn(), + Headers: original.Headers, + }; +}); +(global as any).fetch = fetch; + +describe('readPuppetNodes', () => { + const mockFetch = fetch as unknown as jest.Mocked; + + describe('where no query is specified', () => { + const config: PuppetDBEntityProviderConfig = { + host: 'https://puppetdb', + id: DEFAULT_PROVIDER_ID, + }; + + beforeEach(async () => { + mockFetch.mockReturnValueOnce( + Promise.resolve( + new Response( + JSON.stringify([ + { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + facts: { + data: [ + { + name: 'is_virtual', + value: true, + }, + { + name: 'kernel', + value: 'Linux', + }, + { + name: 'ipaddress', + value: 'ipaddress1', + }, + { + name: 'clientnoop', + value: true, + }, + { + name: 'clientversion', + value: 'clientversion1', + }, + ], + }, + }, + { + certname: 'node2', + timestamp: 'time2', + hash: 'hash2', + producer_timestamp: 'producer_time2', + producer: 'producer2', + environment: 'environment2', + facts: { + data: [ + { + name: 'is_virtual', + value: false, + }, + { + name: 'kernel', + value: 'Windows', + }, + { + name: 'ipaddress', + value: 'ipaddress2', + }, + { + name: 'clientnoop', + value: false, + }, + { + name: 'clientversion', + value: 'clientversion2', + }, + ], + }, + }, + ]), + ), + ), + ); + }); + + describe('where custom transformer is used', () => { + it('should use it for transforming puppet nodes', async () => { + const entities = await readPuppetNodes(config, { + transformer: async (node, _config) => { + return { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: `custom-${node.certname}`, + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }; + }, + }); + + expect(entities).toHaveLength(2); + expect(entities[0]).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'custom-node1', + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }); + expect(entities[1]).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'custom-node2', + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }); + }); + }); + + describe('where default transformer is used', () => { + it('should use it for transforming puppet nodes', async () => { + const entities = await readPuppetNodes(config); + + expect(entities).toHaveLength(2); + expect(entities[0].metadata.annotations).toEqual({ + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }); + expect(entities[1].metadata.annotations).toEqual({ + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + }); + }); + }); + }); + + describe('where query is specified', () => { + const config: PuppetDBEntityProviderConfig = { + host: 'https://puppetdb', + id: DEFAULT_PROVIDER_ID, + query: '["=", "certname", "node1"]', + }; + + describe('where no results are matched', () => { + beforeEach(async () => { + mockFetch.mockReturnValueOnce( + Promise.resolve(new Response(JSON.stringify([]))), + ); + }); + + it('should return empty array', async () => { + const entities = await readPuppetNodes(config); + expect(entities).toHaveLength(0); + }); + }); + + describe('where results are matched', () => { + beforeEach(async () => { + mockFetch.mockReturnValueOnce( + Promise.resolve( + new Response( + JSON.stringify([ + { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + }, + ]), + ), + ), + ); + }); + + it('should return matched results', async () => { + const entities = await readPuppetNodes(config); + expect(mockFetch).toHaveBeenCalledWith( + `${config.host}${ENDPOINT_FACTSETS}?query=%5B%22%3D%22%2C+%22certname%22%2C+%22node1%22%5D`, + { + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + method: 'GET', + }, + ); + expect(entities).toHaveLength(1); + }); + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts new file mode 100644 index 0000000000..30ae79e87b --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PuppetDBEntityProviderConfig } from '../providers'; +import { PuppetNode } from './types'; +import { defaultResourceTransformer } from './transformers'; +import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { DEFAULT_OWNER } from '../providers'; +import { ANNOTATION_PUPPET_CERTNAME } from './constants'; + +describe('defaultResourceTransformer', () => { + it('should transform a puppet node to a resource entity', async () => { + const config: PuppetDBEntityProviderConfig = { + host: '', + id: '', + }; + const node: PuppetNode = { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + facts: { + href: 'facts1', + data: [ + { + name: 'kernel', + value: 'Linux', + }, + { + name: 'ipaddress', + value: 'ipaddress1', + }, + { + name: 'is_virtual', + value: true, + }, + { + name: 'clientnoop', + value: true, + }, + { + name: 'clientversion', + value: 'clientversion1', + }, + ], + }, + }; + + const entity = await defaultResourceTransformer(node, config); + expect(entity).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: DEFAULT_NAMESPACE, + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }, + description: 'ipaddress1', + tags: ['linux'], + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }); + }); +}); From 6746020caf741a540b55bcdf9d436d663d15582e Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 19:59:30 +0200 Subject: [PATCH 05/22] Update naming Signed-off-by: Tomas Dabasinskas --- .../config.d.ts | 2 +- ...test.ts => PuppetDbEntityProvider.test.ts} | 6 ++--- ...yProvider.ts => PuppetDbEntityProvider.ts} | 24 +++++++++---------- ...s => PuppetDbEntityProviderConfig.test.ts} | 2 +- ...fig.ts => PuppetDbEntityProviderConfig.ts} | 8 +++---- .../src/providers/index.ts | 4 ++-- .../src/puppet/read.test.ts | 6 ++--- .../src/puppet/read.ts | 4 ++-- .../src/puppet/transformers.test.ts | 4 ++-- .../src/puppet/types.ts | 4 ++-- 10 files changed, 32 insertions(+), 32 deletions(-) rename plugins/catalog-backend-module-puppetdb/src/providers/{PuppetDBEntityProvider.test.ts => PuppetDbEntityProvider.test.ts} (97%) rename plugins/catalog-backend-module-puppetdb/src/providers/{PuppetDBEntityProvider.ts => PuppetDbEntityProvider.ts} (91%) rename plugins/catalog-backend-module-puppetdb/src/providers/{PuppetDBEntityProviderConfig.test.ts => PuppetDbEntityProviderConfig.test.ts} (98%) rename plugins/catalog-backend-module-puppetdb/src/providers/{PuppetDBEntityProviderConfig.ts => PuppetDbEntityProviderConfig.ts} (93%) diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts index 6514629251..a79438944e 100644 --- a/plugins/catalog-backend-module-puppetdb/config.d.ts +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -40,7 +40,7 @@ export interface Config { } /** - * Configuration of {@link PuppetDBEntityProvider}. + * Configuration of {@link PuppetDbEntityProvider}. */ interface ProviderConfig { /** diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts similarity index 97% rename from plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts rename to plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index 3b665b94b4..7ea191190b 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -17,7 +17,7 @@ import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; -import { PuppetDBEntityProvider } from './PuppetDBEntityProvider'; +import { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import * as p from '../puppet/read'; import { DEFAULT_OWNER } from './constants'; @@ -79,7 +79,7 @@ describe('PuppetEntityProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const providers = PuppetDBEntityProvider.fromConfig(config, { + const providers = PuppetDbEntityProvider.fromConfig(config, { logger, schedule: new PersistingTaskRunner(), }); @@ -144,7 +144,7 @@ describe('PuppetEntityProvider', () => { applyMutation: jest.fn(), refresh: jest.fn(), }; - const providers = PuppetDBEntityProvider.fromConfig(config, { + const providers = PuppetDbEntityProvider.fromConfig(config, { logger, schedule: new PersistingTaskRunner(), }); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts similarity index 91% rename from plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts rename to plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts index a887d4778c..e6c19686a3 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProvider.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts @@ -20,9 +20,9 @@ import { } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { - PuppetDBEntityProviderConfig, + PuppetDbEntityProviderConfig, readProviderConfigs, -} from './PuppetDBEntityProviderConfig'; +} from './PuppetDbEntityProviderConfig'; import { Config } from '@backstage/config'; import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; import * as uuid from 'uuid'; @@ -42,20 +42,20 @@ import { ENDPOINT_NODES } from '../puppet/constants'; * * @public */ -export class PuppetDBEntityProvider implements EntityProvider { - private readonly config: PuppetDBEntityProviderConfig; +export class PuppetDbEntityProvider implements EntityProvider { + private readonly config: PuppetDbEntityProviderConfig; private readonly logger: Logger; private readonly scheduleFn: () => Promise; private readonly transformer: ResourceTransformer; private connection?: EntityProviderConnection; /** - * Creates instances of {@link PuppetDBEntityProvider} from a configuration. + * Creates instances of {@link PuppetDbEntityProvider} from a configuration. * * @param config - The configuration to read provider information from. - * @param deps - The dependencies for {@link PuppetDBEntityProvider}. + * @param deps - The dependencies for {@link PuppetDbEntityProvider}. * - * @returns A list of {@link PuppetDBEntityProvider} instances. + * @returns A list of {@link PuppetDbEntityProvider} instances. */ static fromConfig( config: Config, @@ -65,7 +65,7 @@ export class PuppetDBEntityProvider implements EntityProvider { scheduler?: PluginTaskScheduler; transformer?: ResourceTransformer; }, - ): PuppetDBEntityProvider[] { + ): PuppetDbEntityProvider[] { if (!deps.schedule && !deps.scheduler) { throw new Error('Either schedule or scheduler must be provided.'); } @@ -83,7 +83,7 @@ export class PuppetDBEntityProvider implements EntityProvider { const transformer = deps.transformer ?? defaultResourceTransformer; - return new PuppetDBEntityProvider( + return new PuppetDbEntityProvider( providerConfig, deps.logger, taskRunner, @@ -93,7 +93,7 @@ export class PuppetDBEntityProvider implements EntityProvider { } /** - * Creates an instance of {@link PuppetDBEntityProvider}. + * Creates an instance of {@link PuppetDbEntityProvider}. * * @param config - Configuration of the provider. * @param logger - The instance of a {@link Logger}. @@ -103,7 +103,7 @@ export class PuppetDBEntityProvider implements EntityProvider { * @private */ private constructor( - config: PuppetDBEntityProviderConfig, + config: PuppetDbEntityProviderConfig, logger: Logger, taskRunner: TaskRunner, transformer: ResourceTransformer, @@ -141,7 +141,7 @@ export class PuppetDBEntityProvider implements EntityProvider { id: taskId, fn: async () => { const logger = this.logger.child({ - class: PuppetDBEntityProvider.prototype.constructor.name, + class: PuppetDbEntityProvider.prototype.constructor.name, taskId, taskInstanceId: uuid.v4(), }); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts similarity index 98% rename from plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts rename to plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts index 07fe54bbca..517247fd6d 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { readProviderConfigs } from './PuppetDBEntityProviderConfig'; +import { readProviderConfigs } from './PuppetDbEntityProviderConfig'; import { Duration } from 'luxon'; describe('readProviderConfigs', () => { diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts similarity index 93% rename from plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts rename to plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts index e96a4a11e9..159a1e5504 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDBEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts @@ -22,11 +22,11 @@ import { Config } from '@backstage/config'; import { DEFAULT_PROVIDER_ID } from './constants'; /** - * Configuration of {@link PuppetDBEntityProvider}. + * Configuration of {@link PuppetDbEntityProvider}. * * @public */ -export type PuppetDBEntityProviderConfig = { +export type PuppetDbEntityProviderConfig = { /** * ID of the provider. */ @@ -54,7 +54,7 @@ export type PuppetDBEntityProviderConfig = { */ export function readProviderConfigs( config: Config, -): PuppetDBEntityProviderConfig[] { +): PuppetDbEntityProviderConfig[] { const providersConfig = config.getOptionalConfig( 'catalog.providers.puppetdb', ); @@ -82,7 +82,7 @@ export function readProviderConfigs( function readProviderConfig( id: string, config: Config, -): PuppetDBEntityProviderConfig { +): PuppetDbEntityProviderConfig { const host = config.getString('host').replace(/\/+$/, ''); const query = config.getOptionalString('query'); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts index 4e220395ba..aae8f0cd44 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export { PuppetDBEntityProvider } from './PuppetDBEntityProvider'; -export type { PuppetDBEntityProviderConfig } from './PuppetDBEntityProviderConfig'; +export { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; +export type { PuppetDbEntityProviderConfig } from './PuppetDbEntityProviderConfig'; export { DEFAULT_PROVIDER_ID, DEFAULT_OWNER } from './constants'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts index 573be365dd..afa18c0ba5 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -17,7 +17,7 @@ import { readPuppetNodes } from './read'; import { DEFAULT_PROVIDER_ID, - PuppetDBEntityProviderConfig, + PuppetDbEntityProviderConfig, } from '../providers'; import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import fetch from 'node-fetch'; @@ -37,7 +37,7 @@ describe('readPuppetNodes', () => { const mockFetch = fetch as unknown as jest.Mocked; describe('where no query is specified', () => { - const config: PuppetDBEntityProviderConfig = { + const config: PuppetDbEntityProviderConfig = { host: 'https://puppetdb', id: DEFAULT_PROVIDER_ID, }; @@ -186,7 +186,7 @@ describe('readPuppetNodes', () => { }); describe('where query is specified', () => { - const config: PuppetDBEntityProviderConfig = { + const config: PuppetDbEntityProviderConfig = { host: 'https://puppetdb', id: DEFAULT_PROVIDER_ID, query: '["=", "certname", "node1"]', diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts index edd33bc76a..8ecbb63d91 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PuppetDBEntityProviderConfig } from '../providers'; +import { PuppetDbEntityProviderConfig } from '../providers'; import { PuppetNode, ResourceTransformer } from './types'; import { ResourceEntity } from '@backstage/catalog-model/'; import { defaultResourceTransformer } from './transformers'; @@ -29,7 +29,7 @@ import { Logger } from 'winston'; * @param opts - Additional options. */ export async function readPuppetNodes( - config: PuppetDBEntityProviderConfig, + config: PuppetDbEntityProviderConfig, opts?: { transformer?: ResourceTransformer; logger?: Logger; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts index 30ae79e87b..e5c5698d99 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PuppetDBEntityProviderConfig } from '../providers'; +import { PuppetDbEntityProviderConfig } from '../providers'; import { PuppetNode } from './types'; import { defaultResourceTransformer } from './transformers'; import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; @@ -23,7 +23,7 @@ import { ANNOTATION_PUPPET_CERTNAME } from './constants'; describe('defaultResourceTransformer', () => { it('should transform a puppet node to a resource entity', async () => { - const config: PuppetDBEntityProviderConfig = { + const config: PuppetDbEntityProviderConfig = { host: '', id: '', }; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts index 7dc951f49f..f71ba299bd 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts @@ -16,7 +16,7 @@ import { ResourceEntity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; -import { PuppetDBEntityProviderConfig } from '../providers/PuppetDBEntityProviderConfig'; +import { PuppetDbEntityProviderConfig } from '../providers/PuppetDbEntityProviderConfig'; /** * Customize the ingested Resource entity. @@ -30,7 +30,7 @@ import { PuppetDBEntityProviderConfig } from '../providers/PuppetDBEntityProvide */ export type ResourceTransformer = ( node: PuppetNode, - config: PuppetDBEntityProviderConfig, + config: PuppetDbEntityProviderConfig, ) => Promise; /** From cb9341399a3ab7e61118cfa2b465977bff12392f Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 20:19:56 +0200 Subject: [PATCH 06/22] Generate API Report Signed-off-by: Tomas Dabasinskas --- .../api-report.md | 79 +++++++++++++++++++ .../config.d.ts | 5 +- .../providers/PuppetDbEntityProvider.test.ts | 3 +- .../src/providers/PuppetDbEntityProvider.ts | 2 +- .../src/providers/constants.ts | 7 -- .../src/providers/index.ts | 2 +- .../src/puppet/constants.ts | 7 ++ .../src/puppet/transformers.test.ts | 3 +- .../src/puppet/transformers.ts | 3 +- 9 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 plugins/catalog-backend-module-puppetdb/api-report.md diff --git a/plugins/catalog-backend-module-puppetdb/api-report.md b/plugins/catalog-backend-module-puppetdb/api-report.md new file mode 100644 index 0000000000..798e672f4c --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/api-report.md @@ -0,0 +1,79 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-puppetdb-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { JsonValue } from '@backstage/types'; +import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { ResourceEntity } from '@backstage/catalog-model'; +import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + +// @public +export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname'; + +// @public +export const DEFAULT_PROVIDER_ID = 'default'; + +// @public +export const defaultResourceTransformer: ResourceTransformer; + +// @public +export class PuppetDbEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + static fromConfig( + config: Config, + deps: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + transformer?: ResourceTransformer; + }, + ): PuppetDbEntityProvider[]; + // (undocumented) + getProviderName(): string; + refresh(logger: Logger): Promise; +} + +// @public +export type PuppetDbEntityProviderConfig = { + id: string; + host: string; + query?: string; + schedule?: TaskScheduleDefinition; +}; + +// @public +export type PuppetFact = { + name: string; + value: JsonValue; +}; + +// @public +export type PuppetFactSet = { + data: PuppetFact[]; + href: string; +}; + +// @public +export type PuppetNode = { + timestamp: string; + certname: string; + hash: string; + producer_timestamp: string; + producer: string; + environment: string; + facts: PuppetFactSet; +}; + +// @public +export type ResourceTransformer = ( + node: PuppetNode, + config: PuppetDbEntityProviderConfig, +) => Promise; +``` diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts index a79438944e..6b6c393d6b 100644 --- a/plugins/catalog-backend-module-puppetdb/config.d.ts +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -14,10 +14,7 @@ * limitations under the License. */ -import { - TaskScheduleDefinition, - TaskScheduleDefinitionConfig, -} from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; /** * Represents the configuration for the Backstage. diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index 7ea191190b..b049c16856 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -20,13 +20,12 @@ import { getVoidLogger } from '@backstage/backend-common'; import { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import * as p from '../puppet/read'; -import { DEFAULT_OWNER } from './constants'; import { ANNOTATION_PUPPET_CERTNAME } from '../puppet'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, } from '@backstage/catalog-model/'; -import { ENDPOINT_NODES } from '../puppet/constants'; +import { DEFAULT_OWNER, ENDPOINT_NODES } from '../puppet/constants'; const logger = getVoidLogger(); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts index e6c19686a3..5246c3373d 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts @@ -158,7 +158,7 @@ export class PuppetDbEntityProvider implements EntityProvider { /** * Refreshes the catalog by reading nodes from PuppetDB and registering them as Resource Entities. * - * @param logger - The instance of a {@link Logger}. + * @param logger - The instance of a Logger. */ async refresh(logger: Logger) { if (!this.connection) { diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts index 02b333c613..ca8a48b374 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts @@ -20,10 +20,3 @@ * @public */ export const DEFAULT_PROVIDER_ID = 'default'; - -/** - * Default owner for entities created by the PuppetDB provider. - * - * @public - */ -export const DEFAULT_OWNER = 'unknown'; diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts index aae8f0cd44..43d673c30c 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts @@ -16,4 +16,4 @@ export { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; export type { PuppetDbEntityProviderConfig } from './PuppetDbEntityProviderConfig'; -export { DEFAULT_PROVIDER_ID, DEFAULT_OWNER } from './constants'; +export { DEFAULT_PROVIDER_ID } from './constants'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts index ef355f22d3..672bc48932 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts @@ -30,3 +30,10 @@ export const ENDPOINT_FACTSETS = '/pdb/query/v4/factsets'; * Path of PuppetDB Nodes endpoint. */ export const ENDPOINT_NODES = '/pdb/query/v4/nodes'; + +/** + * Default owner for entities created by the PuppetDB provider. + * + * @public + */ +export const DEFAULT_OWNER = 'unknown'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts index e5c5698d99..f7d3813bf6 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -18,8 +18,7 @@ import { PuppetDbEntityProviderConfig } from '../providers'; import { PuppetNode } from './types'; import { defaultResourceTransformer } from './transformers'; import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { DEFAULT_OWNER } from '../providers'; -import { ANNOTATION_PUPPET_CERTNAME } from './constants'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_OWNER } from './constants'; describe('defaultResourceTransformer', () => { it('should transform a puppet node to a resource entity', async () => { diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts index ca0d2a4ccc..ec4c4a2af3 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -16,8 +16,7 @@ import { ResourceTransformer } from './types'; import { DEFAULT_NAMESPACE, ResourceEntity } from '@backstage/catalog-model'; -import { DEFAULT_OWNER } from '../providers'; -import { ANNOTATION_PUPPET_CERTNAME } from './constants'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_OWNER } from './constants'; /** * A default implementation of the {@link ResourceTransformer}. From a866a0a6b63f4f85cd59c7b18a18dc49314cce32 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 20:32:28 +0200 Subject: [PATCH 07/22] Fix package name Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/api-report.md | 2 +- plugins/catalog-backend-module-puppetdb/package.json | 2 +- yarn.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/api-report.md b/plugins/catalog-backend-module-puppetdb/api-report.md index 798e672f4c..84227af5d3 100644 --- a/plugins/catalog-backend-module-puppetdb/api-report.md +++ b/plugins/catalog-backend-module-puppetdb/api-report.md @@ -1,4 +1,4 @@ -## API Report File for "@backstage/plugin-catalog-backend-module-puppetdb-backend" +## API Report File for "@backstage/plugin-catalog-backend-module-puppetdb" > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json index fa7f08da1a..440c7e48af 100644 --- a/plugins/catalog-backend-module-puppetdb/package.json +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -1,5 +1,5 @@ { - "name": "@backstage/plugin-catalog-backend-module-puppetdb-backend", + "name": "@backstage/plugin-catalog-backend-module-puppetdb", "description": "A Backstage catalog backend module that helps integrate towards PuppetDB", "version": "0.0.1", "main": "src/index.ts", diff --git a/yarn.lock b/yarn.lock index eb17085b2d..7b71cdff35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5206,9 +5206,9 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-puppetdb-backend@workspace:plugins/catalog-backend-module-puppetdb": +"@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb": version: 0.0.0-use.local - resolution: "@backstage/plugin-catalog-backend-module-puppetdb-backend@workspace:plugins/catalog-backend-module-puppetdb" + resolution: "@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb" dependencies: "@backstage/backend-common": "workspace:^" "@backstage/backend-tasks": "workspace:^" From 3b8f0fbf01f7c52022ac0c41d02997538d38cd78 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 21:14:46 +0200 Subject: [PATCH 08/22] Rename default owner constant Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/package.json | 2 +- .../src/providers/PuppetDbEntityProvider.test.ts | 10 +++++----- .../src/puppet/constants.ts | 2 +- .../src/puppet/transformers.test.ts | 4 ++-- .../src/puppet/transformers.ts | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json index 440c7e48af..0ccb2fc24d 100644 --- a/plugins/catalog-backend-module-puppetdb/package.json +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -17,7 +17,7 @@ "repository": { "type": "git", "url": "https://github.com/backstage/backstage", - "directory": "catalog-backend-module-puppetdb" + "directory": "plugins/catalog-backend-module-puppetdb" }, "keywords": [ "backstage", diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index b049c16856..b72e0b8401 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -25,7 +25,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, } from '@backstage/catalog-model/'; -import { DEFAULT_OWNER, ENDPOINT_NODES } from '../puppet/constants'; +import { DEFAULT_ENTITY_OWNER, ENDPOINT_NODES } from '../puppet/constants'; const logger = getVoidLogger(); @@ -110,7 +110,7 @@ describe('PuppetEntityProvider', () => { description: 'Description 1', spec: { type: 'virtual-machine', - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, @@ -129,7 +129,7 @@ describe('PuppetEntityProvider', () => { description: 'Description 2', spec: { type: 'physical-server', - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, @@ -175,7 +175,7 @@ describe('PuppetEntityProvider', () => { description: 'Description 1', spec: { type: 'virtual-machine', - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, @@ -203,7 +203,7 @@ describe('PuppetEntityProvider', () => { description: 'Description 2', spec: { type: 'physical-server', - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts index 672bc48932..bf093182f2 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts @@ -36,4 +36,4 @@ export const ENDPOINT_NODES = '/pdb/query/v4/nodes'; * * @public */ -export const DEFAULT_OWNER = 'unknown'; +export const DEFAULT_ENTITY_OWNER = 'unknown'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts index f7d3813bf6..15f3bc1419 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -18,7 +18,7 @@ import { PuppetDbEntityProviderConfig } from '../providers'; import { PuppetNode } from './types'; import { defaultResourceTransformer } from './transformers'; import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_OWNER } from './constants'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; describe('defaultResourceTransformer', () => { it('should transform a puppet node to a resource entity', async () => { @@ -75,7 +75,7 @@ describe('defaultResourceTransformer', () => { }, spec: { type: 'virtual-machine', - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts index ec4c4a2af3..d99d89b1cf 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -16,7 +16,7 @@ import { ResourceTransformer } from './types'; import { DEFAULT_NAMESPACE, ResourceEntity } from '@backstage/catalog-model'; -import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_OWNER } from './constants'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; /** * A default implementation of the {@link ResourceTransformer}. @@ -54,7 +54,7 @@ export const defaultResourceTransformer: ResourceTransformer = async ( }, spec: { type: type, - owner: DEFAULT_OWNER, + owner: DEFAULT_ENTITY_OWNER, dependsOn: [], dependencyOf: [], }, From d2882d147fc83933b88e2adbac121ea255fffb02 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 22:26:25 +0200 Subject: [PATCH 09/22] Add README Signed-off-by: Tomas Dabasinskas --- .../catalog-backend-module-puppetdb/README.md | 95 +++++++++++++++++-- .../src/puppet/transformers.ts | 2 +- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md index c66627fa8b..9fea95d61e 100644 --- a/plugins/catalog-backend-module-puppetdb/README.md +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -1,14 +1,91 @@ -# catalog-backend-module-puppetdb +# Catalog Backend Module for Puppet -Welcome to the catalog-backend-module-puppetdb backend plugin! +This is an extension module to the `plugin-catalog-backend` plugin, providing an `PuppetDbEntityProvider` that can be used to ingest +[Resource entities](https://backstage.io/docs/features/software-catalog/descriptor-format#kind-resource) from a +[PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html) instance(s). This provider is useful if you want to import nodes +from your PuppetDB into Backstage. -_This plugin was created through the Backstage CLI_ +## Installation -## Getting started +The provider is not installed by default, therefore you have to add a dependency to `@backstage/plugin-catalog-backend-module-puppetdb` +to your backend package: -Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn -start` in the root directory, and then navigating to [/catalog-backend-module-puppetdb](http://localhost:3000/catalog-backend-module-puppetdb). +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-puppetdb +``` -You can also serve the plugin in isolation by running `yarn start` in the plugin directory. -This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. -It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. +Update the catalog plugin initialization in your backend to add the provider and schedule it: + +```diff ++ import { PuppetDbEntityProvider } from '@backstage/plugin-catalog-backend-module-puppetdb'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + ++ builder.addEntityProvider( ++ PuppetDbEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 10 }, ++ timeout: { minutes: 50 }, ++ initialDelay: { seconds: 15} ++ }), ++ }); ++ ); +``` + +After this, you also have to add some configuration in your app-config that describes what you want to import for that target. + +## Configuration + +The following configuration is an example of how a setup could look for importing nodes from an internal PuppetDB instance: + +```yaml +catalog: + providers: + puppet: + default: + # (Required) The host of PuppetDB API instance: + host: https://puppetdb.example.com + + # (Optional) Query to filter PuppetDB nodes: + #query: '["=","certname","example.com"]' +``` + +## Customize the Provider + +The default ingestion behaviour will likely not work for all use cases - you will want to set proper `Owner`, `System` and other fields for the +ingested resources. In case you want to customize the ingested entities, the provider allows to pass a transformer for resources. Here we will show an example +of overriding the default transformer. + +1. Create a transformer: +2. + +```ts +export const customResourceTransformer: ResourceTransformer = async ( + node, + config, +): Promise => { + // Transofrmations may change namespace, owner, change entity naming pattern, add labels, annotations, etc. + + // Create the Resource Entity on your own, or wrap the default transformer + return await defaultResourceTransformer(node, config); +}; +``` + +2. Configure the provider with the transformer: + +```ts +const puppetDbEntityProvider = PuppetDbEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 10 }, + timeout: { minutes: 50 }, + initialDelay: { seconds: 15 }, + }), + transformer: customResourceTransformer, +}); +``` diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts index d99d89b1cf..2062f8e828 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -31,7 +31,7 @@ import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; export const defaultResourceTransformer: ResourceTransformer = async ( node, _config, -) => { +): Promise => { const certName = node.certname.toLowerCase(); const type = node.facts?.data?.find(e => e.name === 'is_virtual')?.value ? 'virtual-machine' From 87263449c6a314c8478199c6da636d4ccbf2799d Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 22:34:06 +0200 Subject: [PATCH 10/22] Update plugins list in MicroSite Signed-off-by: Tomas Dabasinskas --- .../plugins/catalog-backend-module-puppetdb.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 microsite/data/plugins/catalog-backend-module-puppetdb.yaml diff --git a/microsite/data/plugins/catalog-backend-module-puppetdb.yaml b/microsite/data/plugins/catalog-backend-module-puppetdb.yaml new file mode 100644 index 0000000000..3eac117e19 --- /dev/null +++ b/microsite/data/plugins/catalog-backend-module-puppetdb.yaml @@ -0,0 +1,13 @@ +--- +title: PuppetDB Entity Provider +author: TDabasinskas +authorUrl: https://github.com/tdabasinskas +category: Configuration Management +description: Import nodes from PuppetDB into Backstage as Resource Entities +documentation: https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-puppetdb/README.md +iconUrl: https://avatars.githubusercontent.com/u/234268?s=200&v=4 +npmPackageName: '@backstage/plugin-catalog-backend-module-puppetdb' +tags: + - puppet + - puppetdb +addedDate: '2023-02-06' From a1efcf9a6581385f7dae344d06d0416991686551 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 22:41:03 +0200 Subject: [PATCH 11/22] Add changeset Signed-off-by: Tomas Dabasinskas --- .changeset/fifty-beds-dress.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fifty-beds-dress.md diff --git a/.changeset/fifty-beds-dress.md b/.changeset/fifty-beds-dress.md new file mode 100644 index 0000000000..19049732cc --- /dev/null +++ b/.changeset/fifty-beds-dress.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-puppetdb': minor +--- + +Initial version of the plugin. From e3048fc810f0b96af93eb0317aa91c438b922572 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Sun, 5 Feb 2023 23:02:05 +0200 Subject: [PATCH 12/22] Update configuration properties Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/README.md | 2 +- plugins/catalog-backend-module-puppetdb/config.d.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md index 9fea95d61e..3610c3e0fd 100644 --- a/plugins/catalog-backend-module-puppetdb/README.md +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -46,7 +46,7 @@ The following configuration is an example of how a setup could look for importin ```yaml catalog: providers: - puppet: + puppetdb: default: # (Required) The host of PuppetDB API instance: host: https://puppetdb.example.com diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts index 6b6c393d6b..dc48695f04 100644 --- a/plugins/catalog-backend-module-puppetdb/config.d.ts +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -29,9 +29,9 @@ export interface Config { */ providers?: { /** - * Puppet entity provider configuration. Uses "default" as default ID for the single config variant. + * PuppetDB Entity Provider configuration. Uses "default" as default ID for the single config variant. */ - puppet?: ProviderConfig | Record; + puppetdb?: ProviderConfig | Record; }; }; } From 29f496d2d7d3c7e1a10a739c87a0e962ff689be7 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Tue, 7 Feb 2023 07:58:41 +0200 Subject: [PATCH 13/22] Fix tests Signed-off-by: Tomas Dabasinskas --- .../src/puppet/read.test.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts index afa18c0ba5..8dcbff1a4c 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -23,15 +23,9 @@ import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import fetch from 'node-fetch'; import { ANNOTATION_PUPPET_CERTNAME, ENDPOINT_FACTSETS } from './constants'; -jest.mock('node-fetch', () => { - const original = jest.requireActual('node-fetch'); - return { - __esModule: true, - default: jest.fn(), - Headers: original.Headers, - }; -}); +jest.mock('node-fetch'); (global as any).fetch = fetch; +const { Response } = jest.requireActual('node-fetch'); describe('readPuppetNodes', () => { const mockFetch = fetch as unknown as jest.Mocked; From 968a62954f5ef3653a61bfeff268730b78497913 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 07:55:04 +0200 Subject: [PATCH 14/22] Update plugins/catalog-backend-module-puppetdb/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md index 3610c3e0fd..3cc2783561 100644 --- a/plugins/catalog-backend-module-puppetdb/README.md +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -69,7 +69,7 @@ export const customResourceTransformer: ResourceTransformer = async ( node, config, ): Promise => { - // Transofrmations may change namespace, owner, change entity naming pattern, add labels, annotations, etc. + // Transformations may change namespace, owner, change entity naming pattern, add labels, annotations, etc. // Create the Resource Entity on your own, or wrap the default transformer return await defaultResourceTransformer(node, config); From 0b05465b5e714d3ca88a66826bcfc579d3025610 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 07:57:44 +0200 Subject: [PATCH 15/22] Update README Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md index 3cc2783561..eeb363bd32 100644 --- a/plugins/catalog-backend-module-puppetdb/README.md +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -62,7 +62,6 @@ ingested resources. In case you want to customize the ingested entities, the pro of overriding the default transformer. 1. Create a transformer: -2. ```ts export const customResourceTransformer: ResourceTransformer = async ( From c4bdec7d49277f15aa75f5362a51dc058fc8fc1f Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 08:34:29 +0200 Subject: [PATCH 16/22] Address review Signed-off-by: Tomas Dabasinskas --- .../config.d.ts | 51 ++++++++++++------- .../src/providers/PuppetDbEntityProvider.ts | 7 ++- .../src/puppet/read.ts | 4 +- .../src/puppet/transformers.ts | 8 ++- .../src/puppet/types.ts | 2 +- yarn.lock | 34 +++++++++++++ 6 files changed, 74 insertions(+), 32 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts index dc48695f04..e890584f41 100644 --- a/plugins/catalog-backend-module-puppetdb/config.d.ts +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -31,25 +31,38 @@ export interface Config { /** * PuppetDB Entity Provider configuration. Uses "default" as default ID for the single config variant. */ - puppetdb?: ProviderConfig | Record; + puppetdb?: + | { + /** + * (Required) The host of PuppetDB API instance. + */ + host: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; + } + | Record< + string, + { + /** + * (Required) The host of PuppetDB API instance. + */ + host: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; + } + >; }; }; } - -/** - * Configuration of {@link PuppetDbEntityProvider}. - */ -interface ProviderConfig { - /** - * (Required) The host of PuppetDB API instance. - */ - host: string; - /** - * (Optional) PQL query to filter PuppetDB nodes. - */ - query?: string; - /** - * (Optional) Task schedule definition for the refresh. - */ - schedule?: TaskScheduleDefinition; -} diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts index 5246c3373d..a3ac4b140e 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts @@ -192,15 +192,14 @@ export class PuppetDbEntityProvider implements EntityProvider { * @returns Entity with @{@link ANNOTATION_LOCATION} and @{@link ANNOTATION_ORIGIN_LOCATION} annotations. */ function withLocations(host: string, entity: Entity): Entity { - const location = new URL(host); - location.pathname = `${ENDPOINT_NODES}/${entity.metadata?.name}`; + const location = `${host}/${ENDPOINT_NODES}/${entity.metadata?.name}`; return merge( { metadata: { annotations: { - [ANNOTATION_LOCATION]: `url:${location.toString()}`, - [ANNOTATION_ORIGIN_LOCATION]: `url:${location.toString()}`, + [ANNOTATION_LOCATION]: `url:${location}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${location}`, }, }, }, diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts index 8ecbb63d91..bbd7843ecc 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -44,9 +44,7 @@ export async function readPuppetNodes( } if (opts?.logger) { - opts.logger - .child({ url: url.toString() }) - .debug('Reading nodes from PuppetDB'); + opts.logger.debug('Reading nodes from PuppetDB', { url: url.toString() }); } const response = await fetch(url.toString(), { diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts index 2062f8e828..9ec607b83e 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -32,13 +32,13 @@ export const defaultResourceTransformer: ResourceTransformer = async ( node, _config, ): Promise => { - const certName = node.certname.toLowerCase(); + const certName = node.certname.toLocaleLowerCase('en-US'); const type = node.facts?.data?.find(e => e.name === 'is_virtual')?.value ? 'virtual-machine' : 'physical-server'; const kernel = node.facts?.data?.find(e => e.name === 'kernel')?.value; - const entity: ResourceEntity = { + return { apiVersion: 'backstage.io/v1beta1', kind: 'Resource', metadata: { @@ -50,7 +50,7 @@ export const defaultResourceTransformer: ResourceTransformer = async ( description: node.facts?.data ?.find(e => e.name === 'ipaddress') ?.value?.toString(), - tags: kernel ? [kernel.toString().toLowerCase()] : [], + tags: kernel ? [kernel.toString().toLocaleLowerCase('en-US')] : [], }, spec: { type: type, @@ -59,6 +59,4 @@ export const defaultResourceTransformer: ResourceTransformer = async ( dependencyOf: [], }, }; - - return entity; }; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts index f71ba299bd..5bfac5c998 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts @@ -96,7 +96,7 @@ export type PuppetFact = { */ name: string; /** - * The value of the fact, in JSON format. + * The value of the fact. */ value: JsonValue; }; diff --git a/yarn.lock b/yarn.lock index 7b71cdff35..2b88e6ce8c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -30043,6 +30043,40 @@ __metadata: languageName: node linkType: hard +"msw@npm:^0.49.0": + version: 0.49.3 + resolution: "msw@npm:0.49.3" + dependencies: + "@mswjs/cookies": ^0.2.2 + "@mswjs/interceptors": ^0.17.5 + "@open-draft/until": ^1.0.3 + "@types/cookie": ^0.4.1 + "@types/js-levenshtein": ^1.1.1 + chalk: 4.1.1 + chokidar: ^3.4.2 + cookie: ^0.4.2 + graphql: ^15.0.0 || ^16.0.0 + headers-polyfill: ^3.1.0 + inquirer: ^8.2.0 + is-node-process: ^1.0.1 + js-levenshtein: ^1.1.6 + node-fetch: ^2.6.7 + outvariant: ^1.3.0 + path-to-regexp: ^6.2.0 + strict-event-emitter: ^0.4.3 + type-fest: ^2.19.0 + yargs: ^17.3.1 + peerDependencies: + typescript: ">= 4.4.x <= 4.9.x" + peerDependenciesMeta: + typescript: + optional: true + bin: + msw: cli/index.js + checksum: 8322cd42cd69f289c05517d02bde22fc2f10e86fc2d0d209d9df54bd03d10b8123723c5587a2654dcd2cd0f314a016f9eccac88cffa30fafd1f9fead16db639e + languageName: node + linkType: hard + "msw@npm:^1.0.0, msw@npm:^1.0.1": version: 1.0.1 resolution: "msw@npm:1.0.1" From b9838936379f9d6b6e3c7e6ccc3ca7980267c53d Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 09:11:44 +0200 Subject: [PATCH 17/22] Fix tests Signed-off-by: Tomas Dabasinskas --- .../src/providers/PuppetDbEntityProvider.test.ts | 8 ++++---- .../src/puppet/constants.ts | 4 ++-- .../src/puppet/read.test.ts | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index b72e0b8401..e9dc5e0fae 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -166,10 +166,10 @@ describe('PuppetEntityProvider', () => { [ANNOTATION_PUPPET_CERTNAME]: 'node1', [ANNOTATION_LOCATION]: `url:${config.getString( 'catalog.providers.puppetdb.host', - )}${ENDPOINT_NODES}/node1`, + )}/${ENDPOINT_NODES}/node1`, [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( 'catalog.providers.puppetdb.host', - )}${ENDPOINT_NODES}/node1`, + )}/${ENDPOINT_NODES}/node1`, }, tags: ['windows'], description: 'Description 1', @@ -194,10 +194,10 @@ describe('PuppetEntityProvider', () => { [ANNOTATION_PUPPET_CERTNAME]: 'node2', [ANNOTATION_LOCATION]: `url:${config.getString( 'catalog.providers.puppetdb.host', - )}${ENDPOINT_NODES}/node2`, + )}/${ENDPOINT_NODES}/node2`, [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( 'catalog.providers.puppetdb.host', - )}${ENDPOINT_NODES}/node2`, + )}/${ENDPOINT_NODES}/node2`, }, tags: ['linux'], description: 'Description 2', diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts index bf093182f2..730f21ec16 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts @@ -24,12 +24,12 @@ export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname'; /** * Path of PuppetDB FactSets endpoint. */ -export const ENDPOINT_FACTSETS = '/pdb/query/v4/factsets'; +export const ENDPOINT_FACTSETS = 'pdb/query/v4/factsets'; /** * Path of PuppetDB Nodes endpoint. */ -export const ENDPOINT_NODES = '/pdb/query/v4/nodes'; +export const ENDPOINT_NODES = 'pdb/query/v4/nodes'; /** * Default owner for entities created by the PuppetDB provider. diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts index 8dcbff1a4c..e0bf5c6d43 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -222,7 +222,7 @@ describe('readPuppetNodes', () => { it('should return matched results', async () => { const entities = await readPuppetNodes(config); expect(mockFetch).toHaveBeenCalledWith( - `${config.host}${ENDPOINT_FACTSETS}?query=%5B%22%3D%22%2C+%22certname%22%2C+%22node1%22%5D`, + `${config.host}/${ENDPOINT_FACTSETS}?query=%5B%22%3D%22%2C+%22certname%22%2C+%22node1%22%5D`, { headers: { Accept: 'application/json', From 2128963db1dd4986c87902ca6fbc38dbcf4510d1 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 09:48:06 +0200 Subject: [PATCH 18/22] Improve nodes URL initialization Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/src/puppet/read.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts index bbd7843ecc..67247d73a5 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -36,8 +36,7 @@ export async function readPuppetNodes( }, ): Promise { const transformFn = opts?.transformer ?? defaultResourceTransformer; - const url = new URL(config.host); - url.pathname = ENDPOINT_FACTSETS; + const url = new URL(ENDPOINT_FACTSETS, config.host); if (config.query) { url.searchParams.set('query', config.query); From 518e317b6146e127a49a6a9c82b913b90c2be504 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 17:00:48 +0200 Subject: [PATCH 19/22] Use node-fetch and msw Signed-off-by: Tomas Dabasinskas --- .../package.json | 2 + .../src/puppet/read.test.ts | 63 ++++++++++--------- .../src/puppet/read.ts | 1 + yarn.lock | 2 + 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json index 0ccb2fc24d..93646e01e7 100644 --- a/plugins/catalog-backend-module-puppetdb/package.json +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -41,6 +41,7 @@ "@backstage/errors": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/types": "workspace:^", + "@types/node-fetch": "^2.5.12", "lodash": "^4.17.21", "luxon": "^3.0.0", "node-fetch": "^2.6.7", @@ -48,6 +49,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@types/lodash": "^4.14.151", "msw": "^0.49.0" diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts index e0bf5c6d43..78fc856066 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -20,15 +20,18 @@ import { PuppetDbEntityProviderConfig, } from '../providers'; import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; import { ANNOTATION_PUPPET_CERTNAME, ENDPOINT_FACTSETS } from './constants'; -jest.mock('node-fetch'); -(global as any).fetch = fetch; -const { Response } = jest.requireActual('node-fetch'); - describe('readPuppetNodes', () => { - const mockFetch = fetch as unknown as jest.Mocked; + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(() => { + jest.clearAllMocks(); + }); describe('where no query is specified', () => { const config: PuppetDbEntityProviderConfig = { @@ -37,10 +40,12 @@ describe('readPuppetNodes', () => { }; beforeEach(async () => { - mockFetch.mockReturnValueOnce( - Promise.resolve( - new Response( - JSON.stringify([ + worker.use( + rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([ { certname: 'node1', timestamp: 'time1', @@ -106,8 +111,8 @@ describe('readPuppetNodes', () => { }, }, ]), - ), - ), + ); + }), ); }); @@ -188,8 +193,14 @@ describe('readPuppetNodes', () => { describe('where no results are matched', () => { beforeEach(async () => { - mockFetch.mockReturnValueOnce( - Promise.resolve(new Response(JSON.stringify([]))), + worker.use( + rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([]), + ); + }), ); }); @@ -201,10 +212,12 @@ describe('readPuppetNodes', () => { describe('where results are matched', () => { beforeEach(async () => { - mockFetch.mockReturnValueOnce( - Promise.resolve( - new Response( - JSON.stringify([ + worker.use( + rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([ { certname: 'node1', timestamp: 'time1', @@ -214,23 +227,13 @@ describe('readPuppetNodes', () => { environment: 'environment1', }, ]), - ), - ), + ); + }), ); }); it('should return matched results', async () => { const entities = await readPuppetNodes(config); - expect(mockFetch).toHaveBeenCalledWith( - `${config.host}/${ENDPOINT_FACTSETS}?query=%5B%22%3D%22%2C+%22certname%22%2C+%22node1%22%5D`, - { - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - method: 'GET', - }, - ); expect(entities).toHaveLength(1); }); }); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts index 67247d73a5..601ee28207 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -18,6 +18,7 @@ import { PuppetDbEntityProviderConfig } from '../providers'; import { PuppetNode, ResourceTransformer } from './types'; import { ResourceEntity } from '@backstage/catalog-model/'; import { defaultResourceTransformer } from './transformers'; +import fetch from 'node-fetch'; import { ResponseError } from '@backstage/errors'; import { ENDPOINT_FACTSETS } from './constants'; import { Logger } from 'winston'; diff --git a/yarn.lock b/yarn.lock index 2b88e6ce8c..55750f13be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5212,6 +5212,7 @@ __metadata: dependencies: "@backstage/backend-common": "workspace:^" "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" @@ -5219,6 +5220,7 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 + "@types/node-fetch": ^2.5.12 lodash: ^4.17.21 luxon: ^3.0.0 msw: ^0.49.0 From abdb9aed3e63eed27b8b63cc2435976235ddf748 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Mon, 27 Feb 2023 18:58:40 +0200 Subject: [PATCH 20/22] Remove node-types Signed-off-by: Tomas Dabasinskas --- plugins/catalog-backend-module-puppetdb/package.json | 1 - yarn.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json index 93646e01e7..b6539dbbe6 100644 --- a/plugins/catalog-backend-module-puppetdb/package.json +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -41,7 +41,6 @@ "@backstage/errors": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/types": "workspace:^", - "@types/node-fetch": "^2.5.12", "lodash": "^4.17.21", "luxon": "^3.0.0", "node-fetch": "^2.6.7", diff --git a/yarn.lock b/yarn.lock index 55750f13be..6e5fa48907 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5220,7 +5220,6 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/types": "workspace:^" "@types/lodash": ^4.14.151 - "@types/node-fetch": ^2.5.12 lodash: ^4.17.21 luxon: ^3.0.0 msw: ^0.49.0 From aabfc0305b14e28e2b0eca239a742045e80f1007 Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Wed, 1 Mar 2023 08:29:12 +0200 Subject: [PATCH 21/22] Rename host to baseUrl Signed-off-by: Tomas Dabasinskas --- .../catalog-backend-module-puppetdb/README.md | 4 +- .../api-report.md | 2 +- .../config.d.ts | 8 +-- .../providers/PuppetDbEntityProvider.test.ts | 10 ++-- .../src/providers/PuppetDbEntityProvider.ts | 8 +-- .../PuppetDbEntityProviderConfig.test.ts | 18 +++--- .../providers/PuppetDbEntityProviderConfig.ts | 10 ++-- .../src/puppet/read.test.ts | 58 ++++++++++--------- .../src/puppet/read.ts | 2 +- .../src/puppet/transformers.test.ts | 2 +- 10 files changed, 64 insertions(+), 58 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md index eeb363bd32..d4782c6d18 100644 --- a/plugins/catalog-backend-module-puppetdb/README.md +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -48,8 +48,8 @@ catalog: providers: puppetdb: default: - # (Required) The host of PuppetDB API instance: - host: https://puppetdb.example.com + # (Required) The base URL of PuppetDB API instance: + baseUrl: https://puppetdb.example.com # (Optional) Query to filter PuppetDB nodes: #query: '["=","certname","example.com"]' diff --git a/plugins/catalog-backend-module-puppetdb/api-report.md b/plugins/catalog-backend-module-puppetdb/api-report.md index 84227af5d3..8764ac1e7c 100644 --- a/plugins/catalog-backend-module-puppetdb/api-report.md +++ b/plugins/catalog-backend-module-puppetdb/api-report.md @@ -43,7 +43,7 @@ export class PuppetDbEntityProvider implements EntityProvider { // @public export type PuppetDbEntityProviderConfig = { id: string; - host: string; + baseUrl: string; query?: string; schedule?: TaskScheduleDefinition; }; diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts index e890584f41..edd7ab3889 100644 --- a/plugins/catalog-backend-module-puppetdb/config.d.ts +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -34,9 +34,9 @@ export interface Config { puppetdb?: | { /** - * (Required) The host of PuppetDB API instance. + * (Required) The base URL of PuppetDB API instance. */ - host: string; + baseUrl: string; /** * (Optional) PQL query to filter PuppetDB nodes. */ @@ -50,9 +50,9 @@ export interface Config { string, { /** - * (Required) The host of PuppetDB API instance. + * (Required) The base URL of PuppetDB API instance. */ - host: string; + baseUrl: string; /** * (Optional) PQL query to filter PuppetDB nodes. */ diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index e9dc5e0fae..470bc5dbf8 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -53,7 +53,7 @@ describe('PuppetEntityProvider', () => { catalog: { providers: { puppetdb: { - host: 'http://puppetdb:8080', + baseUrl: 'http://puppetdb:8080', schedule: { frequency: { minutes: 10, @@ -165,10 +165,10 @@ describe('PuppetEntityProvider', () => { annotations: { [ANNOTATION_PUPPET_CERTNAME]: 'node1', [ANNOTATION_LOCATION]: `url:${config.getString( - 'catalog.providers.puppetdb.host', + 'catalog.providers.puppetdb.baseUrl', )}/${ENDPOINT_NODES}/node1`, [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( - 'catalog.providers.puppetdb.host', + 'catalog.providers.puppetdb.baseUrl', )}/${ENDPOINT_NODES}/node1`, }, tags: ['windows'], @@ -193,10 +193,10 @@ describe('PuppetEntityProvider', () => { annotations: { [ANNOTATION_PUPPET_CERTNAME]: 'node2', [ANNOTATION_LOCATION]: `url:${config.getString( - 'catalog.providers.puppetdb.host', + 'catalog.providers.puppetdb.baseUrl', )}/${ENDPOINT_NODES}/node2`, [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( - 'catalog.providers.puppetdb.host', + 'catalog.providers.puppetdb.baseUrl', )}/${ENDPOINT_NODES}/node2`, }, tags: ['linux'], diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts index a3ac4b140e..b2d2738fd3 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts @@ -176,7 +176,7 @@ export class PuppetDbEntityProvider implements EntityProvider { type: 'full', entities: [...entities].map(entity => ({ locationKey: this.getProviderName(), - entity: withLocations(this.config.host, entity), + entity: withLocations(this.config.baseUrl, entity), })), }); markCommitComplete(entities); @@ -186,13 +186,13 @@ export class PuppetDbEntityProvider implements EntityProvider { /** * Ensures the entities have required annotation data. * - * @param host - The host of the PuppetDB instance. + * @param baseUrl - The base URL of the PuppetDB instance. * @param entity - The entity to add the annotations to. * * @returns Entity with @{@link ANNOTATION_LOCATION} and @{@link ANNOTATION_ORIGIN_LOCATION} annotations. */ -function withLocations(host: string, entity: Entity): Entity { - const location = `${host}/${ENDPOINT_NODES}/${entity.metadata?.name}`; +function withLocations(baseUrl: string, entity: Entity): Entity { + const location = `${baseUrl}/${ENDPOINT_NODES}/${entity.metadata?.name}`; return merge( { diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts index 517247fd6d..8da0fb4150 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts @@ -33,7 +33,7 @@ describe('readProviderConfigs', () => { catalog: { providers: { puppetdb: { - host: 'https://puppetdb', + baseUrl: 'https://puppetdb', }, }, }, @@ -43,7 +43,7 @@ describe('readProviderConfigs', () => { expect(providerConfigs).toHaveLength(1); expect(providerConfigs[0].id).toEqual('default'); - expect(providerConfigs[0].host).toEqual('https://puppetdb'); + expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb'); }); it('single specific provider config', () => { @@ -52,7 +52,7 @@ describe('readProviderConfigs', () => { providers: { puppetdb: { 'my-provider': { - host: 'https://puppetdb', + baseUrl: 'https://puppetdb', }, }, }, @@ -63,7 +63,7 @@ describe('readProviderConfigs', () => { expect(providerConfigs).toHaveLength(1); expect(providerConfigs[0].id).toEqual('my-provider'); - expect(providerConfigs[0].host).toEqual('https://puppetdb'); + expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb'); }); it('multiple provider configs', () => { @@ -72,11 +72,11 @@ describe('readProviderConfigs', () => { providers: { puppetdb: { 'my-provider': { - host: 'https://my-puppet/', + baseUrl: 'https://my-puppet/', query: 'my-query', }, 'your-provider': { - host: 'https://your-puppet', + baseUrl: 'https://your-puppet', query: 'your-query', }, }, @@ -89,12 +89,12 @@ describe('readProviderConfigs', () => { expect(providerConfigs).toHaveLength(2); expect(providerConfigs[0]).toEqual({ id: 'my-provider', - host: 'https://my-puppet', + baseUrl: 'https://my-puppet', query: 'my-query', }); expect(providerConfigs[1]).toEqual({ id: 'your-provider', - host: 'https://your-puppet', + baseUrl: 'https://your-puppet', query: 'your-query', }); }); @@ -104,7 +104,7 @@ describe('readProviderConfigs', () => { catalog: { providers: { puppetdb: { - host: 'https://puppetdb', + baseUrl: 'https://puppetdb', schedule: { frequency: 'PT30M', timeout: { diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts index 159a1e5504..7b95b7f387 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts @@ -32,9 +32,9 @@ export type PuppetDbEntityProviderConfig = { */ id: string; /** - * (Required) The host of PuppetDB API instance. + * (Required) The base URL of PuppetDB API instance. */ - host: string; + baseUrl: string; /** * (Optional) PQL query to filter PuppetDB nodes. */ @@ -62,7 +62,7 @@ export function readProviderConfigs( return []; } - if (providersConfig.has('host')) { + if (providersConfig.has('baseUrl')) { return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; } @@ -83,7 +83,7 @@ function readProviderConfig( id: string, config: Config, ): PuppetDbEntityProviderConfig { - const host = config.getString('host').replace(/\/+$/, ''); + const baseUrl = config.getString('baseUrl').replace(/\/+$/, ''); const query = config.getOptionalString('query'); const schedule = config.has('schedule') @@ -92,7 +92,7 @@ function readProviderConfig( return { id, - host, + baseUrl, query, schedule, }; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts index 78fc856066..c3b750a417 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -35,13 +35,13 @@ describe('readPuppetNodes', () => { describe('where no query is specified', () => { const config: PuppetDbEntityProviderConfig = { - host: 'https://puppetdb', + baseUrl: 'https://puppetdb', id: DEFAULT_PROVIDER_ID, }; beforeEach(async () => { worker.use( - rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { + rest.get(`${config.baseUrl}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { return res( ctx.status(200), ctx.set('Content-Type', 'application/json'), @@ -186,7 +186,7 @@ describe('readPuppetNodes', () => { describe('where query is specified', () => { const config: PuppetDbEntityProviderConfig = { - host: 'https://puppetdb', + baseUrl: 'https://puppetdb', id: DEFAULT_PROVIDER_ID, query: '["=", "certname", "node1"]', }; @@ -194,13 +194,16 @@ describe('readPuppetNodes', () => { describe('where no results are matched', () => { beforeEach(async () => { worker.use( - rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { - return res( - ctx.status(200), - ctx.set('Content-Type', 'application/json'), - ctx.json([]), - ); - }), + rest.get( + `${config.baseUrl}/${ENDPOINT_FACTSETS}`, + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([]), + ); + }, + ), ); }); @@ -213,22 +216,25 @@ describe('readPuppetNodes', () => { describe('where results are matched', () => { beforeEach(async () => { worker.use( - rest.get(`${config.host}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { - return res( - ctx.status(200), - ctx.set('Content-Type', 'application/json'), - ctx.json([ - { - certname: 'node1', - timestamp: 'time1', - hash: 'hash1', - producer_timestamp: 'producer_time1', - producer: 'producer1', - environment: 'environment1', - }, - ]), - ); - }), + rest.get( + `${config.baseUrl}/${ENDPOINT_FACTSETS}`, + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([ + { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + }, + ]), + ); + }, + ), ); }); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts index 601ee28207..568ea7224e 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -37,7 +37,7 @@ export async function readPuppetNodes( }, ): Promise { const transformFn = opts?.transformer ?? defaultResourceTransformer; - const url = new URL(ENDPOINT_FACTSETS, config.host); + const url = new URL(ENDPOINT_FACTSETS, config.baseUrl); if (config.query) { url.searchParams.set('query', config.query); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts index 15f3bc1419..a56f7b549b 100644 --- a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -23,7 +23,7 @@ import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; describe('defaultResourceTransformer', () => { it('should transform a puppet node to a resource entity', async () => { const config: PuppetDbEntityProviderConfig = { - host: '', + baseUrl: '', id: '', }; const node: PuppetNode = { From 36155899a6f86136e8b55a2a8e845b51d03ab32d Mon Sep 17 00:00:00 2001 From: Tomas Dabasinskas Date: Wed, 1 Mar 2023 10:20:41 +0200 Subject: [PATCH 22/22] Update provider tests Signed-off-by: Tomas Dabasinskas --- .../providers/PuppetDbEntityProvider.test.ts | 86 ++++++++++--------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts index 470bc5dbf8..0ab6fb35b7 100644 --- a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -18,22 +18,26 @@ import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; -import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import * as p from '../puppet/read'; +import { + DeferredEntity, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import * as puppetFunctions from '../puppet/read'; import { ANNOTATION_PUPPET_CERTNAME } from '../puppet'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, + ResourceEntity, } from '@backstage/catalog-model/'; import { DEFAULT_ENTITY_OWNER, ENDPOINT_NODES } from '../puppet/constants'; -const logger = getVoidLogger(); +jest.mock('../puppet/read', () => { + return { + readPuppetNodes: jest.fn(), + }; +}); -jest.mock('../puppet/read', () => ({ - __esModule: true, - default: jest.fn(), - readPuppetNodes: null, -})); +const logger = getVoidLogger(); class PersistingTaskRunner implements TaskRunner { private tasks: TaskInvocationDefinition[] = []; @@ -69,8 +73,7 @@ describe('PuppetEntityProvider', () => { describe('where there are no nodes', () => { beforeEach(() => { - // @ts-ignore - p.readPuppetNodes = jest.fn().mockResolvedValue([]); + jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([]); }); it('creates no entities', async () => { @@ -95,10 +98,9 @@ describe('PuppetEntityProvider', () => { describe('where there are nodes', () => { beforeEach(() => { - // @ts-ignore - p.readPuppetNodes = jest.fn().mockResolvedValue([ + jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([ { - api_version: 'backstage.io/v1beta1', + apiVersion: 'backstage.io/v1beta1', kind: 'Resource', metadata: { name: 'node1', @@ -108,16 +110,16 @@ describe('PuppetEntityProvider', () => { }, tags: ['windows'], description: 'Description 1', - spec: { - type: 'virtual-machine', - owner: DEFAULT_ENTITY_OWNER, - dependsOn: [], - dependencyOf: [], - }, + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], }, }, { - api_version: 'backstage.io/v1beta1', + apiVersion: 'backstage.io/v1beta1', kind: 'Resource', metadata: { name: 'node2', @@ -127,15 +129,15 @@ describe('PuppetEntityProvider', () => { }, tags: ['linux'], description: 'Description 2', - spec: { - type: 'physical-server', - owner: DEFAULT_ENTITY_OWNER, - dependsOn: [], - dependencyOf: [], - }, + }, + spec: { + type: 'physical-server', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], }, }, - ]); + ] as ResourceEntity[]); }); it('creates entities', async () => { @@ -157,7 +159,7 @@ describe('PuppetEntityProvider', () => { { locationKey: providers[0].getProviderName(), entity: { - api_version: 'backstage.io/v1beta1', + apiVersion: 'backstage.io/v1beta1', kind: 'Resource', metadata: { name: 'node1', @@ -173,19 +175,19 @@ describe('PuppetEntityProvider', () => { }, tags: ['windows'], description: 'Description 1', - spec: { - type: 'virtual-machine', - owner: DEFAULT_ENTITY_OWNER, - dependsOn: [], - dependencyOf: [], - }, + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], }, }, }, { locationKey: providers[0].getProviderName(), entity: { - api_version: 'backstage.io/v1beta1', + apiVersion: 'backstage.io/v1beta1', kind: 'Resource', metadata: { name: 'node2', @@ -201,16 +203,16 @@ describe('PuppetEntityProvider', () => { }, tags: ['linux'], description: 'Description 2', - spec: { - type: 'physical-server', - owner: DEFAULT_ENTITY_OWNER, - dependsOn: [], - dependencyOf: [], - }, + }, + spec: { + type: 'physical-server', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], }, }, }, - ], + ] as DeferredEntity[], }); }); });