diff --git a/.changeset/fifty-beds-dress.md b/.changeset/fifty-beds-dress.md new file mode 100644 index 0000000000..19049732cc --- /dev/null +++ b/.changeset/fifty-beds-dress.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-puppetdb': minor +--- + +Initial version of the plugin. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 10f1ac188c..010af451af 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,6 +30,7 @@ yarn.lock @backstage/maintainers @back /plugins/catalog-backend-module-aws @backstage/maintainers @backstage/catalog-core @pjungermann /plugins/catalog-backend-module-bitbucket-cloud @backstage/maintainers @backstage/catalog-core @pjungermann /plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-core @pjungermann +/plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-core @tdabasinskas /plugins/catalog-graph @backstage/maintainers @backstage/catalog-core @backstage/sda-se-reviewers /plugins/circleci @backstage/maintainers @adamdmharvey /plugins/cloudbuild @backstage/maintainers @trivago/ebarrios diff --git a/microsite/data/plugins/catalog-backend-module-puppetdb.yaml b/microsite/data/plugins/catalog-backend-module-puppetdb.yaml new file mode 100644 index 0000000000..3eac117e19 --- /dev/null +++ b/microsite/data/plugins/catalog-backend-module-puppetdb.yaml @@ -0,0 +1,13 @@ +--- +title: PuppetDB Entity Provider +author: TDabasinskas +authorUrl: https://github.com/tdabasinskas +category: Configuration Management +description: Import nodes from PuppetDB into Backstage as Resource Entities +documentation: https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-puppetdb/README.md +iconUrl: https://avatars.githubusercontent.com/u/234268?s=200&v=4 +npmPackageName: '@backstage/plugin-catalog-backend-module-puppetdb' +tags: + - puppet + - puppetdb +addedDate: '2023-02-06' diff --git a/plugins/catalog-backend-module-puppetdb/.eslintrc.js b/plugins/catalog-backend-module-puppetdb/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-puppetdb/README.md b/plugins/catalog-backend-module-puppetdb/README.md new file mode 100644 index 0000000000..d4782c6d18 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/README.md @@ -0,0 +1,90 @@ +# Catalog Backend Module for Puppet + +This is an extension module to the `plugin-catalog-backend` plugin, providing an `PuppetDbEntityProvider` that can be used to ingest +[Resource entities](https://backstage.io/docs/features/software-catalog/descriptor-format#kind-resource) from a +[PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html) instance(s). This provider is useful if you want to import nodes +from your PuppetDB into Backstage. + +## Installation + +The provider is not installed by default, therefore you have to add a dependency to `@backstage/plugin-catalog-backend-module-puppetdb` +to your backend package: + +```bash +# From your Backstage root directory +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-puppetdb +``` + +Update the catalog plugin initialization in your backend to add the provider and schedule it: + +```diff ++ import { PuppetDbEntityProvider } from '@backstage/plugin-catalog-backend-module-puppetdb'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + ++ builder.addEntityProvider( ++ PuppetDbEntityProvider.fromConfig(env.config, { ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: { minutes: 10 }, ++ timeout: { minutes: 50 }, ++ initialDelay: { seconds: 15} ++ }), ++ }); ++ ); +``` + +After this, you also have to add some configuration in your app-config that describes what you want to import for that target. + +## Configuration + +The following configuration is an example of how a setup could look for importing nodes from an internal PuppetDB instance: + +```yaml +catalog: + providers: + puppetdb: + default: + # (Required) The base URL of PuppetDB API instance: + baseUrl: https://puppetdb.example.com + + # (Optional) Query to filter PuppetDB nodes: + #query: '["=","certname","example.com"]' +``` + +## Customize the Provider + +The default ingestion behaviour will likely not work for all use cases - you will want to set proper `Owner`, `System` and other fields for the +ingested resources. In case you want to customize the ingested entities, the provider allows to pass a transformer for resources. Here we will show an example +of overriding the default transformer. + +1. Create a transformer: + +```ts +export const customResourceTransformer: ResourceTransformer = async ( + node, + config, +): Promise => { + // Transformations may change namespace, owner, change entity naming pattern, add labels, annotations, etc. + + // Create the Resource Entity on your own, or wrap the default transformer + return await defaultResourceTransformer(node, config); +}; +``` + +2. Configure the provider with the transformer: + +```ts +const puppetDbEntityProvider = PuppetDbEntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 10 }, + timeout: { minutes: 50 }, + initialDelay: { seconds: 15 }, + }), + transformer: customResourceTransformer, +}); +``` diff --git a/plugins/catalog-backend-module-puppetdb/api-report.md b/plugins/catalog-backend-module-puppetdb/api-report.md new file mode 100644 index 0000000000..8764ac1e7c --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/api-report.md @@ -0,0 +1,79 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-puppetdb" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { JsonValue } from '@backstage/types'; +import { Logger } from 'winston'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { ResourceEntity } from '@backstage/catalog-model'; +import { TaskRunner } from '@backstage/backend-tasks'; +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + +// @public +export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname'; + +// @public +export const DEFAULT_PROVIDER_ID = 'default'; + +// @public +export const defaultResourceTransformer: ResourceTransformer; + +// @public +export class PuppetDbEntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + static fromConfig( + config: Config, + deps: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + transformer?: ResourceTransformer; + }, + ): PuppetDbEntityProvider[]; + // (undocumented) + getProviderName(): string; + refresh(logger: Logger): Promise; +} + +// @public +export type PuppetDbEntityProviderConfig = { + id: string; + baseUrl: string; + query?: string; + schedule?: TaskScheduleDefinition; +}; + +// @public +export type PuppetFact = { + name: string; + value: JsonValue; +}; + +// @public +export type PuppetFactSet = { + data: PuppetFact[]; + href: string; +}; + +// @public +export type PuppetNode = { + timestamp: string; + certname: string; + hash: string; + producer_timestamp: string; + producer: string; + environment: string; + facts: PuppetFactSet; +}; + +// @public +export type ResourceTransformer = ( + node: PuppetNode, + config: PuppetDbEntityProviderConfig, +) => Promise; +``` diff --git a/plugins/catalog-backend-module-puppetdb/config.d.ts b/plugins/catalog-backend-module-puppetdb/config.d.ts new file mode 100644 index 0000000000..edd7ab3889 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/config.d.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskScheduleDefinition } from '@backstage/backend-tasks'; + +/** + * Represents the configuration for the Backstage. + */ +export interface Config { + /** + * Configuration for the catalog. + */ + catalog?: { + /** + * Configuration for the providers. + */ + providers?: { + /** + * PuppetDB Entity Provider configuration. Uses "default" as default ID for the single config variant. + */ + puppetdb?: + | { + /** + * (Required) The base URL of PuppetDB API instance. + */ + baseUrl: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; + } + | Record< + string, + { + /** + * (Required) The base URL of PuppetDB API instance. + */ + baseUrl: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; + } + >; + }; + }; +} diff --git a/plugins/catalog-backend-module-puppetdb/package.json b/plugins/catalog-backend-module-puppetdb/package.json new file mode 100644 index 0000000000..b6539dbbe6 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/package.json @@ -0,0 +1,61 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-puppetdb", + "description": "A Backstage catalog backend module that helps integrate towards PuppetDB", + "version": "0.0.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-puppetdb" + }, + "keywords": [ + "backstage", + "puppetdb", + "puppet" + ], + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-common": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/types": "workspace:^", + "lodash": "^4.17.21", + "luxon": "^3.0.0", + "node-fetch": "^2.6.7", + "uuid": "^8.0.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", + "@backstage/cli": "workspace:^", + "@types/lodash": "^4.14.151", + "msw": "^0.49.0" + }, + "files": [ + "dist", + "config.d.ts" + ], + "configSchema": "config.d.ts" +} diff --git a/plugins/catalog-backend-module-puppetdb/src/index.ts b/plugins/catalog-backend-module-puppetdb/src/index.ts new file mode 100644 index 0000000000..b5c5f9d209 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that helps integrate towards PuppetDB + * + * @packageDocumentation + */ + +export * from './providers'; +export * from './puppet'; diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts new file mode 100644 index 0000000000..0ab6fb35b7 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.test.ts @@ -0,0 +1,219 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; +import { + DeferredEntity, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import * as puppetFunctions from '../puppet/read'; +import { ANNOTATION_PUPPET_CERTNAME } from '../puppet'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + ResourceEntity, +} from '@backstage/catalog-model/'; +import { DEFAULT_ENTITY_OWNER, ENDPOINT_NODES } from '../puppet/constants'; + +jest.mock('../puppet/read', () => { + return { + readPuppetNodes: jest.fn(), + }; +}); + +const logger = getVoidLogger(); + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +describe('PuppetEntityProvider', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + baseUrl: 'http://puppetdb:8080', + schedule: { + frequency: { + minutes: 10, + }, + timeout: { + minutes: 10, + }, + }, + }, + }, + }, + }); + + describe('where there are no nodes', () => { + beforeEach(() => { + jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([]); + }); + + it('creates no entities', async () => { + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const providers = PuppetDbEntityProvider.fromConfig(config, { + logger, + schedule: new PersistingTaskRunner(), + }); + + await providers[0].connect(connection); + await providers[0].refresh(logger); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], + }); + }); + }); + + describe('where there are nodes', () => { + beforeEach(() => { + jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([ + { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }, + tags: ['windows'], + description: 'Description 1', + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node2', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + }, + tags: ['linux'], + description: 'Description 2', + }, + spec: { + type: 'physical-server', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + ] as ResourceEntity[]); + }); + + it('creates entities', async () => { + const connection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + const providers = PuppetDbEntityProvider.fromConfig(config, { + logger, + schedule: new PersistingTaskRunner(), + }); + + await providers[0].connect(connection); + await providers[0].refresh(logger); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [ + { + locationKey: providers[0].getProviderName(), + entity: { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + [ANNOTATION_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.baseUrl', + )}/${ENDPOINT_NODES}/node1`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.baseUrl', + )}/${ENDPOINT_NODES}/node1`, + }, + tags: ['windows'], + description: 'Description 1', + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + { + locationKey: providers[0].getProviderName(), + entity: { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node2', + namespace: 'default', + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + [ANNOTATION_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.baseUrl', + )}/${ENDPOINT_NODES}/node2`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString( + 'catalog.providers.puppetdb.baseUrl', + )}/${ENDPOINT_NODES}/node2`, + }, + tags: ['linux'], + description: 'Description 2', + }, + spec: { + type: 'physical-server', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }, + }, + ] as DeferredEntity[], + }); + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts new file mode 100644 index 0000000000..b2d2738fd3 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProvider.ts @@ -0,0 +1,235 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; +import { Logger } from 'winston'; +import { + PuppetDbEntityProviderConfig, + readProviderConfigs, +} from './PuppetDbEntityProviderConfig'; +import { Config } from '@backstage/config'; +import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks'; +import * as uuid from 'uuid'; +import { ResourceTransformer, defaultResourceTransformer } from '../puppet'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + Entity, +} from '@backstage/catalog-model/'; +import { merge } from 'lodash'; +import { readPuppetNodes } from '../puppet/read'; +import { ENDPOINT_NODES } from '../puppet/constants'; + +/** + * Reads nodes from [PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html) + * based on the provided query and registers them as Resource entities in the catalog. + * + * @public + */ +export class PuppetDbEntityProvider implements EntityProvider { + private readonly config: PuppetDbEntityProviderConfig; + private readonly logger: Logger; + private readonly scheduleFn: () => Promise; + private readonly transformer: ResourceTransformer; + private connection?: EntityProviderConnection; + + /** + * Creates instances of {@link PuppetDbEntityProvider} from a configuration. + * + * @param config - The configuration to read provider information from. + * @param deps - The dependencies for {@link PuppetDbEntityProvider}. + * + * @returns A list of {@link PuppetDbEntityProvider} instances. + */ + static fromConfig( + config: Config, + deps: { + logger: Logger; + schedule?: TaskRunner; + scheduler?: PluginTaskScheduler; + transformer?: ResourceTransformer; + }, + ): PuppetDbEntityProvider[] { + if (!deps.schedule && !deps.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return readProviderConfigs(config).map(providerConfig => { + if (!deps.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for puppet-provider:${providerConfig.id}.`, + ); + } + + const taskRunner = + deps.schedule ?? + deps.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + const transformer = deps.transformer ?? defaultResourceTransformer; + + return new PuppetDbEntityProvider( + providerConfig, + deps.logger, + taskRunner, + transformer, + ); + }); + } + + /** + * Creates an instance of {@link PuppetDbEntityProvider}. + * + * @param config - Configuration of the provider. + * @param logger - The instance of a {@link Logger}. + * @param taskRunner - The instance of {@link TaskRunner}. + * @param transformer - A {@link ResourceTransformer} function. + * + * @private + */ + private constructor( + config: PuppetDbEntityProviderConfig, + logger: Logger, + taskRunner: TaskRunner, + transformer: ResourceTransformer, + ) { + this.config = config; + this.logger = logger.child({ + target: this.getProviderName(), + }); + this.scheduleFn = this.createScheduleFn(taskRunner); + this.transformer = transformer; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `puppetdb-provider:${this.config.id}`; + } + + /** + * Creates a function that can be used to schedule a refresh of the catalog. + * + * @param taskRunner - The instance of {@link TaskRunner}. + * + * @private + */ + private createScheduleFn(taskRunner: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: PuppetDbEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + try { + await this.refresh(logger); + } catch (error) { + logger.error(`${this.getProviderName()} refresh failed`, error); + } + }, + }); + }; + } + + /** + * Refreshes the catalog by reading nodes from PuppetDB and registering them as Resource Entities. + * + * @param logger - The instance of a Logger. + */ + async refresh(logger: Logger) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + const { markReadComplete } = trackProgress(logger); + const entities = await readPuppetNodes(this.config, { + logger, + transformer: this.transformer, + }); + const { markCommitComplete } = markReadComplete(entities); + + await this.connection.applyMutation({ + type: 'full', + entities: [...entities].map(entity => ({ + locationKey: this.getProviderName(), + entity: withLocations(this.config.baseUrl, entity), + })), + }); + markCommitComplete(entities); + } +} + +/** + * Ensures the entities have required annotation data. + * + * @param baseUrl - The base URL of the PuppetDB instance. + * @param entity - The entity to add the annotations to. + * + * @returns Entity with @{@link ANNOTATION_LOCATION} and @{@link ANNOTATION_ORIGIN_LOCATION} annotations. + */ +function withLocations(baseUrl: string, entity: Entity): Entity { + const location = `${baseUrl}/${ENDPOINT_NODES}/${entity.metadata?.name}`; + + return merge( + { + metadata: { + annotations: { + [ANNOTATION_LOCATION]: `url:${location}`, + [ANNOTATION_ORIGIN_LOCATION]: `url:${location}`, + }, + }, + }, + entity, + ) as Entity; +} + +/** + * Tracks the progress of the PuppetDB read and commit operations. + * + * @param logger - The instance of a {@link Logger}. + */ +function trackProgress(logger: Logger) { + let timestamp = Date.now(); + + function markReadComplete(entities: Entity[]) { + const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + timestamp = Date.now(); + logger.info( + `Read ${entities?.length ?? 0} in ${readDuration} seconds. Committing...`, + ); + return { markCommitComplete }; + } + + function markCommitComplete(entities: Entity[]) { + const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1); + logger.info( + `Committed ${entities?.length ?? 0} in ${commitDuration} seconds.`, + ); + } + + return { markReadComplete }; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts new file mode 100644 index 0000000000..8da0fb4150 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readProviderConfigs } from './PuppetDbEntityProviderConfig'; +import { Duration } from 'luxon'; + +describe('readProviderConfigs', () => { + afterEach(() => jest.resetAllMocks()); + + it('no provider config', () => { + const config = new ConfigReader({}); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(0); + }); + + it('single simple provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + baseUrl: 'https://puppetdb', + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('default'); + expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb'); + }); + + it('single specific provider config', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + 'my-provider': { + baseUrl: 'https://puppetdb', + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].id).toEqual('my-provider'); + expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb'); + }); + + it('multiple provider configs', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + 'my-provider': { + baseUrl: 'https://my-puppet/', + query: 'my-query', + }, + 'your-provider': { + baseUrl: 'https://your-puppet', + query: 'your-query', + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(2); + expect(providerConfigs[0]).toEqual({ + id: 'my-provider', + baseUrl: 'https://my-puppet', + query: 'my-query', + }); + expect(providerConfigs[1]).toEqual({ + id: 'your-provider', + baseUrl: 'https://your-puppet', + query: 'your-query', + }); + }); + + it('provider config with schedule', () => { + const config = new ConfigReader({ + catalog: { + providers: { + puppetdb: { + baseUrl: 'https://puppetdb', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 10, + }, + }, + }, + }, + }, + }); + + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs).toHaveLength(1); + expect(providerConfigs[0].schedule).toEqual({ + frequency: Duration.fromISO('PT30M'), + timeout: { + minutes: 10, + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts new file mode 100644 index 0000000000..7b95b7f387 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/PuppetDbEntityProviderConfig.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + readTaskScheduleDefinitionFromConfig, + TaskScheduleDefinition, +} from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { DEFAULT_PROVIDER_ID } from './constants'; + +/** + * Configuration of {@link PuppetDbEntityProvider}. + * + * @public + */ +export type PuppetDbEntityProviderConfig = { + /** + * ID of the provider. + */ + id: string; + /** + * (Required) The base URL of PuppetDB API instance. + */ + baseUrl: string; + /** + * (Optional) PQL query to filter PuppetDB nodes. + */ + query?: string; + /** + * (Optional) Task schedule definition for the refresh. + */ + schedule?: TaskScheduleDefinition; +}; + +/** + * Reads the configuration of the PuppetDB Entity Providers. + * + * @param config - The application configuration. + * + * @returns PuppetDB Entity Provider configurations list. + */ +export function readProviderConfigs( + config: Config, +): PuppetDbEntityProviderConfig[] { + const providersConfig = config.getOptionalConfig( + 'catalog.providers.puppetdb', + ); + if (!providersConfig) { + return []; + } + + if (providersConfig.has('baseUrl')) { + return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)]; + } + + return providersConfig.keys().map(id => { + return readProviderConfig(id, providersConfig.getConfig(id)); + }); +} + +/** + * Reads the configuration for the PuppetDB Entity Provider. + * + * @param id - ID of the provider. + * @param config - The application configuration. + * + * @returns The PuppetDB Entity Provider configuration. + */ +function readProviderConfig( + id: string, + config: Config, +): PuppetDbEntityProviderConfig { + const baseUrl = config.getString('baseUrl').replace(/\/+$/, ''); + const query = config.getOptionalString('query'); + + const schedule = config.has('schedule') + ? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule')) + : undefined; + + return { + id, + baseUrl, + query, + schedule, + }; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts new file mode 100644 index 0000000000..ca8a48b374 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/constants.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Name of the default provider when a using a simple configuration. + * + * @public + */ +export const DEFAULT_PROVIDER_ID = 'default'; diff --git a/plugins/catalog-backend-module-puppetdb/src/providers/index.ts b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts new file mode 100644 index 0000000000..43d673c30c --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/providers/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { PuppetDbEntityProvider } from './PuppetDbEntityProvider'; +export type { PuppetDbEntityProviderConfig } from './PuppetDbEntityProviderConfig'; +export { DEFAULT_PROVIDER_ID } from './constants'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts new file mode 100644 index 0000000000..730f21ec16 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/constants.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Annotation for specifying the certificate name of a node in PuppetDB. + * + * @public + */ +export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname'; + +/** + * Path of PuppetDB FactSets endpoint. + */ +export const ENDPOINT_FACTSETS = 'pdb/query/v4/factsets'; + +/** + * Path of PuppetDB Nodes endpoint. + */ +export const ENDPOINT_NODES = 'pdb/query/v4/nodes'; + +/** + * Default owner for entities created by the PuppetDB provider. + * + * @public + */ +export const DEFAULT_ENTITY_OWNER = 'unknown'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts new file mode 100644 index 0000000000..644d111ac1 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ANNOTATION_PUPPET_CERTNAME } from './constants'; +export { defaultResourceTransformer } from './transformers'; +export type { + PuppetNode, + PuppetFactSet, + PuppetFact, + ResourceTransformer, +} from './types'; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts new file mode 100644 index 0000000000..c3b750a417 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readPuppetNodes } from './read'; +import { + DEFAULT_PROVIDER_ID, + PuppetDbEntityProviderConfig, +} from '../providers'; +import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { ANNOTATION_PUPPET_CERTNAME, ENDPOINT_FACTSETS } from './constants'; + +describe('readPuppetNodes', () => { + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('where no query is specified', () => { + const config: PuppetDbEntityProviderConfig = { + baseUrl: 'https://puppetdb', + id: DEFAULT_PROVIDER_ID, + }; + + beforeEach(async () => { + worker.use( + rest.get(`${config.baseUrl}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([ + { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + facts: { + data: [ + { + name: 'is_virtual', + value: true, + }, + { + name: 'kernel', + value: 'Linux', + }, + { + name: 'ipaddress', + value: 'ipaddress1', + }, + { + name: 'clientnoop', + value: true, + }, + { + name: 'clientversion', + value: 'clientversion1', + }, + ], + }, + }, + { + certname: 'node2', + timestamp: 'time2', + hash: 'hash2', + producer_timestamp: 'producer_time2', + producer: 'producer2', + environment: 'environment2', + facts: { + data: [ + { + name: 'is_virtual', + value: false, + }, + { + name: 'kernel', + value: 'Windows', + }, + { + name: 'ipaddress', + value: 'ipaddress2', + }, + { + name: 'clientnoop', + value: false, + }, + { + name: 'clientversion', + value: 'clientversion2', + }, + ], + }, + }, + ]), + ); + }), + ); + }); + + describe('where custom transformer is used', () => { + it('should use it for transforming puppet nodes', async () => { + const entities = await readPuppetNodes(config, { + transformer: async (node, _config) => { + return { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: `custom-${node.certname}`, + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }; + }, + }); + + expect(entities).toHaveLength(2); + expect(entities[0]).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'custom-node1', + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }); + expect(entities[1]).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'custom-node2', + namespace: DEFAULT_NAMESPACE, + }, + spec: { + type: 'Custom', + owner: 'Custom', + dependsOn: [], + dependencyOf: [], + }, + }); + }); + }); + + describe('where default transformer is used', () => { + it('should use it for transforming puppet nodes', async () => { + const entities = await readPuppetNodes(config); + + expect(entities).toHaveLength(2); + expect(entities[0].metadata.annotations).toEqual({ + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }); + expect(entities[1].metadata.annotations).toEqual({ + [ANNOTATION_PUPPET_CERTNAME]: 'node2', + }); + }); + }); + }); + + describe('where query is specified', () => { + const config: PuppetDbEntityProviderConfig = { + baseUrl: 'https://puppetdb', + id: DEFAULT_PROVIDER_ID, + query: '["=", "certname", "node1"]', + }; + + describe('where no results are matched', () => { + beforeEach(async () => { + worker.use( + rest.get( + `${config.baseUrl}/${ENDPOINT_FACTSETS}`, + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([]), + ); + }, + ), + ); + }); + + it('should return empty array', async () => { + const entities = await readPuppetNodes(config); + expect(entities).toHaveLength(0); + }); + }); + + describe('where results are matched', () => { + beforeEach(async () => { + worker.use( + rest.get( + `${config.baseUrl}/${ENDPOINT_FACTSETS}`, + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json([ + { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + }, + ]), + ); + }, + ), + ); + }); + + it('should return matched results', async () => { + const entities = await readPuppetNodes(config); + expect(entities).toHaveLength(1); + }); + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts new file mode 100644 index 0000000000..568ea7224e --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/read.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PuppetDbEntityProviderConfig } from '../providers'; +import { PuppetNode, ResourceTransformer } from './types'; +import { ResourceEntity } from '@backstage/catalog-model/'; +import { defaultResourceTransformer } from './transformers'; +import fetch from 'node-fetch'; +import { ResponseError } from '@backstage/errors'; +import { ENDPOINT_FACTSETS } from './constants'; +import { Logger } from 'winston'; + +/** + * Reads nodes and their facts from PuppetDB. + * + * @param config - The provider configuration. + * @param opts - Additional options. + */ +export async function readPuppetNodes( + config: PuppetDbEntityProviderConfig, + opts?: { + transformer?: ResourceTransformer; + logger?: Logger; + }, +): Promise { + const transformFn = opts?.transformer ?? defaultResourceTransformer; + const url = new URL(ENDPOINT_FACTSETS, config.baseUrl); + + if (config.query) { + url.searchParams.set('query', config.query); + } + + if (opts?.logger) { + opts.logger.debug('Reading nodes from PuppetDB', { url: url.toString() }); + } + + const response = await fetch(url.toString(), { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + }); + + if (!response.ok) { + throw await ResponseError.fromResponse(response); + } + + const nodes = (await response.json()) as PuppetNode[]; + const entities: ResourceEntity[] = []; + + for (const node of nodes) { + const entity = await transformFn(node, config); + if (entity) { + entities.push(entity); + } + } + + return entities; +} diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts new file mode 100644 index 0000000000..a56f7b549b --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PuppetDbEntityProviderConfig } from '../providers'; +import { PuppetNode } from './types'; +import { defaultResourceTransformer } from './transformers'; +import { DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; + +describe('defaultResourceTransformer', () => { + it('should transform a puppet node to a resource entity', async () => { + const config: PuppetDbEntityProviderConfig = { + baseUrl: '', + id: '', + }; + const node: PuppetNode = { + certname: 'node1', + timestamp: 'time1', + hash: 'hash1', + producer_timestamp: 'producer_time1', + producer: 'producer1', + environment: 'environment1', + facts: { + href: 'facts1', + data: [ + { + name: 'kernel', + value: 'Linux', + }, + { + name: 'ipaddress', + value: 'ipaddress1', + }, + { + name: 'is_virtual', + value: true, + }, + { + name: 'clientnoop', + value: true, + }, + { + name: 'clientversion', + value: 'clientversion1', + }, + ], + }, + }; + + const entity = await defaultResourceTransformer(node, config); + expect(entity).toEqual({ + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: 'node1', + namespace: DEFAULT_NAMESPACE, + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: 'node1', + }, + description: 'ipaddress1', + tags: ['linux'], + }, + spec: { + type: 'virtual-machine', + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }); + }); +}); diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts new file mode 100644 index 0000000000..9ec607b83e --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/transformers.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ResourceTransformer } from './types'; +import { DEFAULT_NAMESPACE, ResourceEntity } from '@backstage/catalog-model'; +import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants'; + +/** + * A default implementation of the {@link ResourceTransformer}. + * + * @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform. + * @param _config - The configuration for the entity provider. + * + * @returns A `ResourceEntity`. + * + * @public + */ +export const defaultResourceTransformer: ResourceTransformer = async ( + node, + _config, +): Promise => { + const certName = node.certname.toLocaleLowerCase('en-US'); + const type = node.facts?.data?.find(e => e.name === 'is_virtual')?.value + ? 'virtual-machine' + : 'physical-server'; + const kernel = node.facts?.data?.find(e => e.name === 'kernel')?.value; + + return { + apiVersion: 'backstage.io/v1beta1', + kind: 'Resource', + metadata: { + name: certName, + annotations: { + [ANNOTATION_PUPPET_CERTNAME]: certName, + }, + namespace: DEFAULT_NAMESPACE, + description: node.facts?.data + ?.find(e => e.name === 'ipaddress') + ?.value?.toString(), + tags: kernel ? [kernel.toString().toLocaleLowerCase('en-US')] : [], + }, + spec: { + type: type, + owner: DEFAULT_ENTITY_OWNER, + dependsOn: [], + dependencyOf: [], + }, + }; +}; diff --git a/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts new file mode 100644 index 0000000000..5bfac5c998 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/puppet/types.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ResourceEntity } from '@backstage/catalog-model'; +import { JsonValue } from '@backstage/types'; +import { PuppetDbEntityProviderConfig } from '../providers/PuppetDbEntityProviderConfig'; + +/** + * Customize the ingested Resource entity. + * + * @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform. + * @param config - The configuration for the entity provider. + * + * @returns A `ResourceEntity` or `undefined` if you want to ignore the found group for being ingested by the catalog. + * + * @public + */ +export type ResourceTransformer = ( + node: PuppetNode, + config: PuppetDbEntityProviderConfig, +) => Promise; + +/** + * A node in PuppetDB. + * + * @public + */ +export type PuppetNode = { + /** + * The most recent time of fact submission from the associated certname. + */ + timestamp: string; + /** + * The certname associated with the factset. + */ + certname: string; + /** + * A hash of the factset's certname, environment, timestamp, facts, and producer_timestamp. + */ + hash: string; + /** + * The most recent time of fact submission for the relevant certname from the Puppet Server. + */ + producer_timestamp: string; + /** + * The certname of the Puppet Server that sent the factset to PuppetDB. + */ + producer: string; + /** + * The environment associated with the fact. + */ + environment: string; + /** + * The facts associated with the factset. + */ + facts: PuppetFactSet; +}; + +/** + * The set of all facts for a single certname in PuppetDB. + * + * @public + */ +export type PuppetFactSet = { + /** + * The array of facts. + */ + data: PuppetFact[]; + /** + * The URL to retrieve more information about the facts. + */ + href: string; +}; + +/** + * A fact in PuppetDB. + * + * @public + */ +export type PuppetFact = { + /** + * The name of the fact. + */ + name: string; + /** + * The value of the fact. + */ + value: JsonValue; +}; diff --git a/plugins/catalog-backend-module-puppetdb/src/setupTests.ts b/plugins/catalog-backend-module-puppetdb/src/setupTests.ts new file mode 100644 index 0000000000..4b9026cde5 --- /dev/null +++ b/plugins/catalog-backend-module-puppetdb/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/yarn.lock b/yarn.lock index fe23c357e4..86c08a9d61 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5243,6 +5243,29 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/types": "workspace:^" + "@types/lodash": ^4.14.151 + lodash: ^4.17.21 + luxon: ^3.0.0 + msw: ^0.49.0 + node-fetch: ^2.6.7 + uuid: ^8.0.0 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend" @@ -30115,6 +30138,40 @@ __metadata: languageName: node linkType: hard +"msw@npm:^0.49.0": + version: 0.49.3 + resolution: "msw@npm:0.49.3" + dependencies: + "@mswjs/cookies": ^0.2.2 + "@mswjs/interceptors": ^0.17.5 + "@open-draft/until": ^1.0.3 + "@types/cookie": ^0.4.1 + "@types/js-levenshtein": ^1.1.1 + chalk: 4.1.1 + chokidar: ^3.4.2 + cookie: ^0.4.2 + graphql: ^15.0.0 || ^16.0.0 + headers-polyfill: ^3.1.0 + inquirer: ^8.2.0 + is-node-process: ^1.0.1 + js-levenshtein: ^1.1.6 + node-fetch: ^2.6.7 + outvariant: ^1.3.0 + path-to-regexp: ^6.2.0 + strict-event-emitter: ^0.4.3 + type-fest: ^2.19.0 + yargs: ^17.3.1 + peerDependencies: + typescript: ">= 4.4.x <= 4.9.x" + peerDependenciesMeta: + typescript: + optional: true + bin: + msw: cli/index.js + checksum: 8322cd42cd69f289c05517d02bde22fc2f10e86fc2d0d209d9df54bd03d10b8123723c5587a2654dcd2cd0f314a016f9eccac88cffa30fafd1f9fead16db639e + languageName: node + linkType: hard + "msw@npm:^1.0.0, msw@npm:^1.0.1": version: 1.0.1 resolution: "msw@npm:1.0.1"