Merge pull request #16184 from tdabasinskas/catalog-backend-module-puppetdb
New catalog-backend-module-puppetdb plugin
This commit is contained in:
Fredrik Adelöw
@@ -0,0 +1,5 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-puppetdb': minor
|
||||
---
|
||||
|
||||
Initial version of the plugin.
|
||||
@@ -30,6 +30,7 @@ yarn.lock @backstage/maintainers @back
|
||||
/plugins/catalog-backend-module-aws @backstage/maintainers @backstage/catalog-core @pjungermann
|
||||
/plugins/catalog-backend-module-bitbucket-cloud @backstage/maintainers @backstage/catalog-core @pjungermann
|
||||
/plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-core @pjungermann
|
||||
/plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-core @tdabasinskas
|
||||
/plugins/catalog-graph @backstage/maintainers @backstage/catalog-core @backstage/sda-se-reviewers
|
||||
/plugins/circleci @backstage/maintainers @adamdmharvey
|
||||
/plugins/cloudbuild @backstage/maintainers @trivago/ebarrios
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
title: PuppetDB Entity Provider
|
||||
author: TDabasinskas
|
||||
authorUrl: https://github.com/tdabasinskas
|
||||
category: Configuration Management
|
||||
description: Import nodes from PuppetDB into Backstage as Resource Entities
|
||||
documentation: https://github.com/backstage/backstage/blob/master/plugins/catalog-backend-module-puppetdb/README.md
|
||||
iconUrl: https://avatars.githubusercontent.com/u/234268?s=200&v=4
|
||||
npmPackageName: '@backstage/plugin-catalog-backend-module-puppetdb'
|
||||
tags:
|
||||
- puppet
|
||||
- puppetdb
|
||||
addedDate: '2023-02-06'
|
||||
@@ -0,0 +1 @@
|
||||
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
|
||||
@@ -0,0 +1,90 @@
|
||||
# Catalog Backend Module for Puppet
|
||||
|
||||
This is an extension module to the `plugin-catalog-backend` plugin, providing an `PuppetDbEntityProvider` that can be used to ingest
|
||||
[Resource entities](https://backstage.io/docs/features/software-catalog/descriptor-format#kind-resource) from a
|
||||
[PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html) instance(s). This provider is useful if you want to import nodes
|
||||
from your PuppetDB into Backstage.
|
||||
|
||||
## Installation
|
||||
|
||||
The provider is not installed by default, therefore you have to add a dependency to `@backstage/plugin-catalog-backend-module-puppetdb`
|
||||
to your backend package:
|
||||
|
||||
```bash
|
||||
# From your Backstage root directory
|
||||
yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-puppetdb
|
||||
```
|
||||
|
||||
Update the catalog plugin initialization in your backend to add the provider and schedule it:
|
||||
|
||||
```diff
|
||||
+ import { PuppetDbEntityProvider } from '@backstage/plugin-catalog-backend-module-puppetdb';
|
||||
|
||||
export default async function createPlugin(
|
||||
env: PluginEnvironment,
|
||||
): Promise<Router> {
|
||||
const builder = await CatalogBuilder.create(env);
|
||||
|
||||
+ builder.addEntityProvider(
|
||||
+ PuppetDbEntityProvider.fromConfig(env.config, {
|
||||
+ logger: env.logger,
|
||||
+ schedule: env.scheduler.createScheduledTaskRunner({
|
||||
+ frequency: { minutes: 10 },
|
||||
+ timeout: { minutes: 50 },
|
||||
+ initialDelay: { seconds: 15}
|
||||
+ }),
|
||||
+ });
|
||||
+ );
|
||||
```
|
||||
|
||||
After this, you also have to add some configuration in your app-config that describes what you want to import for that target.
|
||||
|
||||
## Configuration
|
||||
|
||||
The following configuration is an example of how a setup could look for importing nodes from an internal PuppetDB instance:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
puppetdb:
|
||||
default:
|
||||
# (Required) The base URL of PuppetDB API instance:
|
||||
baseUrl: https://puppetdb.example.com
|
||||
|
||||
# (Optional) Query to filter PuppetDB nodes:
|
||||
#query: '["=","certname","example.com"]'
|
||||
```
|
||||
|
||||
## Customize the Provider
|
||||
|
||||
The default ingestion behaviour will likely not work for all use cases - you will want to set proper `Owner`, `System` and other fields for the
|
||||
ingested resources. In case you want to customize the ingested entities, the provider allows to pass a transformer for resources. Here we will show an example
|
||||
of overriding the default transformer.
|
||||
|
||||
1. Create a transformer:
|
||||
|
||||
```ts
|
||||
export const customResourceTransformer: ResourceTransformer = async (
|
||||
node,
|
||||
config,
|
||||
): Promise<GroupEntity | undefined> => {
|
||||
// Transformations may change namespace, owner, change entity naming pattern, add labels, annotations, etc.
|
||||
|
||||
// Create the Resource Entity on your own, or wrap the default transformer
|
||||
return await defaultResourceTransformer(node, config);
|
||||
};
|
||||
```
|
||||
|
||||
2. Configure the provider with the transformer:
|
||||
|
||||
```ts
|
||||
const puppetDbEntityProvider = PuppetDbEntityProvider.fromConfig(env.config, {
|
||||
logger: env.logger,
|
||||
schedule: env.scheduler.createScheduledTaskRunner({
|
||||
frequency: { minutes: 10 },
|
||||
timeout: { minutes: 50 },
|
||||
initialDelay: { seconds: 15 },
|
||||
}),
|
||||
transformer: customResourceTransformer,
|
||||
});
|
||||
```
|
||||
@@ -0,0 +1,79 @@
|
||||
## API Report File for "@backstage/plugin-catalog-backend-module-puppetdb"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { Config } from '@backstage/config';
|
||||
import { EntityProvider } from '@backstage/plugin-catalog-backend';
|
||||
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
|
||||
import { JsonValue } from '@backstage/types';
|
||||
import { Logger } from 'winston';
|
||||
import { PluginTaskScheduler } from '@backstage/backend-tasks';
|
||||
import { ResourceEntity } from '@backstage/catalog-model';
|
||||
import { TaskRunner } from '@backstage/backend-tasks';
|
||||
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
|
||||
|
||||
// @public
|
||||
export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname';
|
||||
|
||||
// @public
|
||||
export const DEFAULT_PROVIDER_ID = 'default';
|
||||
|
||||
// @public
|
||||
export const defaultResourceTransformer: ResourceTransformer;
|
||||
|
||||
// @public
|
||||
export class PuppetDbEntityProvider implements EntityProvider {
|
||||
// (undocumented)
|
||||
connect(connection: EntityProviderConnection): Promise<void>;
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
deps: {
|
||||
logger: Logger;
|
||||
schedule?: TaskRunner;
|
||||
scheduler?: PluginTaskScheduler;
|
||||
transformer?: ResourceTransformer;
|
||||
},
|
||||
): PuppetDbEntityProvider[];
|
||||
// (undocumented)
|
||||
getProviderName(): string;
|
||||
refresh(logger: Logger): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type PuppetDbEntityProviderConfig = {
|
||||
id: string;
|
||||
baseUrl: string;
|
||||
query?: string;
|
||||
schedule?: TaskScheduleDefinition;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type PuppetFact = {
|
||||
name: string;
|
||||
value: JsonValue;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type PuppetFactSet = {
|
||||
data: PuppetFact[];
|
||||
href: string;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type PuppetNode = {
|
||||
timestamp: string;
|
||||
certname: string;
|
||||
hash: string;
|
||||
producer_timestamp: string;
|
||||
producer: string;
|
||||
environment: string;
|
||||
facts: PuppetFactSet;
|
||||
};
|
||||
|
||||
// @public
|
||||
export type ResourceTransformer = (
|
||||
node: PuppetNode,
|
||||
config: PuppetDbEntityProviderConfig,
|
||||
) => Promise<ResourceEntity | undefined>;
|
||||
```
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
|
||||
|
||||
/**
|
||||
* Represents the configuration for the Backstage.
|
||||
*/
|
||||
export interface Config {
|
||||
/**
|
||||
* Configuration for the catalog.
|
||||
*/
|
||||
catalog?: {
|
||||
/**
|
||||
* Configuration for the providers.
|
||||
*/
|
||||
providers?: {
|
||||
/**
|
||||
* PuppetDB Entity Provider configuration. Uses "default" as default ID for the single config variant.
|
||||
*/
|
||||
puppetdb?:
|
||||
| {
|
||||
/**
|
||||
* (Required) The base URL of PuppetDB API instance.
|
||||
*/
|
||||
baseUrl: string;
|
||||
/**
|
||||
* (Optional) PQL query to filter PuppetDB nodes.
|
||||
*/
|
||||
query?: string;
|
||||
/**
|
||||
* (Optional) Task schedule definition for the refresh.
|
||||
*/
|
||||
schedule?: TaskScheduleDefinition;
|
||||
}
|
||||
| Record<
|
||||
string,
|
||||
{
|
||||
/**
|
||||
* (Required) The base URL of PuppetDB API instance.
|
||||
*/
|
||||
baseUrl: string;
|
||||
/**
|
||||
* (Optional) PQL query to filter PuppetDB nodes.
|
||||
*/
|
||||
query?: string;
|
||||
/**
|
||||
* (Optional) Task schedule definition for the refresh.
|
||||
*/
|
||||
schedule?: TaskScheduleDefinition;
|
||||
}
|
||||
>;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"name": "@backstage/plugin-catalog-backend-module-puppetdb",
|
||||
"description": "A Backstage catalog backend module that helps integrate towards PuppetDB",
|
||||
"version": "0.0.1",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"license": "Apache-2.0",
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"main": "dist/index.cjs.js",
|
||||
"types": "dist/index.d.ts"
|
||||
},
|
||||
"backstage": {
|
||||
"role": "backend-plugin-module"
|
||||
},
|
||||
"homepage": "https://backstage.io",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/backstage/backstage",
|
||||
"directory": "plugins/catalog-backend-module-puppetdb"
|
||||
},
|
||||
"keywords": [
|
||||
"backstage",
|
||||
"puppetdb",
|
||||
"puppet"
|
||||
],
|
||||
"scripts": {
|
||||
"start": "backstage-cli package start",
|
||||
"build": "backstage-cli package build",
|
||||
"lint": "backstage-cli package lint",
|
||||
"test": "backstage-cli package test",
|
||||
"clean": "backstage-cli package clean",
|
||||
"prepack": "backstage-cli package prepack",
|
||||
"postpack": "backstage-cli package postpack"
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/backend-common": "workspace:^",
|
||||
"@backstage/backend-tasks": "workspace:^",
|
||||
"@backstage/catalog-model": "workspace:^",
|
||||
"@backstage/config": "workspace:^",
|
||||
"@backstage/errors": "workspace:^",
|
||||
"@backstage/plugin-catalog-backend": "workspace:^",
|
||||
"@backstage/types": "workspace:^",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.0.0",
|
||||
"node-fetch": "^2.6.7",
|
||||
"uuid": "^8.0.0",
|
||||
"winston": "^3.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/backend-test-utils": "workspace:^",
|
||||
"@backstage/cli": "workspace:^",
|
||||
"@types/lodash": "^4.14.151",
|
||||
"msw": "^0.49.0"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"config.d.ts"
|
||||
],
|
||||
"configSchema": "config.d.ts"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A Backstage catalog backend module that helps integrate towards PuppetDB
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
export * from './providers';
|
||||
export * from './puppet';
|
||||
@@ -0,0 +1,219 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { PuppetDbEntityProvider } from './PuppetDbEntityProvider';
|
||||
import {
|
||||
DeferredEntity,
|
||||
EntityProviderConnection,
|
||||
} from '@backstage/plugin-catalog-backend';
|
||||
import * as puppetFunctions from '../puppet/read';
|
||||
import { ANNOTATION_PUPPET_CERTNAME } from '../puppet';
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
ResourceEntity,
|
||||
} from '@backstage/catalog-model/';
|
||||
import { DEFAULT_ENTITY_OWNER, ENDPOINT_NODES } from '../puppet/constants';
|
||||
|
||||
jest.mock('../puppet/read', () => {
|
||||
return {
|
||||
readPuppetNodes: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
class PersistingTaskRunner implements TaskRunner {
|
||||
private tasks: TaskInvocationDefinition[] = [];
|
||||
|
||||
getTasks() {
|
||||
return this.tasks;
|
||||
}
|
||||
|
||||
run(task: TaskInvocationDefinition): Promise<void> {
|
||||
this.tasks.push(task);
|
||||
return Promise.resolve(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
describe('PuppetEntityProvider', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
puppetdb: {
|
||||
baseUrl: 'http://puppetdb:8080',
|
||||
schedule: {
|
||||
frequency: {
|
||||
minutes: 10,
|
||||
},
|
||||
timeout: {
|
||||
minutes: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
describe('where there are no nodes', () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([]);
|
||||
});
|
||||
|
||||
it('creates no entities', async () => {
|
||||
const connection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
const providers = PuppetDbEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule: new PersistingTaskRunner(),
|
||||
});
|
||||
|
||||
await providers[0].connect(connection);
|
||||
await providers[0].refresh(logger);
|
||||
|
||||
expect(connection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'full',
|
||||
entities: [],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('where there are nodes', () => {
|
||||
beforeEach(() => {
|
||||
jest.spyOn(puppetFunctions, 'readPuppetNodes').mockResolvedValueOnce([
|
||||
{
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'node1',
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node1',
|
||||
},
|
||||
tags: ['windows'],
|
||||
description: 'Description 1',
|
||||
},
|
||||
spec: {
|
||||
type: 'virtual-machine',
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'node2',
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node2',
|
||||
},
|
||||
tags: ['linux'],
|
||||
description: 'Description 2',
|
||||
},
|
||||
spec: {
|
||||
type: 'physical-server',
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
},
|
||||
] as ResourceEntity[]);
|
||||
});
|
||||
|
||||
it('creates entities', async () => {
|
||||
const connection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
refresh: jest.fn(),
|
||||
};
|
||||
const providers = PuppetDbEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule: new PersistingTaskRunner(),
|
||||
});
|
||||
|
||||
await providers[0].connect(connection);
|
||||
await providers[0].refresh(logger);
|
||||
|
||||
expect(connection.applyMutation).toHaveBeenCalledWith({
|
||||
type: 'full',
|
||||
entities: [
|
||||
{
|
||||
locationKey: providers[0].getProviderName(),
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'node1',
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node1',
|
||||
[ANNOTATION_LOCATION]: `url:${config.getString(
|
||||
'catalog.providers.puppetdb.baseUrl',
|
||||
)}/${ENDPOINT_NODES}/node1`,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString(
|
||||
'catalog.providers.puppetdb.baseUrl',
|
||||
)}/${ENDPOINT_NODES}/node1`,
|
||||
},
|
||||
tags: ['windows'],
|
||||
description: 'Description 1',
|
||||
},
|
||||
spec: {
|
||||
type: 'virtual-machine',
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
locationKey: providers[0].getProviderName(),
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'node2',
|
||||
namespace: 'default',
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node2',
|
||||
[ANNOTATION_LOCATION]: `url:${config.getString(
|
||||
'catalog.providers.puppetdb.baseUrl',
|
||||
)}/${ENDPOINT_NODES}/node2`,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: `url:${config.getString(
|
||||
'catalog.providers.puppetdb.baseUrl',
|
||||
)}/${ENDPOINT_NODES}/node2`,
|
||||
},
|
||||
tags: ['linux'],
|
||||
description: 'Description 2',
|
||||
},
|
||||
spec: {
|
||||
type: 'physical-server',
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
] as DeferredEntity[],
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,235 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
} from '@backstage/plugin-catalog-backend';
|
||||
import { Logger } from 'winston';
|
||||
import {
|
||||
PuppetDbEntityProviderConfig,
|
||||
readProviderConfigs,
|
||||
} from './PuppetDbEntityProviderConfig';
|
||||
import { Config } from '@backstage/config';
|
||||
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
|
||||
import * as uuid from 'uuid';
|
||||
import { ResourceTransformer, defaultResourceTransformer } from '../puppet';
|
||||
import {
|
||||
ANNOTATION_LOCATION,
|
||||
ANNOTATION_ORIGIN_LOCATION,
|
||||
Entity,
|
||||
} from '@backstage/catalog-model/';
|
||||
import { merge } from 'lodash';
|
||||
import { readPuppetNodes } from '../puppet/read';
|
||||
import { ENDPOINT_NODES } from '../puppet/constants';
|
||||
|
||||
/**
|
||||
* Reads nodes from [PuppetDB](https://www.puppet.com/docs/puppet/6/puppetdb_overview.html)
|
||||
* based on the provided query and registers them as Resource entities in the catalog.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class PuppetDbEntityProvider implements EntityProvider {
|
||||
private readonly config: PuppetDbEntityProviderConfig;
|
||||
private readonly logger: Logger;
|
||||
private readonly scheduleFn: () => Promise<void>;
|
||||
private readonly transformer: ResourceTransformer;
|
||||
private connection?: EntityProviderConnection;
|
||||
|
||||
/**
|
||||
* Creates instances of {@link PuppetDbEntityProvider} from a configuration.
|
||||
*
|
||||
* @param config - The configuration to read provider information from.
|
||||
* @param deps - The dependencies for {@link PuppetDbEntityProvider}.
|
||||
*
|
||||
* @returns A list of {@link PuppetDbEntityProvider} instances.
|
||||
*/
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
deps: {
|
||||
logger: Logger;
|
||||
schedule?: TaskRunner;
|
||||
scheduler?: PluginTaskScheduler;
|
||||
transformer?: ResourceTransformer;
|
||||
},
|
||||
): PuppetDbEntityProvider[] {
|
||||
if (!deps.schedule && !deps.scheduler) {
|
||||
throw new Error('Either schedule or scheduler must be provided.');
|
||||
}
|
||||
|
||||
return readProviderConfigs(config).map(providerConfig => {
|
||||
if (!deps.schedule && !providerConfig.schedule) {
|
||||
throw new Error(
|
||||
`No schedule provided neither via code nor config for puppet-provider:${providerConfig.id}.`,
|
||||
);
|
||||
}
|
||||
|
||||
const taskRunner =
|
||||
deps.schedule ??
|
||||
deps.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
|
||||
|
||||
const transformer = deps.transformer ?? defaultResourceTransformer;
|
||||
|
||||
return new PuppetDbEntityProvider(
|
||||
providerConfig,
|
||||
deps.logger,
|
||||
taskRunner,
|
||||
transformer,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance of {@link PuppetDbEntityProvider}.
|
||||
*
|
||||
* @param config - Configuration of the provider.
|
||||
* @param logger - The instance of a {@link Logger}.
|
||||
* @param taskRunner - The instance of {@link TaskRunner}.
|
||||
* @param transformer - A {@link ResourceTransformer} function.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
private constructor(
|
||||
config: PuppetDbEntityProviderConfig,
|
||||
logger: Logger,
|
||||
taskRunner: TaskRunner,
|
||||
transformer: ResourceTransformer,
|
||||
) {
|
||||
this.config = config;
|
||||
this.logger = logger.child({
|
||||
target: this.getProviderName(),
|
||||
});
|
||||
this.scheduleFn = this.createScheduleFn(taskRunner);
|
||||
this.transformer = transformer;
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
|
||||
async connect(connection: EntityProviderConnection): Promise<void> {
|
||||
this.connection = connection;
|
||||
await this.scheduleFn();
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
|
||||
getProviderName(): string {
|
||||
return `puppetdb-provider:${this.config.id}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a function that can be used to schedule a refresh of the catalog.
|
||||
*
|
||||
* @param taskRunner - The instance of {@link TaskRunner}.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
private createScheduleFn(taskRunner: TaskRunner): () => Promise<void> {
|
||||
return async () => {
|
||||
const taskId = `${this.getProviderName()}:refresh`;
|
||||
return taskRunner.run({
|
||||
id: taskId,
|
||||
fn: async () => {
|
||||
const logger = this.logger.child({
|
||||
class: PuppetDbEntityProvider.prototype.constructor.name,
|
||||
taskId,
|
||||
taskInstanceId: uuid.v4(),
|
||||
});
|
||||
try {
|
||||
await this.refresh(logger);
|
||||
} catch (error) {
|
||||
logger.error(`${this.getProviderName()} refresh failed`, error);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Refreshes the catalog by reading nodes from PuppetDB and registering them as Resource Entities.
|
||||
*
|
||||
* @param logger - The instance of a Logger.
|
||||
*/
|
||||
async refresh(logger: Logger) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
const { markReadComplete } = trackProgress(logger);
|
||||
const entities = await readPuppetNodes(this.config, {
|
||||
logger,
|
||||
transformer: this.transformer,
|
||||
});
|
||||
const { markCommitComplete } = markReadComplete(entities);
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
entities: [...entities].map(entity => ({
|
||||
locationKey: this.getProviderName(),
|
||||
entity: withLocations(this.config.baseUrl, entity),
|
||||
})),
|
||||
});
|
||||
markCommitComplete(entities);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the entities have required annotation data.
|
||||
*
|
||||
* @param baseUrl - The base URL of the PuppetDB instance.
|
||||
* @param entity - The entity to add the annotations to.
|
||||
*
|
||||
* @returns Entity with @{@link ANNOTATION_LOCATION} and @{@link ANNOTATION_ORIGIN_LOCATION} annotations.
|
||||
*/
|
||||
function withLocations(baseUrl: string, entity: Entity): Entity {
|
||||
const location = `${baseUrl}/${ENDPOINT_NODES}/${entity.metadata?.name}`;
|
||||
|
||||
return merge(
|
||||
{
|
||||
metadata: {
|
||||
annotations: {
|
||||
[ANNOTATION_LOCATION]: `url:${location}`,
|
||||
[ANNOTATION_ORIGIN_LOCATION]: `url:${location}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
entity,
|
||||
) as Entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks the progress of the PuppetDB read and commit operations.
|
||||
*
|
||||
* @param logger - The instance of a {@link Logger}.
|
||||
*/
|
||||
function trackProgress(logger: Logger) {
|
||||
let timestamp = Date.now();
|
||||
|
||||
function markReadComplete(entities: Entity[]) {
|
||||
const readDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
|
||||
timestamp = Date.now();
|
||||
logger.info(
|
||||
`Read ${entities?.length ?? 0} in ${readDuration} seconds. Committing...`,
|
||||
);
|
||||
return { markCommitComplete };
|
||||
}
|
||||
|
||||
function markCommitComplete(entities: Entity[]) {
|
||||
const commitDuration = ((Date.now() - timestamp) / 1000).toFixed(1);
|
||||
logger.info(
|
||||
`Committed ${entities?.length ?? 0} in ${commitDuration} seconds.`,
|
||||
);
|
||||
}
|
||||
|
||||
return { markReadComplete };
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { readProviderConfigs } from './PuppetDbEntityProviderConfig';
|
||||
import { Duration } from 'luxon';
|
||||
|
||||
describe('readProviderConfigs', () => {
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('no provider config', () => {
|
||||
const config = new ConfigReader({});
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('single simple provider config', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
puppetdb: {
|
||||
baseUrl: 'https://puppetdb',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(1);
|
||||
expect(providerConfigs[0].id).toEqual('default');
|
||||
expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb');
|
||||
});
|
||||
|
||||
it('single specific provider config', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
puppetdb: {
|
||||
'my-provider': {
|
||||
baseUrl: 'https://puppetdb',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(1);
|
||||
expect(providerConfigs[0].id).toEqual('my-provider');
|
||||
expect(providerConfigs[0].baseUrl).toEqual('https://puppetdb');
|
||||
});
|
||||
|
||||
it('multiple provider configs', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
puppetdb: {
|
||||
'my-provider': {
|
||||
baseUrl: 'https://my-puppet/',
|
||||
query: 'my-query',
|
||||
},
|
||||
'your-provider': {
|
||||
baseUrl: 'https://your-puppet',
|
||||
query: 'your-query',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(2);
|
||||
expect(providerConfigs[0]).toEqual({
|
||||
id: 'my-provider',
|
||||
baseUrl: 'https://my-puppet',
|
||||
query: 'my-query',
|
||||
});
|
||||
expect(providerConfigs[1]).toEqual({
|
||||
id: 'your-provider',
|
||||
baseUrl: 'https://your-puppet',
|
||||
query: 'your-query',
|
||||
});
|
||||
});
|
||||
|
||||
it('provider config with schedule', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
puppetdb: {
|
||||
baseUrl: 'https://puppetdb',
|
||||
schedule: {
|
||||
frequency: 'PT30M',
|
||||
timeout: {
|
||||
minutes: 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(1);
|
||||
expect(providerConfigs[0].schedule).toEqual({
|
||||
frequency: Duration.fromISO('PT30M'),
|
||||
timeout: {
|
||||
minutes: 10,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
readTaskScheduleDefinitionFromConfig,
|
||||
TaskScheduleDefinition,
|
||||
} from '@backstage/backend-tasks';
|
||||
import { Config } from '@backstage/config';
|
||||
import { DEFAULT_PROVIDER_ID } from './constants';
|
||||
|
||||
/**
|
||||
* Configuration of {@link PuppetDbEntityProvider}.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type PuppetDbEntityProviderConfig = {
|
||||
/**
|
||||
* ID of the provider.
|
||||
*/
|
||||
id: string;
|
||||
/**
|
||||
* (Required) The base URL of PuppetDB API instance.
|
||||
*/
|
||||
baseUrl: string;
|
||||
/**
|
||||
* (Optional) PQL query to filter PuppetDB nodes.
|
||||
*/
|
||||
query?: string;
|
||||
/**
|
||||
* (Optional) Task schedule definition for the refresh.
|
||||
*/
|
||||
schedule?: TaskScheduleDefinition;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads the configuration of the PuppetDB Entity Providers.
|
||||
*
|
||||
* @param config - The application configuration.
|
||||
*
|
||||
* @returns PuppetDB Entity Provider configurations list.
|
||||
*/
|
||||
export function readProviderConfigs(
|
||||
config: Config,
|
||||
): PuppetDbEntityProviderConfig[] {
|
||||
const providersConfig = config.getOptionalConfig(
|
||||
'catalog.providers.puppetdb',
|
||||
);
|
||||
if (!providersConfig) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (providersConfig.has('baseUrl')) {
|
||||
return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
|
||||
}
|
||||
|
||||
return providersConfig.keys().map(id => {
|
||||
return readProviderConfig(id, providersConfig.getConfig(id));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the configuration for the PuppetDB Entity Provider.
|
||||
*
|
||||
* @param id - ID of the provider.
|
||||
* @param config - The application configuration.
|
||||
*
|
||||
* @returns The PuppetDB Entity Provider configuration.
|
||||
*/
|
||||
function readProviderConfig(
|
||||
id: string,
|
||||
config: Config,
|
||||
): PuppetDbEntityProviderConfig {
|
||||
const baseUrl = config.getString('baseUrl').replace(/\/+$/, '');
|
||||
const query = config.getOptionalString('query');
|
||||
|
||||
const schedule = config.has('schedule')
|
||||
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
id,
|
||||
baseUrl,
|
||||
query,
|
||||
schedule,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Name of the default provider when a using a simple configuration.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const DEFAULT_PROVIDER_ID = 'default';
|
||||
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { PuppetDbEntityProvider } from './PuppetDbEntityProvider';
|
||||
export type { PuppetDbEntityProviderConfig } from './PuppetDbEntityProviderConfig';
|
||||
export { DEFAULT_PROVIDER_ID } from './constants';
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Annotation for specifying the certificate name of a node in PuppetDB.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const ANNOTATION_PUPPET_CERTNAME = 'puppet.com/certname';
|
||||
|
||||
/**
|
||||
* Path of PuppetDB FactSets endpoint.
|
||||
*/
|
||||
export const ENDPOINT_FACTSETS = 'pdb/query/v4/factsets';
|
||||
|
||||
/**
|
||||
* Path of PuppetDB Nodes endpoint.
|
||||
*/
|
||||
export const ENDPOINT_NODES = 'pdb/query/v4/nodes';
|
||||
|
||||
/**
|
||||
* Default owner for entities created by the PuppetDB provider.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const DEFAULT_ENTITY_OWNER = 'unknown';
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { ANNOTATION_PUPPET_CERTNAME } from './constants';
|
||||
export { defaultResourceTransformer } from './transformers';
|
||||
export type {
|
||||
PuppetNode,
|
||||
PuppetFactSet,
|
||||
PuppetFact,
|
||||
ResourceTransformer,
|
||||
} from './types';
|
||||
@@ -0,0 +1,247 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { readPuppetNodes } from './read';
|
||||
import {
|
||||
DEFAULT_PROVIDER_ID,
|
||||
PuppetDbEntityProviderConfig,
|
||||
} from '../providers';
|
||||
import { DEFAULT_NAMESPACE } from '@backstage/catalog-model';
|
||||
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { ANNOTATION_PUPPET_CERTNAME, ENDPOINT_FACTSETS } from './constants';
|
||||
|
||||
describe('readPuppetNodes', () => {
|
||||
const worker = setupServer();
|
||||
setupRequestMockHandlers(worker);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('where no query is specified', () => {
|
||||
const config: PuppetDbEntityProviderConfig = {
|
||||
baseUrl: 'https://puppetdb',
|
||||
id: DEFAULT_PROVIDER_ID,
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
worker.use(
|
||||
rest.get(`${config.baseUrl}/${ENDPOINT_FACTSETS}`, (_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json([
|
||||
{
|
||||
certname: 'node1',
|
||||
timestamp: 'time1',
|
||||
hash: 'hash1',
|
||||
producer_timestamp: 'producer_time1',
|
||||
producer: 'producer1',
|
||||
environment: 'environment1',
|
||||
facts: {
|
||||
data: [
|
||||
{
|
||||
name: 'is_virtual',
|
||||
value: true,
|
||||
},
|
||||
{
|
||||
name: 'kernel',
|
||||
value: 'Linux',
|
||||
},
|
||||
{
|
||||
name: 'ipaddress',
|
||||
value: 'ipaddress1',
|
||||
},
|
||||
{
|
||||
name: 'clientnoop',
|
||||
value: true,
|
||||
},
|
||||
{
|
||||
name: 'clientversion',
|
||||
value: 'clientversion1',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
certname: 'node2',
|
||||
timestamp: 'time2',
|
||||
hash: 'hash2',
|
||||
producer_timestamp: 'producer_time2',
|
||||
producer: 'producer2',
|
||||
environment: 'environment2',
|
||||
facts: {
|
||||
data: [
|
||||
{
|
||||
name: 'is_virtual',
|
||||
value: false,
|
||||
},
|
||||
{
|
||||
name: 'kernel',
|
||||
value: 'Windows',
|
||||
},
|
||||
{
|
||||
name: 'ipaddress',
|
||||
value: 'ipaddress2',
|
||||
},
|
||||
{
|
||||
name: 'clientnoop',
|
||||
value: false,
|
||||
},
|
||||
{
|
||||
name: 'clientversion',
|
||||
value: 'clientversion2',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
describe('where custom transformer is used', () => {
|
||||
it('should use it for transforming puppet nodes', async () => {
|
||||
const entities = await readPuppetNodes(config, {
|
||||
transformer: async (node, _config) => {
|
||||
return {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: `custom-${node.certname}`,
|
||||
namespace: DEFAULT_NAMESPACE,
|
||||
},
|
||||
spec: {
|
||||
type: 'Custom',
|
||||
owner: 'Custom',
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
expect(entities).toHaveLength(2);
|
||||
expect(entities[0]).toEqual({
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'custom-node1',
|
||||
namespace: DEFAULT_NAMESPACE,
|
||||
},
|
||||
spec: {
|
||||
type: 'Custom',
|
||||
owner: 'Custom',
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
});
|
||||
expect(entities[1]).toEqual({
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'custom-node2',
|
||||
namespace: DEFAULT_NAMESPACE,
|
||||
},
|
||||
spec: {
|
||||
type: 'Custom',
|
||||
owner: 'Custom',
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('where default transformer is used', () => {
|
||||
it('should use it for transforming puppet nodes', async () => {
|
||||
const entities = await readPuppetNodes(config);
|
||||
|
||||
expect(entities).toHaveLength(2);
|
||||
expect(entities[0].metadata.annotations).toEqual({
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node1',
|
||||
});
|
||||
expect(entities[1].metadata.annotations).toEqual({
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node2',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('where query is specified', () => {
|
||||
const config: PuppetDbEntityProviderConfig = {
|
||||
baseUrl: 'https://puppetdb',
|
||||
id: DEFAULT_PROVIDER_ID,
|
||||
query: '["=", "certname", "node1"]',
|
||||
};
|
||||
|
||||
describe('where no results are matched', () => {
|
||||
beforeEach(async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
`${config.baseUrl}/${ENDPOINT_FACTSETS}`,
|
||||
(_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json([]),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('should return empty array', async () => {
|
||||
const entities = await readPuppetNodes(config);
|
||||
expect(entities).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('where results are matched', () => {
|
||||
beforeEach(async () => {
|
||||
worker.use(
|
||||
rest.get(
|
||||
`${config.baseUrl}/${ENDPOINT_FACTSETS}`,
|
||||
(_req, res, ctx) => {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.set('Content-Type', 'application/json'),
|
||||
ctx.json([
|
||||
{
|
||||
certname: 'node1',
|
||||
timestamp: 'time1',
|
||||
hash: 'hash1',
|
||||
producer_timestamp: 'producer_time1',
|
||||
producer: 'producer1',
|
||||
environment: 'environment1',
|
||||
},
|
||||
]),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('should return matched results', async () => {
|
||||
const entities = await readPuppetNodes(config);
|
||||
expect(entities).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { PuppetDbEntityProviderConfig } from '../providers';
|
||||
import { PuppetNode, ResourceTransformer } from './types';
|
||||
import { ResourceEntity } from '@backstage/catalog-model/';
|
||||
import { defaultResourceTransformer } from './transformers';
|
||||
import fetch from 'node-fetch';
|
||||
import { ResponseError } from '@backstage/errors';
|
||||
import { ENDPOINT_FACTSETS } from './constants';
|
||||
import { Logger } from 'winston';
|
||||
|
||||
/**
|
||||
* Reads nodes and their facts from PuppetDB.
|
||||
*
|
||||
* @param config - The provider configuration.
|
||||
* @param opts - Additional options.
|
||||
*/
|
||||
export async function readPuppetNodes(
|
||||
config: PuppetDbEntityProviderConfig,
|
||||
opts?: {
|
||||
transformer?: ResourceTransformer;
|
||||
logger?: Logger;
|
||||
},
|
||||
): Promise<ResourceEntity[]> {
|
||||
const transformFn = opts?.transformer ?? defaultResourceTransformer;
|
||||
const url = new URL(ENDPOINT_FACTSETS, config.baseUrl);
|
||||
|
||||
if (config.query) {
|
||||
url.searchParams.set('query', config.query);
|
||||
}
|
||||
|
||||
if (opts?.logger) {
|
||||
opts.logger.debug('Reading nodes from PuppetDB', { url: url.toString() });
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw await ResponseError.fromResponse(response);
|
||||
}
|
||||
|
||||
const nodes = (await response.json()) as PuppetNode[];
|
||||
const entities: ResourceEntity[] = [];
|
||||
|
||||
for (const node of nodes) {
|
||||
const entity = await transformFn(node, config);
|
||||
if (entity) {
|
||||
entities.push(entity);
|
||||
}
|
||||
}
|
||||
|
||||
return entities;
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { PuppetDbEntityProviderConfig } from '../providers';
|
||||
import { PuppetNode } from './types';
|
||||
import { defaultResourceTransformer } from './transformers';
|
||||
import { DEFAULT_NAMESPACE } from '@backstage/catalog-model';
|
||||
import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants';
|
||||
|
||||
describe('defaultResourceTransformer', () => {
|
||||
it('should transform a puppet node to a resource entity', async () => {
|
||||
const config: PuppetDbEntityProviderConfig = {
|
||||
baseUrl: '',
|
||||
id: '',
|
||||
};
|
||||
const node: PuppetNode = {
|
||||
certname: 'node1',
|
||||
timestamp: 'time1',
|
||||
hash: 'hash1',
|
||||
producer_timestamp: 'producer_time1',
|
||||
producer: 'producer1',
|
||||
environment: 'environment1',
|
||||
facts: {
|
||||
href: 'facts1',
|
||||
data: [
|
||||
{
|
||||
name: 'kernel',
|
||||
value: 'Linux',
|
||||
},
|
||||
{
|
||||
name: 'ipaddress',
|
||||
value: 'ipaddress1',
|
||||
},
|
||||
{
|
||||
name: 'is_virtual',
|
||||
value: true,
|
||||
},
|
||||
{
|
||||
name: 'clientnoop',
|
||||
value: true,
|
||||
},
|
||||
{
|
||||
name: 'clientversion',
|
||||
value: 'clientversion1',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const entity = await defaultResourceTransformer(node, config);
|
||||
expect(entity).toEqual({
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: 'node1',
|
||||
namespace: DEFAULT_NAMESPACE,
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: 'node1',
|
||||
},
|
||||
description: 'ipaddress1',
|
||||
tags: ['linux'],
|
||||
},
|
||||
spec: {
|
||||
type: 'virtual-machine',
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ResourceTransformer } from './types';
|
||||
import { DEFAULT_NAMESPACE, ResourceEntity } from '@backstage/catalog-model';
|
||||
import { ANNOTATION_PUPPET_CERTNAME, DEFAULT_ENTITY_OWNER } from './constants';
|
||||
|
||||
/**
|
||||
* A default implementation of the {@link ResourceTransformer}.
|
||||
*
|
||||
* @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform.
|
||||
* @param _config - The configuration for the entity provider.
|
||||
*
|
||||
* @returns A `ResourceEntity`.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const defaultResourceTransformer: ResourceTransformer = async (
|
||||
node,
|
||||
_config,
|
||||
): Promise<ResourceEntity | undefined> => {
|
||||
const certName = node.certname.toLocaleLowerCase('en-US');
|
||||
const type = node.facts?.data?.find(e => e.name === 'is_virtual')?.value
|
||||
? 'virtual-machine'
|
||||
: 'physical-server';
|
||||
const kernel = node.facts?.data?.find(e => e.name === 'kernel')?.value;
|
||||
|
||||
return {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
kind: 'Resource',
|
||||
metadata: {
|
||||
name: certName,
|
||||
annotations: {
|
||||
[ANNOTATION_PUPPET_CERTNAME]: certName,
|
||||
},
|
||||
namespace: DEFAULT_NAMESPACE,
|
||||
description: node.facts?.data
|
||||
?.find(e => e.name === 'ipaddress')
|
||||
?.value?.toString(),
|
||||
tags: kernel ? [kernel.toString().toLocaleLowerCase('en-US')] : [],
|
||||
},
|
||||
spec: {
|
||||
type: type,
|
||||
owner: DEFAULT_ENTITY_OWNER,
|
||||
dependsOn: [],
|
||||
dependencyOf: [],
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ResourceEntity } from '@backstage/catalog-model';
|
||||
import { JsonValue } from '@backstage/types';
|
||||
import { PuppetDbEntityProviderConfig } from '../providers/PuppetDbEntityProviderConfig';
|
||||
|
||||
/**
|
||||
* Customize the ingested Resource entity.
|
||||
*
|
||||
* @param node - The found PuppetDB node entry in its source format. This is the entry that you want to transform.
|
||||
* @param config - The configuration for the entity provider.
|
||||
*
|
||||
* @returns A `ResourceEntity` or `undefined` if you want to ignore the found group for being ingested by the catalog.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type ResourceTransformer = (
|
||||
node: PuppetNode,
|
||||
config: PuppetDbEntityProviderConfig,
|
||||
) => Promise<ResourceEntity | undefined>;
|
||||
|
||||
/**
|
||||
* A node in PuppetDB.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type PuppetNode = {
|
||||
/**
|
||||
* The most recent time of fact submission from the associated certname.
|
||||
*/
|
||||
timestamp: string;
|
||||
/**
|
||||
* The certname associated with the factset.
|
||||
*/
|
||||
certname: string;
|
||||
/**
|
||||
* A hash of the factset's certname, environment, timestamp, facts, and producer_timestamp.
|
||||
*/
|
||||
hash: string;
|
||||
/**
|
||||
* The most recent time of fact submission for the relevant certname from the Puppet Server.
|
||||
*/
|
||||
producer_timestamp: string;
|
||||
/**
|
||||
* The certname of the Puppet Server that sent the factset to PuppetDB.
|
||||
*/
|
||||
producer: string;
|
||||
/**
|
||||
* The environment associated with the fact.
|
||||
*/
|
||||
environment: string;
|
||||
/**
|
||||
* The facts associated with the factset.
|
||||
*/
|
||||
facts: PuppetFactSet;
|
||||
};
|
||||
|
||||
/**
|
||||
* The set of all facts for a single certname in PuppetDB.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type PuppetFactSet = {
|
||||
/**
|
||||
* The array of facts.
|
||||
*/
|
||||
data: PuppetFact[];
|
||||
/**
|
||||
* The URL to retrieve more information about the facts.
|
||||
*/
|
||||
href: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A fact in PuppetDB.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type PuppetFact = {
|
||||
/**
|
||||
* The name of the fact.
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* The value of the fact.
|
||||
*/
|
||||
value: JsonValue;
|
||||
};
|
||||
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export {};
|
||||
@@ -5243,6 +5243,29 @@ __metadata:
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
"@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend-module-puppetdb@workspace:plugins/catalog-backend-module-puppetdb"
|
||||
dependencies:
|
||||
"@backstage/backend-common": "workspace:^"
|
||||
"@backstage/backend-tasks": "workspace:^"
|
||||
"@backstage/backend-test-utils": "workspace:^"
|
||||
"@backstage/catalog-model": "workspace:^"
|
||||
"@backstage/cli": "workspace:^"
|
||||
"@backstage/config": "workspace:^"
|
||||
"@backstage/errors": "workspace:^"
|
||||
"@backstage/plugin-catalog-backend": "workspace:^"
|
||||
"@backstage/types": "workspace:^"
|
||||
"@types/lodash": ^4.14.151
|
||||
lodash: ^4.17.21
|
||||
luxon: ^3.0.0
|
||||
msw: ^0.49.0
|
||||
node-fetch: ^2.6.7
|
||||
uuid: ^8.0.0
|
||||
winston: ^3.2.1
|
||||
languageName: unknown
|
||||
linkType: soft
|
||||
|
||||
"@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend":
|
||||
version: 0.0.0-use.local
|
||||
resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend"
|
||||
@@ -30115,6 +30138,40 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"msw@npm:^0.49.0":
|
||||
version: 0.49.3
|
||||
resolution: "msw@npm:0.49.3"
|
||||
dependencies:
|
||||
"@mswjs/cookies": ^0.2.2
|
||||
"@mswjs/interceptors": ^0.17.5
|
||||
"@open-draft/until": ^1.0.3
|
||||
"@types/cookie": ^0.4.1
|
||||
"@types/js-levenshtein": ^1.1.1
|
||||
chalk: 4.1.1
|
||||
chokidar: ^3.4.2
|
||||
cookie: ^0.4.2
|
||||
graphql: ^15.0.0 || ^16.0.0
|
||||
headers-polyfill: ^3.1.0
|
||||
inquirer: ^8.2.0
|
||||
is-node-process: ^1.0.1
|
||||
js-levenshtein: ^1.1.6
|
||||
node-fetch: ^2.6.7
|
||||
outvariant: ^1.3.0
|
||||
path-to-regexp: ^6.2.0
|
||||
strict-event-emitter: ^0.4.3
|
||||
type-fest: ^2.19.0
|
||||
yargs: ^17.3.1
|
||||
peerDependencies:
|
||||
typescript: ">= 4.4.x <= 4.9.x"
|
||||
peerDependenciesMeta:
|
||||
typescript:
|
||||
optional: true
|
||||
bin:
|
||||
msw: cli/index.js
|
||||
checksum: 8322cd42cd69f289c05517d02bde22fc2f10e86fc2d0d209d9df54bd03d10b8123723c5587a2654dcd2cd0f314a016f9eccac88cffa30fafd1f9fead16db639e
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"msw@npm:^1.0.0, msw@npm:^1.0.1":
|
||||
version: 1.0.1
|
||||
resolution: "msw@npm:1.0.1"
|
||||
|
||||
Reference in New Issue
Block a user