paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

feat(ADR): implement ADR plugin

Browse Source 
Signed-off-by: Phil Kuang <pkuang@factset.com>
This commit is contained in:
Phil Kuang
2022-04-13 14:39:17 -04:00
parent 1cb9cec16b
commit e73075a301
37 changed files with 1679 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/adr-backend/.eslintrc.js
+1
View File
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
plugins/adr-backend/README.md
+47
View File
@@ -0,0 +1,47 @@
# ADR Backend
This ADR backend plugin is primarily responsible for the following:
- Provides a `DefaultAdrCollatorFactory`, which can be used in the search backend to index ADR documents associated with entities to your Backstage Search.
## Indexing ADR documents for search
Before you are able to start indexing ADR documents to search, you need to go through the [search getting started guide](https://backstage.io/docs/features/search/getting-started).
When you have your `packages/backend/src/plugins/search.ts` file ready to make modifications, install this plugin and add the following code snippet to add the `DefaultAdrCollatorFactory`. Also make sure to set up the frontend [ADR plugin](../adr/README.md) so search results can be routed correctly.
```bash
# From your Backstage root directory
yarn --cwd packages/backend add @backstage/plugin-adr-backend
```
```ts
import { DefaultAdrCollatorFactory } from '@backstage/plugin-adr-backend';
...
indexBuilder.addCollator({
schedule,
factory: DefaultAdrCollatorFactory.fromConfig({
cache: env.cache,
config: env.config,
discovery: env.discovery,
logger: env.logger,
reader: env.reader,
tokenManager: env.tokenManager,
}),
});
```
### Parsing custom ADR document formats
By default, the `DefaultAdrCollatorFactory` will parse and index documents that follow the [MADR](https://adr.github.io/madr/) standard file name and template format. If you use a different ADR format and file name convention, you can configure `DefaultAdrCollatorFactory` with custom `adrFilePathFilterFn` and `parser` options (see type definitions for details):
```ts
DefaultAdrCollatorFactory.fromConfig({
...
parser: myCustomAdrParser,
adrFilePathFilterFn: myCustomAdrFilePathFilter,
...
})
```
plugins/adr-backend/api-report.md
+66
View File
@@ -0,0 +1,66 @@
## API Report File for "@backstage/plugin-adr-backend"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
/// <reference types="node" />
import { AdrDocument } from '@backstage/plugin-adr-common';
import { AdrFilePathFilterFn } from '@backstage/plugin-adr-common';
import { CatalogApi } from '@backstage/catalog-client';
import { Config } from '@backstage/config';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { Entity } from '@backstage/catalog-model';
import { Logger } from 'winston';
import { PluginCacheManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { Readable } from 'stream';
import { TokenManager } from '@backstage/backend-common';
import { UrlReader } from '@backstage/backend-common';
// @public
export type AdrCollatorFactoryOptions = {
adrFilePathFilterFn?: AdrFilePathFilterFn;
cache: PluginCacheManager;
config: Config;
catalogClient?: CatalogApi;
discovery: PluginEndpointDiscovery;
logger: Logger;
parser?: AdrParser;
reader: UrlReader;
tokenManager: TokenManager;
};
// @public
export type AdrParser = (ctx: AdrParserContext) => Promise<AdrDocument>;
// @public
export type AdrParserContext = {
entity: Entity;
content: string;
path: string;
};
// @public
export const createMadrParser: (options?: MadrParserOptions) => AdrParser;
// @public
export class DefaultAdrCollatorFactory implements DocumentCollatorFactory {
// (undocumented)
execute(): AsyncGenerator<AdrDocument>;
// (undocumented)
static fromConfig(
options: AdrCollatorFactoryOptions,
): DefaultAdrCollatorFactory;
// (undocumented)
getCollator(): Promise<Readable>;
// (undocumented)
readonly type: string;
}
// @public
export type MadrParserOptions = {
locationTemplate?: string;
dateFormat?: string;
};
```
plugins/adr-backend/package.json
+56
View File
@@ -0,0 +1,56 @@
{
"name": "@backstage/plugin-adr-backend",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/adr-backend"
},
"scripts": {
"start": "backstage-cli package start",
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"clean": "backstage-cli package clean",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/backend-common": "^0.13.3-next.0",
"@backstage/catalog-client": "^1.0.1",
"@backstage/catalog-model": "^1.0.1",
"@backstage/config": "^1.0.0",
"@backstage/errors": "^1.0.0",
"@backstage/integration": "^1.2.0-next.0",
"@backstage/plugin-adr-common": "^0.0.0",
"@backstage/plugin-search-common": "^0.3.3",
"luxon": "^2.0.2",
"marked": "^4.0.14",
"winston": "^3.2.1",
"node-fetch": "^2.6.5",
"yn": "^4.0.0"
},
"devDependencies": {
"@backstage/cli": "^0.17.1-next.0",
"@types/marked": "^4.0.0",
"@types/supertest": "^2.0.8",
"supertest": "^6.1.3",
"msw": "^0.35.0"
},
"files": [
"dist"
]
}
plugins/adr-backend/src/index.ts
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* ADR backend plugin
*
* @packageDocumentation
*/
export * from './search';
plugins/adr-backend/src/search/DefaultAdrCollatorFactory.ts
+212
View File
@@ -0,0 +1,212 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Readable } from 'stream';
import { Logger } from 'winston';
import {
CacheClient,
PluginCacheManager,
PluginEndpointDiscovery,
TokenManager,
UrlReader,
} from '@backstage/backend-common';
import {
CatalogApi,
CatalogClient,
CATALOG_FILTER_EXISTS,
} from '@backstage/catalog-client';
import { stringifyEntityRef } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotModifiedError } from '@backstage/errors';
import {
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import {
AdrDocument,
AdrFilePathFilterFn,
ANNOTATION_ADR_LOCATION,
getAdrLocationUrl,
madrFilePathFilter,
} from '@backstage/plugin-adr-common';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { createMadrParser } from './createMadrParser';
import { AdrParser } from './types';
/**
* Options to configure the AdrCollatorFactory
* @public
*/
export type AdrCollatorFactoryOptions = {
/**
* Function used to filter ADR file paths.
* Defaults to filtering paths following MADR filename format.
*/
adrFilePathFilterFn?: AdrFilePathFilterFn;
/**
* Plugin cache manager
*/
cache: PluginCacheManager;
/**
* App Config
*/
config: Config;
/**
* Catalog API client. Defaults to CatalogClient.
*/
catalogClient?: CatalogApi;
/**
* Plugin Endpoint Discovery client
*/
discovery: PluginEndpointDiscovery;
/**
* Logger
*/
logger: Logger;
/**
* ADR content parser. Defaults to built in MADR parser.
*/
parser?: AdrParser;
/**
* URL Reader
*/
reader: UrlReader;
/**
* Token Manager
*/
tokenManager: TokenManager;
};
/**
* Default collator to index ADR documents for Backstage search.
* @public
*/
export class DefaultAdrCollatorFactory implements DocumentCollatorFactory {
public readonly type: string = 'adr';
private readonly adrFilePathFilterFn: AdrFilePathFilterFn;
private readonly cacheClient: CacheClient;
private readonly catalogClient: CatalogApi;
private readonly logger: Logger;
private readonly parser: AdrParser;
private readonly reader: UrlReader;
private readonly scmIntegrations: ScmIntegrationRegistry;
private readonly tokenManager: TokenManager;
private constructor(options: AdrCollatorFactoryOptions) {
this.adrFilePathFilterFn =
options.adrFilePathFilterFn ?? madrFilePathFilter;
this.cacheClient = options.cache.getClient();
this.catalogClient =
options.catalogClient ??
new CatalogClient({ discoveryApi: options.discovery });
this.logger = options.logger;
this.parser = options.parser ?? createMadrParser();
this.reader = options.reader;
this.scmIntegrations = ScmIntegrations.fromConfig(options.config);
this.tokenManager = options.tokenManager;
}
static fromConfig(options: AdrCollatorFactoryOptions) {
return new DefaultAdrCollatorFactory(options);
}
async getCollator() {
return Readable.from(this.execute());
}
async *execute(): AsyncGenerator<AdrDocument> {
const { token } = await this.tokenManager.getToken();
const entities = (
await this.catalogClient.getEntities(
{
filter: {
[`metadata.annotations.${ANNOTATION_ADR_LOCATION}`]:
CATALOG_FILTER_EXISTS,
},
fields: [
'kind',
'metadata.annotations',
'metadata.name',
'metadata.namespace',
],
},
{ token },
)
).items;
for (const ent of entities) {
let adrsUrl: string;
try {
adrsUrl = getAdrLocationUrl(ent, this.scmIntegrations);
} catch (e: any) {
this.logger.error(
`Unable to get ADR location URL for ${stringifyEntityRef(ent)}: ${
e.message
}`,
);
continue;
}
const cacheItem = (await this.cacheClient.get(adrsUrl)) as {
adrFiles: {
path: string;
content: string;
}[];
etag: string;
};
let adrFiles = cacheItem?.adrFiles;
try {
const tree = await this.reader.readTree(adrsUrl, {
etag: cacheItem?.etag,
filter: filePath => this.adrFilePathFilterFn(filePath),
});
adrFiles = await Promise.all(
(
await tree.files()
).map(async f => ({
path: f.path,
content: (await f.content()).toString('utf8'),
})),
);
await this.cacheClient.set(adrsUrl, { adrFiles, etag: tree.etag });
} catch (error: any) {
// Ignore error if we're able to use cached response
if (!cacheItem || error.name !== NotModifiedError.name) {
throw error;
}
}
for (const { content, path } of adrFiles) {
try {
const adrDoc = await this.parser({
entity: ent,
path,
content,
});
yield adrDoc;
} catch (e: any) {
this.logger.error(`Unable to parse ADR ${path}: ${e.message}`);
}
}
this.logger.info(`Indexed ${adrFiles.length} ADRs from ${adrsUrl}`);
}
}
}
plugins/adr-backend/src/search/createMadrParser.ts
+112
View File
@@ -0,0 +1,112 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DateTime } from 'luxon';
import { marked } from 'marked';
import { MADR_DATE_FORMAT } from '@backstage/plugin-adr-common';
import { AdrParser } from './types';
const applyArgsToFormat = (
format: string,
args: Record<string, string>,
): string => {
let formatted = format;
for (const [key, value] of Object.entries(args)) {
formatted = formatted.replace(`:${key}`, value);
}
return formatted.toLowerCase();
};
const DEFAULT_LOCATION_TEMPLATE =
'/catalog/:namespace/:kind/:name/adrs?record=:record';
/**
*
* Options for the default MADR content parser
* @public
*/
export type MadrParserOptions = {
/**
* Location template for the route of the frontend plugin
* Defaults to '/catalog/:namespace/:kind/:name/adrs?record=:record'
*/
locationTemplate?: string;
/**
* luxon DateTime format string to parse ADR dates with.
* Defaults to 'yyyy-MM-dd'
*/
dateFormat?: string;
};
/**
* Default content parser for ADRs following the MADR template (https://adr.github.io/madr/)
* @public
*/
export const createMadrParser = (
options: MadrParserOptions = {},
): AdrParser => {
const locationTemplate =
options.locationTemplate ?? DEFAULT_LOCATION_TEMPLATE;
const dateFormat = options.dateFormat ?? MADR_DATE_FORMAT;
return async ({ entity, content, path }) => {
const tokens = marked.lexer(content);
if (!tokens.length) {
throw new Error('ADR has no content');
}
// First h1 header should contain ADR title
const adrTitle = (
tokens.find(
t => t.type === 'heading' && t.depth === 1,
) as marked.Tokens.Heading
)?.text;
// First list should contain status & date metadata (if defined)
const listTokens = (
tokens.find(t => t.type === 'list') as marked.Tokens.List
)?.items;
const adrStatus = listTokens
.find(t => /^status:/i.test(t.text))
?.text.replace(/^status:/i, '')
.trim()
.toLocaleLowerCase('en-US');
const adrDateTime = DateTime.fromFormat(
listTokens
.find(t => /^date:/i.test(t.text))
?.text.replace(/^date:/i, '')
.trim() ?? '',
dateFormat,
);
const adrDate = adrDateTime.isValid
? adrDateTime.toFormat(MADR_DATE_FORMAT)
: undefined;
return {
title: adrTitle ?? path.replace(/\.md$/, ''),
text: content,
status: adrStatus,
date: adrDate,
location: applyArgsToFormat(locationTemplate, {
namespace: entity.metadata.namespace || 'default',
kind: entity.kind,
name: entity.metadata.name,
record: path,
}),
};
};
};
plugins/adr-backend/src/search/index.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './DefaultAdrCollatorFactory';
export * from './createMadrParser';
export * from './types';
plugins/adr-backend/src/search/types.ts
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { AdrDocument } from '@backstage/plugin-adr-common';
/**
* Context passed to a AdrParser.
* @public
*/
export type AdrParserContext = {
/**
* The entity associated with the ADR.
*/
entity: Entity;
/**
* The ADR content string.
*/
content: string;
/**
* The ADR file path.
*/
path: string;
};
/**
* ADR parser function type.
* @public
*/
export type AdrParser = (ctx: AdrParserContext) => Promise<AdrDocument>;