paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

fix(catalog-backend): address comments and add docs

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-09-18 09:49:21 +02:00
parent a4e563ed2d
commit 94b8e88eba
7 changed files with 106 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+4
View File
@@ -21,6 +21,10 @@ If you encounter issues while upgrading to a newer version, don't hesitate to re
- Change root `tsc` output dir to `dist-types`, in order to allow for standalone plugin repos. [#2278](https://github.com/spotify/backstage/pull/2278)
### @backstage/catalog-backend
- We have simplified the way that GitHub ingestion works. The `catalog.processors.githubApi` key is deprecated, in favor of `catalog.processors.github`. At the same time, the location type `github/api` is likewise deprecated, in favor of `github`. This location type now serves both raw HTTP reads and APIv3 reads, depending on how you configure it. It also supports having several providers at once - for example, both public GitHub and an internal GitHub Enterprise, with different keys. If you still use the `catalog.processors.githubApi` config key, things will work but you will get a deprecation warning at startup. In a later release, support for the old key will go away entirely. See the [configuration section in the docs](https://backstage.io/docs/features/software-catalog/configuration) for more details.
## v0.1.1-alpha.21
- Added many more frontend plugins to the template along with the sidebar. [#1942](https://github.com/spotify/backstage/pull/1942), [#2084](https://github.com/spotify/backstage/pull/2084)
docs/features/software-catalog/configuration.md
+61
View File
@@ -4,6 +4,67 @@ title: Catalog Configuration
description: Documentation on Software Catalog Configuration
---
## Processors
The catalog makes use of so called processors to perform all kinds of ingestion
tasks, such as reading raw entity data from a remote source, parsing it,
transforming it, and validating it. These processors are configured under the
`catalog.processors` key.
### Processor: github
The `github` processor is responsible for fetching entity data from files on
GitHub or GitHub Enterprise. The configuration for this processor lives under
`catalog.processors.github`. Example:
```yaml
catalog:
processors:
github:
providers:
- target: https://github.com
token:
$secret:
env: GITHUB_PRIVATE_TOKEN
- target: https://ghe.example.net
apiBaseUrl: https://ghe.example.net/api/v3
rawBaseUrl: https://ghe.example.net/raw
token:
$secret:
env: GHE_PRIVATE_TOKEN
```
The main subkey is `providers`, where you can list the various GitHub compatible
providers you want to be able to fetch data from. Each entry is a structure with
up to four elements:
- `target` (required): The string prefix of the location target that you want to
match on, with no trailing slash. For GitHub, it should be exactly
`https://github.com`.
- `token` (optional): An authentication token as expected by GitHub. If
supplied, it will be passed along with all calls to this provider, both API
and raw. If it is not supplied, anonymous access will be used.
- `apiBaseUrl` (optional): If you want to communicate using the APIv3 method
with this provider, specify the base URL for its endpoint here, with no
trailing slash. Specifically when the target is github, you can leave it out
to be inferred automatically. For a GitHub Enterprise installation, it is
commonly at `https://api.<host>` or `https://<host>/api/v3`.
- `rawBaseUrl` (optional): If you want to communicate using the raw HTTP method
with this provider, specify the base URL for its endpoint here, with no
trailing slash. Specifically when the target is public GitHub, you can leave
it out to be inferred automatically. For a GitHub Enterprise installation, it
is commonly at `https://api.<host>` or `https://<host>/api/v3`.
You need to supply either `apiBaseUrl` or `rawBaseUrl` or both (except for
public GitHub, for which we can infer them). The `apiBaseUrl` will always be
preferred over the other if a `token` is given, otherwise `rawBaseUrl` will be
preferred.
If you do not supply a public GitHub provider, one will be added automatically,
silently at startup for convenience. So you only have to list it if you want to
supply a token for it - and if you do, you can also leave out the `apiBaseUrl`
and `rawBaseUrl` fields.
## Static Location Configuration
To enable declarative catalog setups, it is possible to add locations to the
microsite/sidebars.json
+1
View File
@@ -40,6 +40,7 @@
"ids": [
"features/software-catalog/software-catalog-overview",
"features/software-catalog/installation",
"features/software-catalog/configuration",
"features/software-catalog/system-model",
"features/software-catalog/descriptor-format",
"features/software-catalog/well-known-annotations",
mkdocs.yml
+2 -1
View File
@@ -28,10 +28,11 @@ nav:
- Features:
- Software Catalog:
- Overview: 'features/software-catalog/index.md'
- Installation: 'features/software-catalog/installation.md'
- Configuration: 'features/software-catalog/configuration.md'
- System model: 'features/software-catalog/system-model.md'
- YAML File Format: 'features/software-catalog/descriptor-format.md'
- Well-known Annotations: 'features/software-catalog/well-known-annotations.md'
- Configuration: 'features/software-catalog/configuration.md'
- Extending the model: 'features/software-catalog/extending-the-model.md'
- External integrations: 'features/software-catalog/external-integrations.md'
- API: 'features/software-catalog/api.md'
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+4 -2
View File
@@ -66,17 +66,19 @@ export class LocationReaders implements LocationReader {
private readonly rulesEnforcer: CatalogRulesEnforcer;
static defaultProcessors(options: {
logger: Logger;
config?: Config;
entityPolicy?: EntityPolicy;
}): LocationProcessor[] {
const {
logger,
config = new ConfigReader({}, 'missing-config'),
entityPolicy = new EntityPolicies(),
} = options;
return [
StaticLocationProcessor.fromConfig(config),
new FileReaderProcessor(),
GithubReaderProcessor.fromConfig(config),
GithubReaderProcessor.fromConfig(config, logger),
new GitlabApiReaderProcessor(config),
new GitlabReaderProcessor(),
new BitbucketApiReaderProcessor(config),
@@ -92,7 +94,7 @@ export class LocationReaders implements LocationReader {
constructor({
logger = getVoidLogger(),
config,
processors = LocationReaders.defaultProcessors({ config }),
processors = LocationReaders.defaultProcessors({ logger, config }),
}: Options) {
this.logger = logger;
this.processors = processors;
plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts
+23 -12
View File
@@ -14,13 +14,14 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { LocationSpec } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import {
getApiUrl,
getApiRequestOptions,
getRawUrl,
getApiUrl,
getRawRequestOptions,
getRawUrl,
GithubReaderProcessor,
ProviderConfig,
readConfig,
@@ -179,7 +180,7 @@ describe('GithubReaderProcessor', () => {
}
it('adds a default GitHub entry when missing', () => {
const output = readConfig(config([]));
const output = readConfig(config([]), getVoidLogger());
expect(output).toEqual([
{
target: 'https://github.com',
@@ -190,7 +191,10 @@ describe('GithubReaderProcessor', () => {
});
it('injects the correct GitHub API base URL when missing', () => {
const output = readConfig(config([{ target: 'https://github.com' }]));
const output = readConfig(
config([{ target: 'https://github.com' }]),
getVoidLogger(),
);
expect(output).toEqual([
{
target: 'https://github.com',
@@ -202,26 +206,33 @@ describe('GithubReaderProcessor', () => {
it('rejects custom targets with no base URLs', () => {
expect(() =>
readConfig(config([{ target: 'https://ghe.company.com' }])),
readConfig(
config([{ target: 'https://ghe.company.com' }]),
getVoidLogger(),
),
).toThrow(
'Provider at https://ghe.company.com must configure an explicit apiBaseUrl or rawBaseUrl',
);
});
it('rejects funky configs', () => {
expect(() => readConfig(config([{ target: 7 } as any]))).toThrow(
/target/,
);
expect(() => readConfig(config([{ noTarget: '7' } as any]))).toThrow(
/target/,
);
expect(() =>
readConfig(config([{ target: 7 } as any]), getVoidLogger()),
).toThrow(/target/);
expect(() =>
readConfig(config([{ noTarget: '7' } as any]), getVoidLogger()),
).toThrow(/target/);
expect(() =>
readConfig(
config([{ target: 'https://github.com', apiBaseUrl: 7 } as any]),
getVoidLogger(),
),
).toThrow(/apiBaseUrl/);
expect(() =>
readConfig(config([{ target: 'https://github.com', token: 7 } as any])),
readConfig(
config([{ target: 'https://github.com', token: 7 } as any]),
getVoidLogger(),
),
).toThrow(/token/);
});
});
plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts
+11 -4
View File
@@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import parseGitUri from 'git-url-parse';
import fetch, { HeadersInit, RequestInit } from 'node-fetch';
import { Logger } from 'winston';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
@@ -139,9 +140,16 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL {
}
}
export function readConfig(config: Config): ProviderConfig[] {
export function readConfig(config: Config, logger: Logger): ProviderConfig[] {
const providers: ProviderConfig[] = [];
// TODO(freben): Deprecate the old config root entirely in a later release
if (config.has('catalog.processors.githubApi')) {
logger.warn(
'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead',
);
}
// In a previous version of the configuration, we only supported github,
// and the "privateToken" key held the token to use for it. The new
// configuration method is to use the "providers" key instead.
@@ -202,8 +210,8 @@ export function readConfig(config: Config): ProviderConfig[] {
export class GithubReaderProcessor implements LocationProcessor {
private providers: ProviderConfig[];
static fromConfig(config: Config) {
return new GithubReaderProcessor(readConfig(config));
static fromConfig(config: Config, logger: Logger) {
return new GithubReaderProcessor(readConfig(config, logger));
}
constructor(providers: ProviderConfig[]) {
@@ -238,7 +246,6 @@ export class GithubReaderProcessor implements LocationProcessor {
const options = useApi
? getApiRequestOptions(provider)
: getRawRequestOptions(provider);
console.log(url.toString());
const response = await fetch(url.toString(), options);
if (response.ok) {