paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 25 Packages Projects Releases Wiki Activity

Merge pull request #2169 from o-farooq/master

Browse Source 
catalog-backend: update ingestion processors to use config system
This commit is contained in:
Fredrik Adelöw
2020-08-30 10:11:51 +02:00
committed by GitHub
parent 314eeb31d7 298a9881dd
commit fe4ef73eac
11 changed files with 237 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+20
View File
@@ -45,6 +45,26 @@ newrelic:
key: NEW_RELIC_REST_API_KEY
catalog:
processors:
githubApi:
privateToken:
$secret:
env: GITHUB_PRIVATE_TOKEN
bitbucketApi:
username:
$secret:
env: BITBUCKET_USERNAME
appPassword:
$secret:
env: BITBUCKET_APP_PASSWORD
gitlabApi:
privateToken:
$secret:
env: GITLAB_PRIVATE_TOKEN
azureApi:
privateToken:
$secret:
env: AZURE_PRIVATE_TOKEN
exampleEntityLocations:
github:
- https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+4 -4
View File
@@ -76,11 +76,11 @@ export class LocationReaders implements LocationReader {
StaticLocationProcessor.fromConfig(config),
new FileReaderProcessor(),
new GithubReaderProcessor(),
new GithubApiReaderProcessor(),
new GitlabApiReaderProcessor(),
new GithubApiReaderProcessor(config),
new GitlabApiReaderProcessor(config),
new GitlabReaderProcessor(),
new BitbucketApiReaderProcessor(),
new AzureApiReaderProcessor(),
new BitbucketApiReaderProcessor(config),
new AzureApiReaderProcessor(config),
new UrlReaderProcessor(),
new YamlProcessor(),
new EntityPolicyProcessor(entityPolicy),
plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+34 -4
View File
@@ -15,10 +15,27 @@
*/
import { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('AzureApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
catalog: {
processors: {
azureApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new AzureApiReaderProcessor();
const processor = new AzureApiReaderProcessor(createConfig(undefined));
const tests = [
{
target:
@@ -72,13 +89,26 @@ describe('AzureApiReaderProcessor', () => {
expect: {
headers: {},
},
err:
"Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string",
},
{
token: undefined,
expect: {
headers: {},
},
},
];
for (const test of tests) {
process.env.AZURE_PRIVATE_TOKEN = test.token;
const processor = new AzureApiReaderProcessor();
expect(processor.getRequestOptions()).toEqual(test.expect);
if (test.err) {
expect(
() => new AzureApiReaderProcessor(createConfig(test.token)),
).toThrowError(test.err);
} else {
const processor = new AzureApiReaderProcessor(createConfig(test.token));
expect(processor.getRequestOptions()).toEqual(test.expect);
}
}
});
});
plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+10 -3
View File
@@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class AzureApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('catalog.processors.azureApi.privateToken') ??
'';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {};
@@ -53,7 +60,8 @@ export class AzureApiReaderProcessor implements LocationProcessor {
const response = await fetch(url.toString(), this.getRequestOptions());
if (response.ok) {
// for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
if (response.ok && response.status !== 203) {
const data = await response.buffer();
emit(result.data(location, data));
} else {
@@ -76,7 +84,6 @@ export class AzureApiReaderProcessor implements LocationProcessor {
// Converts
// from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
// to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1
buildRawUrl(target: string): URL {
try {
const url = new URL(target);
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+64 -5
View File
@@ -15,10 +15,33 @@
*/
import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('BitbucketApiReaderProcessor', () => {
const createConfig = (
username: string | undefined,
appPassword: string | undefined,
) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
catalog: {
processors: {
bitbucketApi: {
username: username,
appPassword: appPassword,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new BitbucketApiReaderProcessor();
const processor = new BitbucketApiReaderProcessor(
createConfig(undefined, undefined),
);
const tests = [
{
@@ -66,6 +89,8 @@ describe('BitbucketApiReaderProcessor', () => {
expect: {
headers: {},
},
err:
"Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string",
},
{
username: 'only-user-provided',
@@ -73,6 +98,8 @@ describe('BitbucketApiReaderProcessor', () => {
expect: {
headers: {},
},
err:
"Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string",
},
{
username: '',
@@ -80,6 +107,8 @@ describe('BitbucketApiReaderProcessor', () => {
expect: {
headers: {},
},
err:
"Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string",
},
{
username: 'some-user',
@@ -90,13 +119,43 @@ describe('BitbucketApiReaderProcessor', () => {
},
},
},
{
username: undefined,
password: undefined,
expect: {
headers: {},
},
},
{
username: 'only-user-provided',
password: undefined,
expect: {
headers: {},
},
},
{
username: undefined,
password: 'only-password-provided',
expect: {
headers: {},
},
},
];
for (const test of tests) {
process.env.BITBUCKET_USERNAME = test.username;
process.env.BITBUCKET_APP_PASSWORD = test.password;
const processor = new BitbucketApiReaderProcessor();
expect(processor.getRequestOptions()).toEqual(test.expect);
if (test.err) {
expect(
() =>
new BitbucketApiReaderProcessor(
createConfig(test.username, test.password),
),
).toThrowError(test.err);
} else {
const processor = new BitbucketApiReaderProcessor(
createConfig(test.username, test.password),
);
expect(processor.getRequestOptions()).toEqual(test.expect);
}
}
});
});
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+12 -2
View File
@@ -18,10 +18,20 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class BitbucketApiReaderProcessor implements LocationProcessor {
private username: string = process.env.BITBUCKET_USERNAME || '';
private password: string = process.env.BITBUCKET_APP_PASSWORD || '';
private username: string;
private password: string;
constructor(config: Config) {
this.username =
config.getOptionalString('catalog.processors.bitbucketApi.username') ??
'';
this.password =
config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ??
'';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {};
plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+38 -4
View File
@@ -15,10 +15,27 @@
*/
import { GithubApiReaderProcessor } from './GithubApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('GithubApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
catalog: {
processors: {
githubApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new GithubApiReaderProcessor();
const processor = new GithubApiReaderProcessor(createConfig(undefined));
const tests = [
{
@@ -78,6 +95,16 @@ describe('GithubApiReaderProcessor', () => {
},
{
token: '',
err:
"Invalid type in config for key 'catalog.processors.githubApi.privateToken' in '', got empty-string, wanted string",
expect: {
headers: {
Accept: 'application/vnd.github.v3.raw',
},
},
},
{
token: undefined,
expect: {
headers: {
Accept: 'application/vnd.github.v3.raw',
@@ -87,9 +114,16 @@ describe('GithubApiReaderProcessor', () => {
];
for (const test of tests) {
process.env.GITHUB_PRIVATE_TOKEN = test.token;
const processor = new GithubApiReaderProcessor();
expect(processor.getRequestOptions()).toEqual(test.expect);
if (test.err) {
expect(
() => new GithubApiReaderProcessor(createConfig(test.token)),
).toThrowError(test.err);
} else {
const processor = new GithubApiReaderProcessor(
createConfig(test.token),
);
expect(processor.getRequestOptions()).toEqual(test.expect);
}
}
});
});
plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+8 -1
View File
@@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class GithubApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('catalog.processors.githubApi.privateToken') ??
'';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {
plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+38 -4
View File
@@ -15,10 +15,27 @@
*/
import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('GitlabApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
catalog: {
processors: {
gitlabApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new GitlabApiReaderProcessor();
const processor = new GitlabApiReaderProcessor(createConfig(undefined));
const tests = [
{
@@ -83,6 +100,16 @@ describe('GitlabApiReaderProcessor', () => {
},
{
token: '',
err:
"Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string",
expect: {
headers: {
'PRIVATE-TOKEN': '',
},
},
},
{
token: undefined,
expect: {
headers: {
'PRIVATE-TOKEN': '',
@@ -92,9 +119,16 @@ describe('GitlabApiReaderProcessor', () => {
];
for (const test of tests) {
process.env.GITLAB_PRIVATE_TOKEN = test.token;
const processor = new GitlabApiReaderProcessor();
expect(processor.getRequestOptions()).toEqual(test.expect);
if (test.err) {
expect(
() => new GitlabApiReaderProcessor(createConfig(test.token)),
).toThrowError(test.err);
} else {
const processor = new GitlabApiReaderProcessor(
createConfig(test.token),
);
expect(processor.getRequestOptions()).toEqual(test.expect);
}
}
});
});
plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+8 -1
View File
@@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class GitlabApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
'';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = { 'PRIVATE-TOKEN': '' };
plugins/register-component/src/util/validate.test.ts
+1
View File
@@ -37,6 +37,7 @@ describe('ComponentIdValidators', () => {
[true, 'http://example.com/blob/master/service.yaml'],
[true, 'https://example.yaml'],
[true, 'https://example.com?path=abc.yaml&c=1'],
[errorMessage, 'https://example.com?path=abc_yaml&c=1'],
[errorMessage, '.yml'],
[errorMessage, 'http://example.com/blob/master/service'],
[errorMessage, undefined],