paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

Update ingestion processors to use config system

Browse Source
This commit is contained in:
Omer Farooq
2020-08-29 15:34:09 +12:00
parent 314eeb31d7
commit dd3963093d
11 changed files with 187 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+20
View File
@@ -13,6 +13,26 @@ backend:
database:
client: sqlite3
connection: ':memory:'
ingestionProcessors:
githubApi:
privateToken:
$secret:
env: GITHUB_PRIVATE_TOKEN
bitbucketApi:
userName:
$secret:
env: BITBUCKET_USERNAME
appPassword:
$secret:
env: BITBUCKET_APP_PASSWORD
gitlabApi:
privateToken:
$secret:
env: GITLAB_PRIVATE_TOKEN
azureApi:
privateToken:
$secret:
env: AZURE_PRIVATE_TOKEN
# See README.md in the proxy-backend plugin for information on the configuration format
proxy:
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+4 -4
View File
@@ -76,11 +76,11 @@ export class LocationReaders implements LocationReader {
StaticLocationProcessor.fromConfig(config),
new FileReaderProcessor(),
new GithubReaderProcessor(),
new GithubApiReaderProcessor(),
new GitlabApiReaderProcessor(),
new GithubApiReaderProcessor(config),
new GitlabApiReaderProcessor(config),
new GitlabReaderProcessor(),
new BitbucketApiReaderProcessor(),
new AzureApiReaderProcessor(),
new BitbucketApiReaderProcessor(config),
new AzureApiReaderProcessor(config),
new UrlReaderProcessor(),
new YamlProcessor(),
new EntityPolicyProcessor(entityPolicy),
plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+25 -3
View File
@@ -15,10 +15,27 @@
*/
import { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('AzureApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
backend: {
ingestionProcessors: {
azureApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new AzureApiReaderProcessor();
const processor = new AzureApiReaderProcessor(createConfig(undefined));
const tests = [
{
target:
@@ -73,11 +90,16 @@ describe('AzureApiReaderProcessor', () => {
headers: {},
},
},
{
token: undefined,
expect: {
headers: {},
},
},
];
for (const test of tests) {
process.env.AZURE_PRIVATE_TOKEN = test.token;
const processor = new AzureApiReaderProcessor();
const processor = new AzureApiReaderProcessor(createConfig(test.token));
expect(processor.getRequestOptions()).toEqual(test.expect);
}
});
plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+11 -3
View File
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class AzureApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
(config.getOptional(
'backend.ingestionProcessors.azureApi.privateToken',
) as string) ?? '';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {};
@@ -53,7 +61,8 @@ export class AzureApiReaderProcessor implements LocationProcessor {
const response = await fetch(url.toString(), this.getRequestOptions());
if (response.ok) {
// for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
if (response.ok && response.status !== 203) {
const data = await response.buffer();
emit(result.data(location, data));
} else {
@@ -76,7 +85,6 @@ export class AzureApiReaderProcessor implements LocationProcessor {
// Converts
// from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
// to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1
buildRawUrl(target: string): URL {
try {
const url = new URL(target);
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+48 -4
View File
@@ -15,10 +15,33 @@
*/
import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('BitbucketApiReaderProcessor', () => {
const createConfig = (
userName: string | undefined,
appPassword: string | undefined,
) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
backend: {
ingestionProcessors: {
bitbucketApi: {
userName: userName,
appPassword: appPassword,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new BitbucketApiReaderProcessor();
const processor = new BitbucketApiReaderProcessor(
createConfig(undefined, undefined),
);
const tests = [
{
@@ -90,12 +113,33 @@ describe('BitbucketApiReaderProcessor', () => {
},
},
},
{
username: undefined,
password: undefined,
expect: {
headers: {},
},
},
{
username: 'only-user-provided',
password: undefined,
expect: {
headers: {},
},
},
{
username: undefined,
password: 'only-password-provided',
expect: {
headers: {},
},
},
];
for (const test of tests) {
process.env.BITBUCKET_USERNAME = test.username;
process.env.BITBUCKET_APP_PASSWORD = test.password;
const processor = new BitbucketApiReaderProcessor();
const processor = new BitbucketApiReaderProcessor(
createConfig(test.username, test.password),
);
expect(processor.getRequestOptions()).toEqual(test.expect);
}
});
plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+14 -2
View File
@@ -18,10 +18,22 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class BitbucketApiReaderProcessor implements LocationProcessor {
private username: string = process.env.BITBUCKET_USERNAME || '';
private password: string = process.env.BITBUCKET_APP_PASSWORD || '';
private username: string;
private password: string;
constructor(config: Config) {
this.username =
(config.getOptional(
'backend.ingestionProcessors.bitbucketApi.userName',
) as string) ?? '';
this.password =
(config.getOptional(
'backend.ingestionProcessors.bitbucketApi.appPassword',
) as string) ?? '';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {};
plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+19 -3
View File
@@ -15,10 +15,27 @@
*/
import { GithubApiReaderProcessor } from './GithubApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('GithubApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
backend: {
ingestionProcessors: {
githubApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new GithubApiReaderProcessor();
const processor = new GithubApiReaderProcessor(createConfig(undefined));
const tests = [
{
@@ -87,8 +104,7 @@ describe('GithubApiReaderProcessor', () => {
];
for (const test of tests) {
process.env.GITHUB_PRIVATE_TOKEN = test.token;
const processor = new GithubApiReaderProcessor();
const processor = new GithubApiReaderProcessor(createConfig(test.token));
expect(processor.getRequestOptions()).toEqual(test.expect);
}
});
plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+9 -1
View File
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class GithubApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
(config.getOptional(
'backend.ingestionProcessors.githubApi.privateToken',
) as string) ?? '';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = {
plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+27 -3
View File
@@ -15,10 +15,27 @@
*/
import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor';
import { ConfigReader } from '@backstage/config';
describe('GitlabApiReaderProcessor', () => {
const createConfig = (token: string | undefined) =>
ConfigReader.fromConfigs([
{
context: '',
data: {
backend: {
ingestionProcessors: {
gitlabApi: {
privateToken: token,
},
},
},
},
},
]);
it('should build raw api', () => {
const processor = new GitlabApiReaderProcessor();
const processor = new GitlabApiReaderProcessor(createConfig(undefined));
const tests = [
{
@@ -89,11 +106,18 @@ describe('GitlabApiReaderProcessor', () => {
},
},
},
{
token: undefined,
expect: {
headers: {
'PRIVATE-TOKEN': '',
},
},
},
];
for (const test of tests) {
process.env.GITLAB_PRIVATE_TOKEN = test.token;
const processor = new GitlabApiReaderProcessor();
const processor = new GitlabApiReaderProcessor(createConfig(test.token));
expect(processor.getRequestOptions()).toEqual(test.expect);
}
});
plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+9 -1
View File
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch, { RequestInit, HeadersInit } from 'node-fetch';
import * as result from './results';
import { LocationProcessor, LocationProcessorEmit } from './types';
import { Config } from '@backstage/config';
export class GitlabApiReaderProcessor implements LocationProcessor {
private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || '';
private privateToken: string;
constructor(config: Config) {
this.privateToken =
(config.getOptional(
'backend.ingestionProcessors.gitlabApi.privateToken',
) as string) ?? '';
}
getRequestOptions(): RequestInit {
const headers: HeadersInit = { 'PRIVATE-TOKEN': '' };
plugins/register-component/src/util/validate.test.ts
+1
View File
@@ -37,6 +37,7 @@ describe('ComponentIdValidators', () => {
[true, 'http://example.com/blob/master/service.yaml'],
[true, 'https://example.yaml'],
[true, 'https://example.com?path=abc.yaml&c=1'],
[errorMessage, 'https://example.com?path=abc_yaml&c=1'],
[errorMessage, '.yml'],
[errorMessage, 'http://example.com/blob/master/service'],
[errorMessage, undefined],