diff --git a/app-config.yaml b/app-config.yaml index 2efe6adc32..5347fbe617 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -45,6 +45,26 @@ newrelic: key: NEW_RELIC_REST_API_KEY catalog: + processors: + githubApi: + privateToken: + $secret: + env: GITHUB_PRIVATE_TOKEN + bitbucketApi: + username: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + gitlabApi: + privateToken: + $secret: + env: GITLAB_PRIVATE_TOKEN + azureApi: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN exampleEntityLocations: github: - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 29892bb2dc..46fa6781f1 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -76,11 +76,11 @@ export class LocationReaders implements LocationReader { StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), new GithubReaderProcessor(), - new GithubApiReaderProcessor(), - new GitlabApiReaderProcessor(), + new GithubApiReaderProcessor(config), + new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), - new BitbucketApiReaderProcessor(), - new AzureApiReaderProcessor(), + new BitbucketApiReaderProcessor(config), + new AzureApiReaderProcessor(config), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts index 0ef111fcb1..62259f87a2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('AzureApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + azureApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new AzureApiReaderProcessor(); + const processor = new AzureApiReaderProcessor(createConfig(undefined)); const tests = [ { target: @@ -72,13 +89,26 @@ describe('AzureApiReaderProcessor', () => { expect: { headers: {}, }, + err: + "Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string", + }, + { + token: undefined, + expect: { + headers: {}, + }, }, ]; for (const test of tests) { - process.env.AZURE_PRIVATE_TOKEN = test.token; - const processor = new AzureApiReaderProcessor(); - expect(processor.getRequestOptions()).toEqual(test.expect); + if (test.err) { + expect( + () => new AzureApiReaderProcessor(createConfig(test.token)), + ).toThrowError(test.err); + } else { + const processor = new AzureApiReaderProcessor(createConfig(test.token)); + expect(processor.getRequestOptions()).toEqual(test.expect); + } } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts index 41b4336b4f..7033521da6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class AzureApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.azureApi.privateToken') ?? + ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; @@ -53,7 +60,8 @@ export class AzureApiReaderProcessor implements LocationProcessor { const response = await fetch(url.toString(), this.getRequestOptions()); - if (response.ok) { + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { const data = await response.buffer(); emit(result.data(location, data)); } else { @@ -76,7 +84,6 @@ export class AzureApiReaderProcessor implements LocationProcessor { // Converts // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 - buildRawUrl(target: string): URL { try { const url = new URL(target); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts index fa4731a40e..953f0703be 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts @@ -15,10 +15,33 @@ */ import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('BitbucketApiReaderProcessor', () => { + const createConfig = ( + username: string | undefined, + appPassword: string | undefined, + ) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + bitbucketApi: { + username: username, + appPassword: appPassword, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new BitbucketApiReaderProcessor(); + const processor = new BitbucketApiReaderProcessor( + createConfig(undefined, undefined), + ); const tests = [ { @@ -66,6 +89,8 @@ describe('BitbucketApiReaderProcessor', () => { expect: { headers: {}, }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", }, { username: 'only-user-provided', @@ -73,6 +98,8 @@ describe('BitbucketApiReaderProcessor', () => { expect: { headers: {}, }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string", }, { username: '', @@ -80,6 +107,8 @@ describe('BitbucketApiReaderProcessor', () => { expect: { headers: {}, }, + err: + "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string", }, { username: 'some-user', @@ -90,13 +119,43 @@ describe('BitbucketApiReaderProcessor', () => { }, }, }, + { + username: undefined, + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: 'only-user-provided', + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: undefined, + password: 'only-password-provided', + expect: { + headers: {}, + }, + }, ]; for (const test of tests) { - process.env.BITBUCKET_USERNAME = test.username; - process.env.BITBUCKET_APP_PASSWORD = test.password; - const processor = new BitbucketApiReaderProcessor(); - expect(processor.getRequestOptions()).toEqual(test.expect); + if (test.err) { + expect( + () => + new BitbucketApiReaderProcessor( + createConfig(test.username, test.password), + ), + ).toThrowError(test.err); + } else { + const processor = new BitbucketApiReaderProcessor( + createConfig(test.username, test.password), + ); + expect(processor.getRequestOptions()).toEqual(test.expect); + } } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts index 8162e505d6..97ecf0cd1f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -18,10 +18,20 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class BitbucketApiReaderProcessor implements LocationProcessor { - private username: string = process.env.BITBUCKET_USERNAME || ''; - private password: string = process.env.BITBUCKET_APP_PASSWORD || ''; + private username: string; + private password: string; + + constructor(config: Config) { + this.username = + config.getOptionalString('catalog.processors.bitbucketApi.username') ?? + ''; + this.password = + config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ?? + ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index 63ddb071c3..51c3cf4cd0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('GithubApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + githubApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new GithubApiReaderProcessor(); + const processor = new GithubApiReaderProcessor(createConfig(undefined)); const tests = [ { @@ -78,6 +95,16 @@ describe('GithubApiReaderProcessor', () => { }, { token: '', + err: + "Invalid type in config for key 'catalog.processors.githubApi.privateToken' in '', got empty-string, wanted string", + expect: { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }, + }, + { + token: undefined, expect: { headers: { Accept: 'application/vnd.github.v3.raw', @@ -87,9 +114,16 @@ describe('GithubApiReaderProcessor', () => { ]; for (const test of tests) { - process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubApiReaderProcessor(); - expect(processor.getRequestOptions()).toEqual(test.expect); + if (test.err) { + expect( + () => new GithubApiReaderProcessor(createConfig(test.token)), + ).toThrowError(test.err); + } else { + const processor = new GithubApiReaderProcessor( + createConfig(test.token), + ); + expect(processor.getRequestOptions()).toEqual(test.expect); + } } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts index ff61d004ca..f8d1d6caf8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class GithubApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.githubApi.privateToken') ?? + ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = { diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts index c43c68591e..48ed270049 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('GitlabApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + catalog: { + processors: { + gitlabApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new GitlabApiReaderProcessor(); + const processor = new GitlabApiReaderProcessor(createConfig(undefined)); const tests = [ { @@ -83,6 +100,16 @@ describe('GitlabApiReaderProcessor', () => { }, { token: '', + err: + "Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string", + expect: { + headers: { + 'PRIVATE-TOKEN': '', + }, + }, + }, + { + token: undefined, expect: { headers: { 'PRIVATE-TOKEN': '', @@ -92,9 +119,16 @@ describe('GitlabApiReaderProcessor', () => { ]; for (const test of tests) { - process.env.GITLAB_PRIVATE_TOKEN = test.token; - const processor = new GitlabApiReaderProcessor(); - expect(processor.getRequestOptions()).toEqual(test.expect); + if (test.err) { + expect( + () => new GitlabApiReaderProcessor(createConfig(test.token)), + ).toThrowError(test.err); + } else { + const processor = new GitlabApiReaderProcessor( + createConfig(test.token), + ); + expect(processor.getRequestOptions()).toEqual(test.expect); + } } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts index 3e585bf2b3..067edba3d9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -18,9 +18,16 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class GitlabApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + config.getOptionalString('catalog.processors.gitlabApi.privateToken') ?? + ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index e4fc90699b..2b465a536e 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -37,6 +37,7 @@ describe('ComponentIdValidators', () => { [true, 'http://example.com/blob/master/service.yaml'], [true, 'https://example.yaml'], [true, 'https://example.com?path=abc.yaml&c=1'], + [errorMessage, 'https://example.com?path=abc_yaml&c=1'], [errorMessage, '.yml'], [errorMessage, 'http://example.com/blob/master/service'], [errorMessage, undefined],