diff --git a/app-config.yaml b/app-config.yaml index 2efe6adc32..880e8d0bd9 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -13,6 +13,26 @@ backend: database: client: sqlite3 connection: ':memory:' + ingestionProcessors: + githubApi: + privateToken: + $secret: + env: GITHUB_PRIVATE_TOKEN + bitbucketApi: + userName: + $secret: + env: BITBUCKET_USERNAME + appPassword: + $secret: + env: BITBUCKET_APP_PASSWORD + gitlabApi: + privateToken: + $secret: + env: GITLAB_PRIVATE_TOKEN + azureApi: + privateToken: + $secret: + env: AZURE_PRIVATE_TOKEN # See README.md in the proxy-backend plugin for information on the configuration format proxy: diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 29892bb2dc..46fa6781f1 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -76,11 +76,11 @@ export class LocationReaders implements LocationReader { StaticLocationProcessor.fromConfig(config), new FileReaderProcessor(), new GithubReaderProcessor(), - new GithubApiReaderProcessor(), - new GitlabApiReaderProcessor(), + new GithubApiReaderProcessor(config), + new GitlabApiReaderProcessor(config), new GitlabReaderProcessor(), - new BitbucketApiReaderProcessor(), - new AzureApiReaderProcessor(), + new BitbucketApiReaderProcessor(config), + new AzureApiReaderProcessor(config), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts index 0ef111fcb1..6ccb461721 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('AzureApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + backend: { + ingestionProcessors: { + azureApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new AzureApiReaderProcessor(); + const processor = new AzureApiReaderProcessor(createConfig(undefined)); const tests = [ { target: @@ -73,11 +90,16 @@ describe('AzureApiReaderProcessor', () => { headers: {}, }, }, + { + token: undefined, + expect: { + headers: {}, + }, + }, ]; for (const test of tests) { - process.env.AZURE_PRIVATE_TOKEN = test.token; - const processor = new AzureApiReaderProcessor(); + const processor = new AzureApiReaderProcessor(createConfig(test.token)); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts index 41b4336b4f..053d7138db 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class AzureApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + (config.getOptional( + 'backend.ingestionProcessors.azureApi.privateToken', + ) as string) ?? ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; @@ -53,7 +61,8 @@ export class AzureApiReaderProcessor implements LocationProcessor { const response = await fetch(url.toString(), this.getRequestOptions()); - if (response.ok) { + // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html + if (response.ok && response.status !== 203) { const data = await response.buffer(); emit(result.data(location, data)); } else { @@ -76,7 +85,6 @@ export class AzureApiReaderProcessor implements LocationProcessor { // Converts // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 - buildRawUrl(target: string): URL { try { const url = new URL(target); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts index fa4731a40e..7adc9c271c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts @@ -15,10 +15,33 @@ */ import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('BitbucketApiReaderProcessor', () => { + const createConfig = ( + userName: string | undefined, + appPassword: string | undefined, + ) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + backend: { + ingestionProcessors: { + bitbucketApi: { + userName: userName, + appPassword: appPassword, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new BitbucketApiReaderProcessor(); + const processor = new BitbucketApiReaderProcessor( + createConfig(undefined, undefined), + ); const tests = [ { @@ -90,12 +113,33 @@ describe('BitbucketApiReaderProcessor', () => { }, }, }, + { + username: undefined, + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: 'only-user-provided', + password: undefined, + expect: { + headers: {}, + }, + }, + { + username: undefined, + password: 'only-password-provided', + expect: { + headers: {}, + }, + }, ]; for (const test of tests) { - process.env.BITBUCKET_USERNAME = test.username; - process.env.BITBUCKET_APP_PASSWORD = test.password; - const processor = new BitbucketApiReaderProcessor(); + const processor = new BitbucketApiReaderProcessor( + createConfig(test.username, test.password), + ); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts index 8162e505d6..5c11fa83a6 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts @@ -18,10 +18,22 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class BitbucketApiReaderProcessor implements LocationProcessor { - private username: string = process.env.BITBUCKET_USERNAME || ''; - private password: string = process.env.BITBUCKET_APP_PASSWORD || ''; + private username: string; + private password: string; + + constructor(config: Config) { + this.username = + (config.getOptional( + 'backend.ingestionProcessors.bitbucketApi.userName', + ) as string) ?? ''; + this.password = + (config.getOptional( + 'backend.ingestionProcessors.bitbucketApi.appPassword', + ) as string) ?? ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = {}; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index 63ddb071c3..bdec8f1cdf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('GithubApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + backend: { + ingestionProcessors: { + githubApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new GithubApiReaderProcessor(); + const processor = new GithubApiReaderProcessor(createConfig(undefined)); const tests = [ { @@ -87,8 +104,7 @@ describe('GithubApiReaderProcessor', () => { ]; for (const test of tests) { - process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubApiReaderProcessor(); + const processor = new GithubApiReaderProcessor(createConfig(test.token)); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts index ff61d004ca..40f8714655 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class GithubApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + (config.getOptional( + 'backend.ingestionProcessors.githubApi.privateToken', + ) as string) ?? ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = { diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts index c43c68591e..bcc7c0017c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts @@ -15,10 +15,27 @@ */ import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; +import { ConfigReader } from '@backstage/config'; describe('GitlabApiReaderProcessor', () => { + const createConfig = (token: string | undefined) => + ConfigReader.fromConfigs([ + { + context: '', + data: { + backend: { + ingestionProcessors: { + gitlabApi: { + privateToken: token, + }, + }, + }, + }, + }, + ]); + it('should build raw api', () => { - const processor = new GitlabApiReaderProcessor(); + const processor = new GitlabApiReaderProcessor(createConfig(undefined)); const tests = [ { @@ -89,11 +106,18 @@ describe('GitlabApiReaderProcessor', () => { }, }, }, + { + token: undefined, + expect: { + headers: { + 'PRIVATE-TOKEN': '', + }, + }, + }, ]; for (const test of tests) { - process.env.GITLAB_PRIVATE_TOKEN = test.token; - const processor = new GitlabApiReaderProcessor(); + const processor = new GitlabApiReaderProcessor(createConfig(test.token)); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts index 3e585bf2b3..910a387643 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +import { Config } from '@backstage/config'; export class GitlabApiReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || ''; + private privateToken: string; + + constructor(config: Config) { + this.privateToken = + (config.getOptional( + 'backend.ingestionProcessors.gitlabApi.privateToken', + ) as string) ?? ''; + } getRequestOptions(): RequestInit { const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index e4fc90699b..2b465a536e 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -37,6 +37,7 @@ describe('ComponentIdValidators', () => { [true, 'http://example.com/blob/master/service.yaml'], [true, 'https://example.yaml'], [true, 'https://example.com?path=abc.yaml&c=1'], + [errorMessage, 'https://example.com?path=abc_yaml&c=1'], [errorMessage, '.yml'], [errorMessage, 'http://example.com/blob/master/service'], [errorMessage, undefined],