From dd3963093de7a208c24f14f61cf109a13b156696 Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sat, 29 Aug 2020 15:34:09 +1200
Subject: [PATCH 1/6] Update ingestion processors to use config system

---
 app-config.yaml                               | 20 +++++++
 .../src/ingestion/LocationReaders.ts          |  8 +--
 .../AzureApiReaderProcessor.test.ts           | 28 ++++++++--
 .../processors/AzureApiReaderProcessor.ts     | 14 +++--
 .../BitbucketApiReaderProcessor.test.ts       | 52 +++++++++++++++++--
 .../processors/BitbucketApiReaderProcessor.ts | 16 +++++-
 .../GithubApiReaderProcessor.test.ts          | 22 ++++++--
 .../processors/GithubApiReaderProcessor.ts    | 10 +++-
 .../GitlabApiReaderProcessor.test.ts          | 30 +++++++++--
 .../processors/GitlabApiReaderProcessor.ts    | 10 +++-
 .../src/util/validate.test.ts                 |  1 +
 11 files changed, 187 insertions(+), 24 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 2efe6adc32..880e8d0bd9 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -13,6 +13,26 @@ backend:
   database:
     client: sqlite3
     connection: ':memory:'
+  ingestionProcessors:
+    githubApi:
+      privateToken:
+        $secret:
+          env: GITHUB_PRIVATE_TOKEN
+    bitbucketApi:
+      userName:
+        $secret:
+          env: BITBUCKET_USERNAME
+      appPassword:
+        $secret:
+          env: BITBUCKET_APP_PASSWORD
+    gitlabApi:
+      privateToken:
+        $secret:
+          env: GITLAB_PRIVATE_TOKEN
+    azureApi:
+      privateToken:
+        $secret:
+          env: AZURE_PRIVATE_TOKEN
 
 # See README.md in the proxy-backend plugin for information on the configuration format
 proxy:
diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts
index 29892bb2dc..46fa6781f1 100644
--- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts
+++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts
@@ -76,11 +76,11 @@ export class LocationReaders implements LocationReader {
       StaticLocationProcessor.fromConfig(config),
       new FileReaderProcessor(),
       new GithubReaderProcessor(),
-      new GithubApiReaderProcessor(),
-      new GitlabApiReaderProcessor(),
+      new GithubApiReaderProcessor(config),
+      new GitlabApiReaderProcessor(config),
       new GitlabReaderProcessor(),
-      new BitbucketApiReaderProcessor(),
-      new AzureApiReaderProcessor(),
+      new BitbucketApiReaderProcessor(config),
+      new AzureApiReaderProcessor(config),
       new UrlReaderProcessor(),
       new YamlProcessor(),
       new EntityPolicyProcessor(entityPolicy),
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
index 0ef111fcb1..6ccb461721 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
@@ -15,10 +15,27 @@
  */
 
 import { AzureApiReaderProcessor } from './AzureApiReaderProcessor';
+import { ConfigReader } from '@backstage/config';
 
 describe('AzureApiReaderProcessor', () => {
+  const createConfig = (token: string | undefined) =>
+    ConfigReader.fromConfigs([
+      {
+        context: '',
+        data: {
+          backend: {
+            ingestionProcessors: {
+              azureApi: {
+                privateToken: token,
+              },
+            },
+          },
+        },
+      },
+    ]);
+
   it('should build raw api', () => {
-    const processor = new AzureApiReaderProcessor();
+    const processor = new AzureApiReaderProcessor(createConfig(undefined));
     const tests = [
       {
         target:
@@ -73,11 +90,16 @@ describe('AzureApiReaderProcessor', () => {
           headers: {},
         },
       },
+      {
+        token: undefined,
+        expect: {
+          headers: {},
+        },
+      },
     ];
 
     for (const test of tests) {
-      process.env.AZURE_PRIVATE_TOKEN = test.token;
-      const processor = new AzureApiReaderProcessor();
+      const processor = new AzureApiReaderProcessor(createConfig(test.token));
       expect(processor.getRequestOptions()).toEqual(test.expect);
     }
   });
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
index 41b4336b4f..053d7138db 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
 import fetch, { RequestInit, HeadersInit } from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
+import { Config } from '@backstage/config';
 
 export class AzureApiReaderProcessor implements LocationProcessor {
-  private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || '';
+  private privateToken: string;
+
+  constructor(config: Config) {
+    this.privateToken =
+      (config.getOptional(
+        'backend.ingestionProcessors.azureApi.privateToken',
+      ) as string) ?? '';
+  }
 
   getRequestOptions(): RequestInit {
     const headers: HeadersInit = {};
@@ -53,7 +61,8 @@ export class AzureApiReaderProcessor implements LocationProcessor {
 
       const response = await fetch(url.toString(), this.getRequestOptions());
 
-      if (response.ok) {
+      // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
+      if (response.ok && response.status !== 203) {
         const data = await response.buffer();
         emit(result.data(location, data));
       } else {
@@ -76,7 +85,6 @@ export class AzureApiReaderProcessor implements LocationProcessor {
   // Converts
   // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
   // to:   https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1
-
   buildRawUrl(target: string): URL {
     try {
       const url = new URL(target);
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
index fa4731a40e..7adc9c271c 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
@@ -15,10 +15,33 @@
  */
 
 import { BitbucketApiReaderProcessor } from './BitbucketApiReaderProcessor';
+import { ConfigReader } from '@backstage/config';
 
 describe('BitbucketApiReaderProcessor', () => {
+  const createConfig = (
+    userName: string | undefined,
+    appPassword: string | undefined,
+  ) =>
+    ConfigReader.fromConfigs([
+      {
+        context: '',
+        data: {
+          backend: {
+            ingestionProcessors: {
+              bitbucketApi: {
+                userName: userName,
+                appPassword: appPassword,
+              },
+            },
+          },
+        },
+      },
+    ]);
+
   it('should build raw api', () => {
-    const processor = new BitbucketApiReaderProcessor();
+    const processor = new BitbucketApiReaderProcessor(
+      createConfig(undefined, undefined),
+    );
 
     const tests = [
       {
@@ -90,12 +113,33 @@ describe('BitbucketApiReaderProcessor', () => {
           },
         },
       },
+      {
+        username: undefined,
+        password: undefined,
+        expect: {
+          headers: {},
+        },
+      },
+      {
+        username: 'only-user-provided',
+        password: undefined,
+        expect: {
+          headers: {},
+        },
+      },
+      {
+        username: undefined,
+        password: 'only-password-provided',
+        expect: {
+          headers: {},
+        },
+      },
     ];
 
     for (const test of tests) {
-      process.env.BITBUCKET_USERNAME = test.username;
-      process.env.BITBUCKET_APP_PASSWORD = test.password;
-      const processor = new BitbucketApiReaderProcessor();
+      const processor = new BitbucketApiReaderProcessor(
+        createConfig(test.username, test.password),
+      );
       expect(processor.getRequestOptions()).toEqual(test.expect);
     }
   });
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
index 8162e505d6..5c11fa83a6 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
@@ -18,10 +18,22 @@ import { LocationSpec } from '@backstage/catalog-model';
 import fetch, { RequestInit, HeadersInit } from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
+import { Config } from '@backstage/config';
 
 export class BitbucketApiReaderProcessor implements LocationProcessor {
-  private username: string = process.env.BITBUCKET_USERNAME || '';
-  private password: string = process.env.BITBUCKET_APP_PASSWORD || '';
+  private username: string;
+  private password: string;
+
+  constructor(config: Config) {
+    this.username =
+      (config.getOptional(
+        'backend.ingestionProcessors.bitbucketApi.userName',
+      ) as string) ?? '';
+    this.password =
+      (config.getOptional(
+        'backend.ingestionProcessors.bitbucketApi.appPassword',
+      ) as string) ?? '';
+  }
 
   getRequestOptions(): RequestInit {
     const headers: HeadersInit = {};
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
index 63ddb071c3..bdec8f1cdf 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
@@ -15,10 +15,27 @@
  */
 
 import { GithubApiReaderProcessor } from './GithubApiReaderProcessor';
+import { ConfigReader } from '@backstage/config';
 
 describe('GithubApiReaderProcessor', () => {
+  const createConfig = (token: string | undefined) =>
+    ConfigReader.fromConfigs([
+      {
+        context: '',
+        data: {
+          backend: {
+            ingestionProcessors: {
+              githubApi: {
+                privateToken: token,
+              },
+            },
+          },
+        },
+      },
+    ]);
+
   it('should build raw api', () => {
-    const processor = new GithubApiReaderProcessor();
+    const processor = new GithubApiReaderProcessor(createConfig(undefined));
 
     const tests = [
       {
@@ -87,8 +104,7 @@ describe('GithubApiReaderProcessor', () => {
     ];
 
     for (const test of tests) {
-      process.env.GITHUB_PRIVATE_TOKEN = test.token;
-      const processor = new GithubApiReaderProcessor();
+      const processor = new GithubApiReaderProcessor(createConfig(test.token));
       expect(processor.getRequestOptions()).toEqual(test.expect);
     }
   });
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
index ff61d004ca..40f8714655 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
 import fetch, { RequestInit, HeadersInit } from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
+import { Config } from '@backstage/config';
 
 export class GithubApiReaderProcessor implements LocationProcessor {
-  private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || '';
+  private privateToken: string;
+
+  constructor(config: Config) {
+    this.privateToken =
+      (config.getOptional(
+        'backend.ingestionProcessors.githubApi.privateToken',
+      ) as string) ?? '';
+  }
 
   getRequestOptions(): RequestInit {
     const headers: HeadersInit = {
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
index c43c68591e..bcc7c0017c 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
@@ -15,10 +15,27 @@
  */
 
 import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor';
+import { ConfigReader } from '@backstage/config';
 
 describe('GitlabApiReaderProcessor', () => {
+  const createConfig = (token: string | undefined) =>
+    ConfigReader.fromConfigs([
+      {
+        context: '',
+        data: {
+          backend: {
+            ingestionProcessors: {
+              gitlabApi: {
+                privateToken: token,
+              },
+            },
+          },
+        },
+      },
+    ]);
+
   it('should build raw api', () => {
-    const processor = new GitlabApiReaderProcessor();
+    const processor = new GitlabApiReaderProcessor(createConfig(undefined));
 
     const tests = [
       {
@@ -89,11 +106,18 @@ describe('GitlabApiReaderProcessor', () => {
           },
         },
       },
+      {
+        token: undefined,
+        expect: {
+          headers: {
+            'PRIVATE-TOKEN': '',
+          },
+        },
+      },
     ];
 
     for (const test of tests) {
-      process.env.GITLAB_PRIVATE_TOKEN = test.token;
-      const processor = new GitlabApiReaderProcessor();
+      const processor = new GitlabApiReaderProcessor(createConfig(test.token));
       expect(processor.getRequestOptions()).toEqual(test.expect);
     }
   });
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
index 3e585bf2b3..910a387643 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
@@ -18,9 +18,17 @@ import { LocationSpec } from '@backstage/catalog-model';
 import fetch, { RequestInit, HeadersInit } from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
+import { Config } from '@backstage/config';
 
 export class GitlabApiReaderProcessor implements LocationProcessor {
-  private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || '';
+  private privateToken: string;
+
+  constructor(config: Config) {
+    this.privateToken =
+      (config.getOptional(
+        'backend.ingestionProcessors.gitlabApi.privateToken',
+      ) as string) ?? '';
+  }
 
   getRequestOptions(): RequestInit {
     const headers: HeadersInit = { 'PRIVATE-TOKEN': '' };
diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts
index e4fc90699b..2b465a536e 100644
--- a/plugins/register-component/src/util/validate.test.ts
+++ b/plugins/register-component/src/util/validate.test.ts
@@ -37,6 +37,7 @@ describe('ComponentIdValidators', () => {
       [true, 'http://example.com/blob/master/service.yaml'],
       [true, 'https://example.yaml'],
       [true, 'https://example.com?path=abc.yaml&c=1'],
+      [errorMessage, 'https://example.com?path=abc_yaml&c=1'],
       [errorMessage, '.yml'],
       [errorMessage, 'http://example.com/blob/master/service'],
       [errorMessage, undefined],

From a711b6fab4ed8303ce6abaebe28bdca7c5d7ba35 Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sat, 29 Aug 2020 22:12:27 +1200
Subject: [PATCH 2/6] rename ingestionProcessors to processors

---
 app-config.yaml                                               | 2 +-
 .../src/ingestion/processors/AzureApiReaderProcessor.test.ts  | 2 +-
 .../src/ingestion/processors/AzureApiReaderProcessor.ts       | 2 +-
 .../ingestion/processors/BitbucketApiReaderProcessor.test.ts  | 2 +-
 .../src/ingestion/processors/BitbucketApiReaderProcessor.ts   | 4 ++--
 .../src/ingestion/processors/GithubApiReaderProcessor.test.ts | 2 +-
 .../src/ingestion/processors/GithubApiReaderProcessor.ts      | 2 +-
 .../src/ingestion/processors/GitlabApiReaderProcessor.test.ts | 2 +-
 .../src/ingestion/processors/GitlabApiReaderProcessor.ts      | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index 880e8d0bd9..a273e815c2 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -13,7 +13,7 @@ backend:
   database:
     client: sqlite3
     connection: ':memory:'
-  ingestionProcessors:
+  processors:
     githubApi:
       privateToken:
         $secret:
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
index 6ccb461721..9e2b265b93 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
@@ -24,7 +24,7 @@ describe('AzureApiReaderProcessor', () => {
         context: '',
         data: {
           backend: {
-            ingestionProcessors: {
+            processors: {
               azureApi: {
                 privateToken: token,
               },
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
index 053d7138db..94be3a9650 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class AzureApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.ingestionProcessors.azureApi.privateToken',
+        'backend.processors.azureApi.privateToken',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
index 7adc9c271c..ddc00c23fc 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
@@ -27,7 +27,7 @@ describe('BitbucketApiReaderProcessor', () => {
         context: '',
         data: {
           backend: {
-            ingestionProcessors: {
+            processors: {
               bitbucketApi: {
                 userName: userName,
                 appPassword: appPassword,
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
index 5c11fa83a6..90a6689adb 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
@@ -27,11 +27,11 @@ export class BitbucketApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.username =
       (config.getOptional(
-        'backend.ingestionProcessors.bitbucketApi.userName',
+        'backend.processors.bitbucketApi.userName',
       ) as string) ?? '';
     this.password =
       (config.getOptional(
-        'backend.ingestionProcessors.bitbucketApi.appPassword',
+        'backend.processors.bitbucketApi.appPassword',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
index bdec8f1cdf..501562fe58 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
@@ -24,7 +24,7 @@ describe('GithubApiReaderProcessor', () => {
         context: '',
         data: {
           backend: {
-            ingestionProcessors: {
+            processors: {
               githubApi: {
                 privateToken: token,
               },
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
index 40f8714655..ad549837d6 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class GithubApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.ingestionProcessors.githubApi.privateToken',
+        'backend.processors.githubApi.privateToken',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
index bcc7c0017c..2deae1dcb0 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
@@ -24,7 +24,7 @@ describe('GitlabApiReaderProcessor', () => {
         context: '',
         data: {
           backend: {
-            ingestionProcessors: {
+            processors: {
               gitlabApi: {
                 privateToken: token,
               },
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
index 910a387643..c807389440 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class GitlabApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.ingestionProcessors.gitlabApi.privateToken',
+        'backend.processors.gitlabApi.privateToken',
       ) as string) ?? '';
   }
 

From 36376778ecf6313b9765d4a175ed89b1b23777aa Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sat, 29 Aug 2020 22:22:19 +1200
Subject: [PATCH 3/6] move processor config to catalog from backend

---
 app-config.yaml                               | 40 +++++++++----------
 .../AzureApiReaderProcessor.test.ts           |  2 +-
 .../processors/AzureApiReaderProcessor.ts     |  2 +-
 .../BitbucketApiReaderProcessor.test.ts       |  2 +-
 .../processors/BitbucketApiReaderProcessor.ts |  4 +-
 .../GithubApiReaderProcessor.test.ts          |  2 +-
 .../processors/GithubApiReaderProcessor.ts    |  2 +-
 .../GitlabApiReaderProcessor.test.ts          |  2 +-
 .../processors/GitlabApiReaderProcessor.ts    |  2 +-
 9 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/app-config.yaml b/app-config.yaml
index a273e815c2..aac8e233f0 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -13,26 +13,6 @@ backend:
   database:
     client: sqlite3
     connection: ':memory:'
-  processors:
-    githubApi:
-      privateToken:
-        $secret:
-          env: GITHUB_PRIVATE_TOKEN
-    bitbucketApi:
-      userName:
-        $secret:
-          env: BITBUCKET_USERNAME
-      appPassword:
-        $secret:
-          env: BITBUCKET_APP_PASSWORD
-    gitlabApi:
-      privateToken:
-        $secret:
-          env: GITLAB_PRIVATE_TOKEN
-    azureApi:
-      privateToken:
-        $secret:
-          env: AZURE_PRIVATE_TOKEN
 
 # See README.md in the proxy-backend plugin for information on the configuration format
 proxy:
@@ -65,6 +45,26 @@ newrelic:
     key: NEW_RELIC_REST_API_KEY
 
 catalog:
+  processors:
+    githubApi:
+      privateToken:
+        $secret:
+          env: GITHUB_PRIVATE_TOKEN
+    bitbucketApi:
+      userName:
+        $secret:
+          env: BITBUCKET_USERNAME
+      appPassword:
+        $secret:
+          env: BITBUCKET_APP_PASSWORD
+    gitlabApi:
+      privateToken:
+        $secret:
+          env: GITLAB_PRIVATE_TOKEN
+    azureApi:
+      privateToken:
+        $secret:
+          env: AZURE_PRIVATE_TOKEN
   exampleEntityLocations:
     github:
       - https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/artist-lookup-component.yaml
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
index 9e2b265b93..182343133e 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
@@ -23,7 +23,7 @@ describe('AzureApiReaderProcessor', () => {
       {
         context: '',
         data: {
-          backend: {
+          catalog: {
             processors: {
               azureApi: {
                 privateToken: token,
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
index 94be3a9650..02b0981a27 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class AzureApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.processors.azureApi.privateToken',
+        'catalog.processors.azureApi.privateToken',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
index ddc00c23fc..6c82f73e97 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
@@ -26,7 +26,7 @@ describe('BitbucketApiReaderProcessor', () => {
       {
         context: '',
         data: {
-          backend: {
+          catalog: {
             processors: {
               bitbucketApi: {
                 userName: userName,
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
index 90a6689adb..5f5b5750ee 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
@@ -27,11 +27,11 @@ export class BitbucketApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.username =
       (config.getOptional(
-        'backend.processors.bitbucketApi.userName',
+        'catalog.processors.bitbucketApi.userName',
       ) as string) ?? '';
     this.password =
       (config.getOptional(
-        'backend.processors.bitbucketApi.appPassword',
+        'catalog.processors.bitbucketApi.appPassword',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
index 501562fe58..31537954f5 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
@@ -23,7 +23,7 @@ describe('GithubApiReaderProcessor', () => {
       {
         context: '',
         data: {
-          backend: {
+          catalog: {
             processors: {
               githubApi: {
                 privateToken: token,
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
index ad549837d6..89e6c97ab1 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class GithubApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.processors.githubApi.privateToken',
+        'catalog.processors.githubApi.privateToken',
       ) as string) ?? '';
   }
 
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
index 2deae1dcb0..a239afee04 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
@@ -23,7 +23,7 @@ describe('GitlabApiReaderProcessor', () => {
       {
         context: '',
         data: {
-          backend: {
+          catalog: {
             processors: {
               gitlabApi: {
                 privateToken: token,
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
index c807389440..8915d73bb9 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
@@ -26,7 +26,7 @@ export class GitlabApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.privateToken =
       (config.getOptional(
-        'backend.processors.gitlabApi.privateToken',
+        'catalog.processors.gitlabApi.privateToken',
       ) as string) ?? '';
   }
 

From 4b85a3ac2173b628ceec47859eb0290844c6f786 Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sat, 29 Aug 2020 23:08:36 +1200
Subject: [PATCH 4/6] update casing

---
 .../ingestion/processors/BitbucketApiReaderProcessor.test.ts  | 4 ++--
 .../src/ingestion/processors/BitbucketApiReaderProcessor.ts   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
index 6c82f73e97..7d2a2c57de 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
@@ -19,7 +19,7 @@ import { ConfigReader } from '@backstage/config';
 
 describe('BitbucketApiReaderProcessor', () => {
   const createConfig = (
-    userName: string | undefined,
+    username: string | undefined,
     appPassword: string | undefined,
   ) =>
     ConfigReader.fromConfigs([
@@ -29,7 +29,7 @@ describe('BitbucketApiReaderProcessor', () => {
           catalog: {
             processors: {
               bitbucketApi: {
-                userName: userName,
+                username: username,
                 appPassword: appPassword,
               },
             },
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
index 5f5b5750ee..3e19d3cf78 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
@@ -27,7 +27,7 @@ export class BitbucketApiReaderProcessor implements LocationProcessor {
   constructor(config: Config) {
     this.username =
       (config.getOptional(
-        'catalog.processors.bitbucketApi.userName',
+        'catalog.processors.bitbucketApi.username',
       ) as string) ?? '';
     this.password =
       (config.getOptional(

From 13623d9185b0a6613df4214dbf8eed31114ee504 Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sun, 30 Aug 2020 19:36:28 +1200
Subject: [PATCH 5/6] change getOptional to getOptionalString

---
 .../AzureApiReaderProcessor.test.ts           | 12 ++++++++--
 .../processors/AzureApiReaderProcessor.ts     |  5 ++--
 .../BitbucketApiReaderProcessor.test.ts       | 23 +++++++++++++++----
 .../processors/BitbucketApiReaderProcessor.ts | 10 ++++----
 .../GithubApiReaderProcessor.test.ts          | 22 ++++++++++++++++--
 .../processors/GithubApiReaderProcessor.ts    |  5 ++--
 .../GitlabApiReaderProcessor.test.ts          | 14 +++++++++--
 .../processors/GitlabApiReaderProcessor.ts    |  5 ++--
 8 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
index 182343133e..62259f87a2 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts
@@ -89,6 +89,8 @@ describe('AzureApiReaderProcessor', () => {
         expect: {
           headers: {},
         },
+        err:
+          "Invalid type in config for key 'catalog.processors.azureApi.privateToken' in '', got empty-string, wanted string",
       },
       {
         token: undefined,
@@ -99,8 +101,14 @@ describe('AzureApiReaderProcessor', () => {
     ];
 
     for (const test of tests) {
-      const processor = new AzureApiReaderProcessor(createConfig(test.token));
-      expect(processor.getRequestOptions()).toEqual(test.expect);
+      if (test.err) {
+        expect(
+          () => new AzureApiReaderProcessor(createConfig(test.token)),
+        ).toThrowError(test.err);
+      } else {
+        const processor = new AzureApiReaderProcessor(createConfig(test.token));
+        expect(processor.getRequestOptions()).toEqual(test.expect);
+      }
     }
   });
 });
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
index 02b0981a27..7033521da6 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
@@ -25,9 +25,8 @@ export class AzureApiReaderProcessor implements LocationProcessor {
 
   constructor(config: Config) {
     this.privateToken =
-      (config.getOptional(
-        'catalog.processors.azureApi.privateToken',
-      ) as string) ?? '';
+      config.getOptionalString('catalog.processors.azureApi.privateToken') ??
+      '';
   }
 
   getRequestOptions(): RequestInit {
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
index 7d2a2c57de..953f0703be 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.test.ts
@@ -89,6 +89,8 @@ describe('BitbucketApiReaderProcessor', () => {
         expect: {
           headers: {},
         },
+        err:
+          "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string",
       },
       {
         username: 'only-user-provided',
@@ -96,6 +98,8 @@ describe('BitbucketApiReaderProcessor', () => {
         expect: {
           headers: {},
         },
+        err:
+          "Invalid type in config for key 'catalog.processors.bitbucketApi.appPassword' in '', got empty-string, wanted string",
       },
       {
         username: '',
@@ -103,6 +107,8 @@ describe('BitbucketApiReaderProcessor', () => {
         expect: {
           headers: {},
         },
+        err:
+          "Invalid type in config for key 'catalog.processors.bitbucketApi.username' in '', got empty-string, wanted string",
       },
       {
         username: 'some-user',
@@ -137,10 +143,19 @@ describe('BitbucketApiReaderProcessor', () => {
     ];
 
     for (const test of tests) {
-      const processor = new BitbucketApiReaderProcessor(
-        createConfig(test.username, test.password),
-      );
-      expect(processor.getRequestOptions()).toEqual(test.expect);
+      if (test.err) {
+        expect(
+          () =>
+            new BitbucketApiReaderProcessor(
+              createConfig(test.username, test.password),
+            ),
+        ).toThrowError(test.err);
+      } else {
+        const processor = new BitbucketApiReaderProcessor(
+          createConfig(test.username, test.password),
+        );
+        expect(processor.getRequestOptions()).toEqual(test.expect);
+      }
     }
   });
 });
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
index 3e19d3cf78..97ecf0cd1f 100644
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
@@ -26,13 +26,11 @@ export class BitbucketApiReaderProcessor implements LocationProcessor {
 
   constructor(config: Config) {
     this.username =
-      (config.getOptional(
-        'catalog.processors.bitbucketApi.username',
-      ) as string) ?? '';
+      config.getOptionalString('catalog.processors.bitbucketApi.username') ??
+      '';
     this.password =
-      (config.getOptional(
-        'catalog.processors.bitbucketApi.appPassword',
-      ) as string) ?? '';
+      config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ??
+      '';
   }
 
   getRequestOptions(): RequestInit {
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
index 31537954f5..51c3cf4cd0 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts
@@ -95,6 +95,16 @@ describe('GithubApiReaderProcessor', () => {
       },
       {
         token: '',
+        err:
+          "Invalid type in config for key 'catalog.processors.githubApi.privateToken' in '', got empty-string, wanted string",
+        expect: {
+          headers: {
+            Accept: 'application/vnd.github.v3.raw',
+          },
+        },
+      },
+      {
+        token: undefined,
         expect: {
           headers: {
             Accept: 'application/vnd.github.v3.raw',
@@ -104,8 +114,16 @@ describe('GithubApiReaderProcessor', () => {
     ];
 
     for (const test of tests) {
-      const processor = new GithubApiReaderProcessor(createConfig(test.token));
-      expect(processor.getRequestOptions()).toEqual(test.expect);
+      if (test.err) {
+        expect(
+          () => new GithubApiReaderProcessor(createConfig(test.token)),
+        ).toThrowError(test.err);
+      } else {
+        const processor = new GithubApiReaderProcessor(
+          createConfig(test.token),
+        );
+        expect(processor.getRequestOptions()).toEqual(test.expect);
+      }
     }
   });
 });
diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
index 89e6c97ab1..f8d1d6caf8 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts
@@ -25,9 +25,8 @@ export class GithubApiReaderProcessor implements LocationProcessor {
 
   constructor(config: Config) {
     this.privateToken =
-      (config.getOptional(
-        'catalog.processors.githubApi.privateToken',
-      ) as string) ?? '';
+      config.getOptionalString('catalog.processors.githubApi.privateToken') ??
+      '';
   }
 
   getRequestOptions(): RequestInit {
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
index a239afee04..48ed270049 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts
@@ -100,6 +100,8 @@ describe('GitlabApiReaderProcessor', () => {
       },
       {
         token: '',
+        err:
+          "Invalid type in config for key 'catalog.processors.gitlabApi.privateToken' in '', got empty-string, wanted string",
         expect: {
           headers: {
             'PRIVATE-TOKEN': '',
@@ -117,8 +119,16 @@ describe('GitlabApiReaderProcessor', () => {
     ];
 
     for (const test of tests) {
-      const processor = new GitlabApiReaderProcessor(createConfig(test.token));
-      expect(processor.getRequestOptions()).toEqual(test.expect);
+      if (test.err) {
+        expect(
+          () => new GitlabApiReaderProcessor(createConfig(test.token)),
+        ).toThrowError(test.err);
+      } else {
+        const processor = new GitlabApiReaderProcessor(
+          createConfig(test.token),
+        );
+        expect(processor.getRequestOptions()).toEqual(test.expect);
+      }
     }
   });
 });
diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
index 8915d73bb9..067edba3d9 100644
--- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts
@@ -25,9 +25,8 @@ export class GitlabApiReaderProcessor implements LocationProcessor {
 
   constructor(config: Config) {
     this.privateToken =
-      (config.getOptional(
-        'catalog.processors.gitlabApi.privateToken',
-      ) as string) ?? '';
+      config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
+      '';
   }
 
   getRequestOptions(): RequestInit {

From 298a9881ddf68b224877a3a6cf5572fe6849deb8 Mon Sep 17 00:00:00 2001
From: Omer Farooq <omer.farooq@countdown.co.nz>
Date: Sun, 30 Aug 2020 19:43:43 +1200
Subject: [PATCH 6/6] update casing

---
 app-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app-config.yaml b/app-config.yaml
index aac8e233f0..5347fbe617 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -51,7 +51,7 @@ catalog:
         $secret:
           env: GITHUB_PRIVATE_TOKEN
     bitbucketApi:
-      userName:
+      username:
         $secret:
           env: BITBUCKET_USERNAME
       appPassword: