From bc9d62f4f7b452141b383ad00124ed9ba2c1c3e0 Mon Sep 17 00:00:00 2001
From: Dominik Henneke <dominik.henneke@sda-se.com>
Date: Thu, 8 Apr 2021 18:38:39 +0200
Subject: [PATCH 01/23] Create a new method to check the configuration of a
 techdocs publisher to not crash the application on errors

Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
---
 .changeset/fluffy-lies-complain.md            |  18 ++
 .../__mocks__/@google-cloud/storage.ts        |  10 +-
 packages/techdocs-common/__mocks__/aws-sdk.ts |  14 +-
 .../techdocs-common/__mocks__/pkgcloud.ts     |  15 +-
 .../src/stages/publish/awsS3.test.ts          |  41 ++-
 .../src/stages/publish/awsS3.ts               |  62 ++--
 .../stages/publish/azureBlobStorage.test.ts   | 264 +++++++-----------
 .../src/stages/publish/azureBlobStorage.ts    |  63 +++--
 .../src/stages/publish/googleStorage.test.ts  |  31 +-
 .../src/stages/publish/googleStorage.ts       |  54 ++--
 .../src/stages/publish/local.ts               |   7 +
 .../src/stages/publish/openStackSwift.test.ts |  45 ++-
 .../src/stages/publish/openStackSwift.ts      |  59 ++--
 .../src/stages/publish/publish.ts             |  14 +-
 .../src/stages/publish/types.ts               |  16 ++
 15 files changed, 423 insertions(+), 290 deletions(-)
 create mode 100644 .changeset/fluffy-lies-complain.md

diff --git a/.changeset/fluffy-lies-complain.md b/.changeset/fluffy-lies-complain.md
new file mode 100644
index 0000000000..addffc2baa
--- /dev/null
+++ b/.changeset/fluffy-lies-complain.md
@@ -0,0 +1,18 @@
+---
+'@backstage/techdocs-common': patch
+---
+
+Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#validateConfiguration()` method instead of throwing an error when doing `Publisher.fromConfig(...)`.
+If you want to preserve this check in your application, use the following code:
+
+```ts
+const publisher = await Publisher.fromConfig(config, {
+  logger,
+  discovery,
+});
+
+const validation = await publisher.validateConfiguration();
+if (!validation.isValid) {
+  throw new Error('Invalid TechDocs publisher configuration');
+}
+```
diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts
index 264852d31c..f8d0368f27 100644
--- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts
+++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts
@@ -81,10 +81,12 @@ class Bucket {
     this.bucketName = bucketName;
   }
 
-  getMetadata() {
-    return new Promise(resolve => {
-      resolve('');
-    });
+  async getMetadata() {
+    if (this.bucketName === 'errorBucket') {
+      throw Error('Bucket does not exist');
+    }
+
+    return '';
   }
 
   upload(source: string, { destination }) {
diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts
index 363662c63c..eb15a218ab 100644
--- a/packages/techdocs-common/__mocks__/aws-sdk.ts
+++ b/packages/techdocs-common/__mocks__/aws-sdk.ts
@@ -80,10 +80,16 @@ export class S3 {
     };
   }
 
-  headBucket() {
-    return new Promise(resolve => {
-      resolve('');
-    });
+  headBucket({ Bucket }) {
+    return {
+      promise: async () => {
+        if (Bucket === 'errorBucket') {
+          throw new Error('Bucket does not exist');
+        }
+
+        return {};
+      },
+    };
   }
 
   upload({ Key }: { Key: string }) {
diff --git a/packages/techdocs-common/__mocks__/pkgcloud.ts b/packages/techdocs-common/__mocks__/pkgcloud.ts
index 5d9f81cda0..dca86238e4 100644
--- a/packages/techdocs-common/__mocks__/pkgcloud.ts
+++ b/packages/techdocs-common/__mocks__/pkgcloud.ts
@@ -13,10 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { EventEmitter } from 'events';
 import fs from 'fs-extra';
 import os from 'os';
 import path from 'path';
-import { EventEmitter } from 'events';
+import { ClientError } from 'pkgcloud';
 
 const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
 
@@ -37,12 +38,11 @@ class PkgCloudStorageClient {
   getFile(
     containerName: string,
     file: string,
-    callback: (err: any, file: string) => any,
+    callback: (err: any, file: any) => any,
   ) {
     checkFileExists(file).then(res => {
       if (!res) {
-        callback('File does not exist', file);
-        throw new Error('File does not exist');
+        callback('File does not exist', undefined);
       } else {
         callback(undefined, 'success');
       }
@@ -51,13 +51,12 @@ class PkgCloudStorageClient {
 
   getContainer(
     containerName: string,
-    callback: (err: string, container: string) => any,
+    callback: (err: ClientError, container: any) => any,
   ) {
     if (containerName !== 'mock') {
-      callback('Container does not exist', containerName);
-      throw new Error('Container does not exist');
+      callback(new Error('Container does not exist'), undefined);
     } else {
-      callback('Container does not exist', 'success');
+      callback(undefined, 'success');
     }
   }
 
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
index d3281c2645..b0000b6464 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
@@ -13,16 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { getVoidLogger } from '@backstage/backend-common';
 import {
   Entity,
-  EntityName,
   ENTITY_DEFAULT_NAMESPACE,
+  EntityName,
 } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import mockFs from 'mock-fs';
 import os from 'os';
 import path from 'path';
-import * as winston from 'winston';
 import { AwsS3Publish } from './awsS3';
 import { PublisherBase, TechDocsMetadata } from './types';
 
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
   return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
 };
 
-const logger = winston.createLogger();
-jest.spyOn(logger, 'info').mockReturnValue(logger);
-jest.spyOn(logger, 'error').mockReturnValue(logger);
+const logger = getVoidLogger();
 
 let publisher: PublisherBase;
 
@@ -87,6 +85,39 @@ beforeEach(() => {
 });
 
 describe('AwsS3Publish', () => {
+  describe('validateConfiguration', () => {
+    it('should validate correct config', async () => {
+      expect(await publisher.validateConfiguration()).toEqual({
+        isValid: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7000',
+          publisher: {
+            type: 'awsS3',
+            awsS3: {
+              credentials: {
+                accessKeyId: 'accessKeyId',
+                secretAccessKey: 'secretAccessKey',
+              },
+              // this bucket name will throw an error
+              bucketName: 'errorBucket',
+            },
+          },
+        },
+      });
+
+      const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger);
+
+      expect(await errorPublisher.validateConfiguration()).toEqual({
+        isValid: false,
+      });
+    });
+  });
+
   describe('publish', () => {
     beforeEach(() => {
       const entity = createMockEntity();
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts
index c4c39898f4..77bb771fb6 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.ts
@@ -17,16 +17,21 @@ import { Entity, EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
 import aws, { Credentials } from 'aws-sdk';
 import { ManagedUpload } from 'aws-sdk/clients/s3';
+import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import express from 'express';
 import fs from 'fs-extra';
 import JSON5 from 'json5';
 import createLimiter from 'p-limit';
-import { CredentialsOptions } from 'aws-sdk/lib/credentials';
 import path from 'path';
 import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
-import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import {
+  ConfigurationValidationResponse,
+  PublisherBase,
+  PublishRequest,
+  TechDocsMetadata,
+} from './types';
 
 const streamToBuffer = (stream: Readable): Promise<Buffer> => {
   return new Promise((resolve, reject) => {
@@ -81,30 +86,6 @@ export class AwsS3Publish implements PublisherBase {
       ...(endpoint && { endpoint }),
     });
 
-    // Check if the defined bucket exists. Being able to connect means the configuration is good
-    // and the storage client will work.
-    storageClient.headBucket(
-      {
-        Bucket: bucketName,
-      },
-      err => {
-        if (err) {
-          logger.error(
-            `Could not retrieve metadata about the AWS S3 bucket ${bucketName}. ` +
-              'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
-              'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
-              'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
-          );
-          logger.error(`from AWS client library: ${err.message}`);
-          throw new Error();
-        } else {
-          logger.info(
-            `Successfully connected to the AWS S3 bucket ${bucketName}.`,
-          );
-        }
-      },
-    );
-
     return new AwsS3Publish(storageClient, bucketName, logger);
   }
 
@@ -149,6 +130,35 @@ export class AwsS3Publish implements PublisherBase {
     this.logger = logger;
   }
 
+  /**
+   * Check if the defined bucket exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+    try {
+      await this.storageClient
+        .headBucket({ Bucket: this.bucketName })
+        .promise();
+
+      this.logger.info(
+        `Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
+      );
+
+      return { isValid: true };
+    } catch (error) {
+      this.logger.error(
+        `Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` +
+          'Make sure the bucket exists. Also make sure that authentication is setup either by ' +
+          'explicitly defining credentials and region in techdocs.publisher.awsS3 in app config or ' +
+          'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+      );
+      this.logger.error(`from AWS client library`, error);
+      return {
+        isValid: false,
+      };
+    }
+  }
+
   /**
    * Upload all the files from the generated `directory` to the S3 bucket.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
index 9442fb0e37..b67e47c675 100644
--- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
@@ -16,8 +16,8 @@
 import { getVoidLogger } from '@backstage/backend-common';
 import {
   Entity,
-  EntityName,
   ENTITY_DEFAULT_NAMESPACE,
+  EntityName,
 } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import mockFs from 'mock-fs';
@@ -59,12 +59,9 @@ const getEntityRootDir = (entity: Entity) => {
   return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
 };
 
-function createLogger() {
-  const logger = getVoidLogger();
-  jest.spyOn(logger, 'info').mockReturnValue(logger);
-  jest.spyOn(logger, 'error').mockReturnValue(logger);
-  return logger;
-}
+const logger = getVoidLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
 
 let publisher: PublisherBase;
 beforeEach(async () => {
@@ -85,13 +82,51 @@ beforeEach(async () => {
     },
   });
 
-  publisher = await AzureBlobStoragePublish.fromConfig(
-    mockConfig,
-    createLogger(),
-  );
+  publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger);
 });
 
 describe('publishing with valid credentials', () => {
+  describe('validateConfiguration', () => {
+    it('should validate correct config', async () => {
+      expect(await publisher.validateConfiguration()).toEqual({
+        isValid: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7000',
+          publisher: {
+            type: 'azureBlobStorage',
+            azureBlobStorage: {
+              credentials: {
+                accountName: 'accountName',
+                accountKey: 'accountKey',
+              },
+              containerName: 'bad_container',
+            },
+          },
+        },
+      });
+
+      const errorPublisher = await AzureBlobStoragePublish.fromConfig(
+        mockConfig,
+        logger,
+      );
+
+      expect(await errorPublisher.validateConfiguration()).toEqual({
+        isValid: false,
+      });
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
+        ),
+      );
+    });
+  });
+
   describe('publish', () => {
     beforeEach(() => {
       const entity = createMockEntity();
@@ -151,6 +186,60 @@ describe('publishing with valid credentials', () => {
         });
       mockFs.restore();
     });
+
+    it('reports an error when bad account credentials', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7000',
+          publisher: {
+            type: 'azureBlobStorage',
+            azureBlobStorage: {
+              credentials: {
+                accountName: 'failupload',
+                accountKey: 'accountKey',
+              },
+              containerName: 'containerName',
+            },
+          },
+        },
+      });
+
+      publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
+
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+        },
+      });
+
+      let error;
+      try {
+        await publisher.publish({
+          entity,
+          directory: entityRootDir,
+        });
+      } catch (e) {
+        error = e;
+      }
+
+      expect(error.message).toContain(
+        `Unable to upload file(s) to Azure Blob Storage.`,
+      );
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining(
+          `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
+            entityRootDir,
+            'index.html',
+          )} with status code 500`,
+        ),
+      );
+
+      mockFs.restore();
+    });
   });
 
   describe('hasDocsBeenGenerated', () => {
@@ -243,156 +332,3 @@ describe('publishing with valid credentials', () => {
     });
   });
 });
-
-describe('error reporting', () => {
-  it('reports an error when unable to read container properties', async () => {
-    const mockConfig = new ConfigReader({
-      techdocs: {
-        requestUrl: 'http://localhost:7000',
-        publisher: {
-          type: 'azureBlobStorage',
-          azureBlobStorage: {
-            credentials: {
-              accountName: 'accountName',
-            },
-            containerName: 'bad_container',
-          },
-        },
-      },
-    });
-
-    const logger = createLogger();
-
-    let error;
-    try {
-      publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
-    } catch (e) {
-      error = e;
-    }
-
-    expect(error).toBeInstanceOf(Error);
-
-    expect(logger.error).toHaveBeenCalledWith(
-      expect.stringContaining(
-        `Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
-      ),
-    );
-  });
-
-  it('reports an error when bad account credentials', async () => {
-    const mockConfig = new ConfigReader({
-      techdocs: {
-        requestUrl: 'http://localhost:7000',
-        publisher: {
-          type: 'azureBlobStorage',
-          azureBlobStorage: {
-            credentials: {
-              accountName: 'failupload',
-              accountKey: 'accountKey',
-            },
-            containerName: 'containerName',
-          },
-        },
-      },
-    });
-
-    const logger = createLogger();
-
-    publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
-
-    const entity = createMockEntity();
-    const entityRootDir = getEntityRootDir(entity);
-
-    mockFs({
-      [entityRootDir]: {
-        'index.html': '',
-      },
-    });
-
-    let error;
-    try {
-      await publisher.publish({
-        entity,
-        directory: entityRootDir,
-      });
-    } catch (e) {
-      error = e;
-    }
-
-    expect(error.message).toContain(
-      `Unable to upload file(s) to Azure Blob Storage.`,
-    );
-
-    expect(logger.error).toHaveBeenCalledWith(
-      expect.stringContaining(
-        `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
-          entityRootDir,
-          'index.html',
-        )} with status code 500`,
-      ),
-    );
-
-    mockFs.restore();
-  });
-
-  describe('fetchTechDocsMetadata', () => {
-    it('should return tech docs metadata', async () => {
-      const entityNameMock = createMockEntityName();
-      const entity = createMockEntity();
-      const entityRootDir = getEntityRootDir(entity);
-
-      mockFs({
-        [entityRootDir]: {
-          'techdocs_metadata.json':
-            '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}',
-        },
-      });
-      const expectedMetadata: TechDocsMetadata = {
-        site_name: 'backstage',
-        site_description: 'site_content',
-        etag: 'etag',
-      };
-      expect(
-        await publisher.fetchTechDocsMetadata(entityNameMock),
-      ).toStrictEqual(expectedMetadata);
-      mockFs.restore();
-    });
-
-    it('should return tech docs metadata when json encoded with single quotes', async () => {
-      const entityNameMock = createMockEntityName();
-      const entity = createMockEntity();
-      const entityRootDir = getEntityRootDir(entity);
-
-      mockFs({
-        [entityRootDir]: {
-          'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`,
-        },
-      });
-
-      const expectedMetadata: TechDocsMetadata = {
-        site_name: 'backstage',
-        site_description: 'site_content',
-        etag: 'etag',
-      };
-      expect(
-        await publisher.fetchTechDocsMetadata(entityNameMock),
-      ).toStrictEqual(expectedMetadata);
-      mockFs.restore();
-    });
-
-    it('should return an error if the techdocs_metadata.json file is not present', async () => {
-      const entityNameMock = createMockEntityName();
-
-      let error;
-      try {
-        await publisher.fetchTechDocsMetadata(entityNameMock);
-      } catch (e) {
-        error = e;
-      }
-
-      expect(error.message).toEqual(
-        expect.stringContaining('TechDocs metadata fetch'),
-      );
-    });
-  });
-});
diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
index 155d14fbb5..6ecc2b76cc 100644
--- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
@@ -26,16 +26,18 @@ import limiterFactory from 'p-limit';
 import { default as path, default as platformPath } from 'path';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
-import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import {
+  ConfigurationValidationResponse,
+  PublisherBase,
+  PublishRequest,
+  TechDocsMetadata,
+} from './types';
 
 // The number of batches that may be ongoing at the same time.
 const BATCH_CONCURRENCY = 3;
 
 export class AzureBlobStoragePublish implements PublisherBase {
-  static async fromConfig(
-    config: Config,
-    logger: Logger,
-  ): Promise<PublisherBase> {
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
     let containerName = '';
     try {
       containerName = config.getString(
@@ -78,26 +80,6 @@ export class AzureBlobStoragePublish implements PublisherBase {
       credential,
     );
 
-    try {
-      const response = await storageClient
-        .getContainerClient(containerName)
-        .getProperties();
-
-      if (response._response.status >= 400) {
-        throw new Error(
-          `Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
-        );
-      }
-    } catch (e) {
-      logger.error(
-        `Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
-          'Make sure that the Azure project and container exist and the access key is setup correctly ' +
-          'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
-          'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
-      );
-      throw new Error(`from Azure Blob Storage client library: ${e.message}`);
-    }
-
     return new AzureBlobStoragePublish(storageClient, containerName, logger);
   }
 
@@ -111,6 +93,37 @@ export class AzureBlobStoragePublish implements PublisherBase {
     this.logger = logger;
   }
 
+  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+    try {
+      const response = await this.storageClient
+        .getContainerClient(this.containerName)
+        .getProperties();
+
+      if (response._response.status === 200) {
+        return {
+          isValid: true,
+        };
+      }
+
+      if (response._response.status >= 400) {
+        this.logger.error(
+          `Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
+        );
+      }
+    } catch (e) {
+      this.logger.error(`from Azure Blob Storage client library: ${e.message}`);
+    }
+
+    this.logger.error(
+      `Could not retrieve metadata about the Azure Blob Storage container ${this.containerName}. ` +
+        'Make sure that the Azure project and container exist and the access key is setup correctly ' +
+        'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
+        'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+    );
+
+    return { isValid: false };
+  }
+
   /**
    * Upload all the files from the generated `directory` to the Azure Blob Storage container.
    * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
index 07bc5c06b9..c8cbcb0fda 100644
--- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
@@ -16,8 +16,8 @@
 import { getVoidLogger } from '@backstage/backend-common';
 import {
   Entity,
-  EntityName,
   ENTITY_DEFAULT_NAMESPACE,
+  EntityName,
 } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import mockFs from 'mock-fs';
@@ -83,6 +83,35 @@ beforeEach(async () => {
 });
 
 describe('GoogleGCSPublish', () => {
+  describe('validateConfiguration', () => {
+    it('should validate correct config', async () => {
+      expect(await publisher.validateConfiguration()).toEqual({
+        isValid: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7000',
+          publisher: {
+            type: 'googleGcs',
+            googleGcs: {
+              credentials: '{}',
+              bucketName: 'errorBucket',
+            },
+          },
+        },
+      });
+
+      const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger);
+
+      expect(await errorPublisher.validateConfiguration()).toEqual({
+        isValid: false,
+      });
+    });
+  });
+
   describe('publish', () => {
     beforeEach(() => {
       const entity = createMockEntity();
diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts
index 12ecf45b2d..047518d43f 100644
--- a/packages/techdocs-common/src/stages/publish/googleStorage.ts
+++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts
@@ -26,13 +26,15 @@ import createLimiter from 'p-limit';
 import path from 'path';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
-import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import {
+  ConfigurationValidationResponse,
+  PublisherBase,
+  PublishRequest,
+  TechDocsMetadata,
+} from './types';
 
 export class GoogleGCSPublish implements PublisherBase {
-  static async fromConfig(
-    config: Config,
-    logger: Logger,
-  ): Promise<PublisherBase> {
+  static fromConfig(config: Config, logger: Logger): PublisherBase {
     let bucketName = '';
     try {
       bucketName = config.getString('techdocs.publisher.googleGcs.bucketName');
@@ -65,21 +67,6 @@ export class GoogleGCSPublish implements PublisherBase {
       }),
     });
 
-    // Check if the defined bucket exists. Being able to connect means the configuration is good
-    // and the storage client will work.
-    try {
-      await storageClient.bucket(bucketName).getMetadata();
-      logger.info(`Successfully connected to the GCS bucket ${bucketName}.`);
-    } catch (err) {
-      logger.error(
-        `Could not retrieve metadata about the GCS bucket ${bucketName}. ` +
-          'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
-          'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
-          'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
-      );
-      throw new Error(err.message);
-    }
-
     return new GoogleGCSPublish(storageClient, bucketName, logger);
   }
 
@@ -93,6 +80,33 @@ export class GoogleGCSPublish implements PublisherBase {
     this.logger = logger;
   }
 
+  /**
+   * Check if the defined bucket exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+    try {
+      await this.storageClient.bucket(this.bucketName).getMetadata();
+      this.logger.info(
+        `Successfully connected to the GCS bucket ${this.bucketName}.`,
+      );
+
+      return {
+        isValid: true,
+      };
+    } catch (err) {
+      this.logger.error(
+        `Could not retrieve metadata about the GCS bucket ${this.bucketName}. ` +
+          'Make sure the bucket exists. Also make sure that authentication is setup either by explicitly defining ' +
+          'techdocs.publisher.googleGcs.credentials in app config or by using environment variables. ' +
+          'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+      );
+      this.logger.error(`from GCS client library: ${err.message}`);
+
+      return { isValid: false };
+    }
+  }
+
   /**
    * Upload all the files from the generated `directory` to the GCS bucket.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts
index 3088c1a695..f641c63031 100644
--- a/packages/techdocs-common/src/stages/publish/local.ts
+++ b/packages/techdocs-common/src/stages/publish/local.ts
@@ -25,6 +25,7 @@ import os from 'os';
 import path from 'path';
 import { Logger } from 'winston';
 import {
+  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
   PublishResponse,
@@ -65,6 +66,12 @@ export class LocalPublish implements PublisherBase {
     this.discovery = discovery;
   }
 
+  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+    return {
+      isValid: true,
+    };
+  }
+
   publish({ entity, directory }: PublishRequest): Promise<PublishResponse> {
     const entityNamespace = entity.metadata.namespace ?? 'default';
 
diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
index f3b788916f..67de30245b 100644
--- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
+++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
@@ -13,16 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { getVoidLogger } from '@backstage/backend-common';
 import {
   Entity,
-  EntityName,
   ENTITY_DEFAULT_NAMESPACE,
+  EntityName,
 } from '@backstage/catalog-model';
 import { ConfigReader } from '@backstage/config';
 import mockFs from 'mock-fs';
 import os from 'os';
 import path from 'path';
-import * as winston from 'winston';
 import { OpenStackSwiftPublish } from './openStackSwift';
 import { PublisherBase, TechDocsMetadata } from './types';
 
@@ -59,9 +59,7 @@ const getEntityRootDir = (entity: Entity) => {
   return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
 };
 
-const logger = winston.createLogger();
-jest.spyOn(logger, 'info').mockReturnValue(logger);
-jest.spyOn(logger, 'error').mockReturnValue(logger);
+const logger = getVoidLogger();
 
 let publisher: PublisherBase;
 
@@ -89,6 +87,43 @@ beforeEach(() => {
 });
 
 describe('OpenStackSwiftPublish', () => {
+  describe('validateConfiguration', () => {
+    it('should validate correct config', async () => {
+      expect(await publisher.validateConfiguration()).toEqual({
+        isValid: true,
+      });
+    });
+
+    it('should reject incorrect config', async () => {
+      const mockConfig = new ConfigReader({
+        techdocs: {
+          requestUrl: 'http://localhost:7000',
+          publisher: {
+            type: 'openStackSwift',
+            openStackSwift: {
+              credentials: {
+                username: 'mockuser',
+                password: 'verystrongpass',
+              },
+              authUrl: 'mockauthurl',
+              region: 'mockregion',
+              containerName: 'errorBucket',
+            },
+          },
+        },
+      });
+
+      const errorPublisher = OpenStackSwiftPublish.fromConfig(
+        mockConfig,
+        logger,
+      );
+
+      expect(await errorPublisher.validateConfiguration()).toEqual({
+        isValid: false,
+      });
+    });
+  });
+
   describe('publish', () => {
     beforeEach(() => {
       const entity = createMockEntity();
diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts
index 0e60bc5796..ca05cf2ce5 100644
--- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts
+++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts
@@ -15,16 +15,21 @@
  */
 import { Entity, EntityName } from '@backstage/catalog-model';
 import { Config } from '@backstage/config';
-import { storage } from 'pkgcloud';
 import express from 'express';
 import fs from 'fs-extra';
 import JSON5 from 'json5';
 import createLimiter from 'p-limit';
 import path from 'path';
+import { storage } from 'pkgcloud';
 import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
-import { PublisherBase, PublishRequest, TechDocsMetadata } from './types';
+import {
+  ConfigurationValidationResponse,
+  PublisherBase,
+  PublishRequest,
+  TechDocsMetadata,
+} from './types';
 
 const streamToBuffer = (stream: Readable): Promise<Buffer> => {
   return new Promise((resolve, reject) => {
@@ -70,25 +75,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
       region: openStackSwiftConfig.getString('region'),
     });
 
-    // Check if the defined container exists. Being able to connect means the configuration is good
-    // and the storage client will work.
-    storageClient.getContainer(containerName, (err, container) => {
-      if (container) {
-        logger.info(
-          `Successfully connected to the OpenStack Swift container ${containerName}.`,
-        );
-      } else {
-        logger.error(
-          `Could not retrieve metadata about the OpenStack Swift container ${containerName}. ` +
-            'Make sure the container exists. Also make sure that authentication is setup either by ' +
-            'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
-            'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
-        );
-
-        logger.error(`from OpenStack client library: ${err.message}`);
-      }
-    });
-
     return new OpenStackSwiftPublish(storageClient, containerName, logger);
   }
 
@@ -102,6 +88,37 @@ export class OpenStackSwiftPublish implements PublisherBase {
     this.logger = logger;
   }
 
+  /*
+   * Check if the defined container exists. Being able to connect means the configuration is good
+   * and the storage client will work.
+   */
+  validateConfiguration(): Promise<ConfigurationValidationResponse> {
+    return new Promise(resolve => {
+      this.storageClient.getContainer(this.containerName, (err, container) => {
+        if (container) {
+          this.logger.info(
+            `Successfully connected to the OpenStack Swift container ${this.containerName}.`,
+          );
+          resolve({
+            isValid: true,
+          });
+        } else {
+          this.logger.error(
+            `Could not retrieve metadata about the OpenStack Swift container ${this.containerName}. ` +
+              'Make sure the container exists. Also make sure that authentication is setup either by ' +
+              'explicitly defining credentials and region in techdocs.publisher.openStackSwift in app config or ' +
+              'by using environment variables. Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
+          );
+
+          this.logger.error(`from OpenStack client library: ${err.message}`);
+          resolve({
+            isValid: false,
+          });
+        }
+      });
+    });
+  }
+
   /**
    * Upload all the files from the generated `directory` to the OpenStack Swift container.
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts
index f70a5f7626..909a4092ac 100644
--- a/packages/techdocs-common/src/stages/publish/publish.ts
+++ b/packages/techdocs-common/src/stages/publish/publish.ts
@@ -13,16 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Logger } from 'winston';
-import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
 
-import { PublisherType, PublisherBase } from './types';
-import { LocalPublish } from './local';
-import { GoogleGCSPublish } from './googleStorage';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+import { Logger } from 'winston';
 import { AwsS3Publish } from './awsS3';
 import { AzureBlobStoragePublish } from './azureBlobStorage';
+import { GoogleGCSPublish } from './googleStorage';
+import { LocalPublish } from './local';
 import { OpenStackSwiftPublish } from './openStackSwift';
+import { PublisherBase, PublisherType } from './types';
 
 type factoryOptions = {
   logger: Logger;
@@ -45,7 +45,7 @@ export class Publisher {
     switch (publisherType) {
       case 'googleGcs':
         logger.info('Creating Google Storage Bucket publisher for TechDocs');
-        return await GoogleGCSPublish.fromConfig(config, logger);
+        return GoogleGCSPublish.fromConfig(config, logger);
       case 'awsS3':
         logger.info('Creating AWS S3 Bucket publisher for TechDocs');
         return AwsS3Publish.fromConfig(config, logger);
diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts
index 83fee55133..a4a02ae285 100644
--- a/packages/techdocs-common/src/stages/publish/types.ts
+++ b/packages/techdocs-common/src/stages/publish/types.ts
@@ -37,6 +37,14 @@ export type PublishResponse = {
   remoteUrl?: string;
 } | void;
 
+/**
+ * Result for the validation check.
+ */
+export type ConfigurationValidationResponse = {
+  /** Tells whether the configuration is valid.   */
+  isValid: boolean;
+};
+
 /**
  * Type to hold metadata found in techdocs_metadata.json and associated with each site
  * @param etag ETag of the resource used to generate the site. Usually the latest commit sha of the source repository.
@@ -53,6 +61,14 @@ export type TechDocsMetadata = {
  * It also provides APIs to communicate with the storage service.
  */
 export interface PublisherBase {
+  /**
+   * Check if the configuration is valid. This check tries to perform certain checks to see if the
+   * publisher is configured correctly and can be used to publish or read documentations.
+   * The different implementations might e.g. use the provided service credentials to access the
+   * target or check if a folder/bucket is available.
+   */
+  validateConfiguration(): Promise<ConfigurationValidationResponse>;
+
   /**
    * Store the generated static files onto a storage service (either local filesystem or external service).
    *

From d541d9fd07e7e2b31488472b4bd8718b0163b941 Mon Sep 17 00:00:00 2001
From: Dominik Henneke <dominik.henneke@sda-se.com>
Date: Mon, 12 Apr 2021 12:02:15 +0200
Subject: [PATCH 02/23] Include review comments

Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
---
 .changeset/fluffy-lies-complain.md            | 39 ++++++++++++++++---
 packages/backend/src/plugins/techdocs.ts      |  3 ++
 .../packages/backend/src/plugins/techdocs.ts  |  3 ++
 .../src/stages/publish/awsS3.test.ts          | 10 ++---
 .../src/stages/publish/awsS3.ts               |  8 ++--
 .../stages/publish/azureBlobStorage.test.ts   | 10 ++---
 .../src/stages/publish/azureBlobStorage.ts    |  8 ++--
 .../src/stages/publish/googleStorage.test.ts  | 10 ++---
 .../src/stages/publish/googleStorage.ts       |  8 ++--
 .../src/stages/publish/local.ts               |  6 +--
 .../src/stages/publish/openStackSwift.test.ts | 10 ++---
 .../src/stages/publish/openStackSwift.ts      |  8 ++--
 .../src/stages/publish/types.ts               | 10 ++---
 13 files changed, 84 insertions(+), 49 deletions(-)

diff --git a/.changeset/fluffy-lies-complain.md b/.changeset/fluffy-lies-complain.md
index addffc2baa..b770928e1c 100644
--- a/.changeset/fluffy-lies-complain.md
+++ b/.changeset/fluffy-lies-complain.md
@@ -1,9 +1,38 @@
 ---
-'@backstage/techdocs-common': patch
+'@backstage/techdocs-common': minor
 ---
 
-Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#validateConfiguration()` method instead of throwing an error when doing `Publisher.fromConfig(...)`.
-If you want to preserve this check in your application, use the following code:
+Move the sanity checks of the publisher configurations to a dedicated `PublisherBase#getReadiness()` method instead of throwing an error when doing `Publisher.fromConfig(...)`.
+You should include the check when your backend to get early feedback about a potential misconfiguration:
+
+```diff
+  // packages/backend/src/plugins/techdocs.ts
+
+  export default async function createPlugin({
+    logger,
+    config,
+    discovery,
+    reader,
+  }: PluginEnvironment): Promise<Router> {
+    // ...
+
+    const publisher = await Publisher.fromConfig(config, {
+      logger,
+      discovery,
+    })
+
++   // checks if the publisher is working and logs the result
++   await publisher.getReadiness();
+
+    // Docker client (conditionally) used by the generators, based on techdocs.generators config.
+    const dockerClient = new Docker();
+
+    // ...
+}
+```
+
+If you want to crash your application on invalid configurations, you can throw an `Error` to preserve the old behavior.
+Please be aware that this is not the recommended for the use in a Backstage backend but might be helpful in CLI tools such as the `techdocs-cli`.
 
 ```ts
 const publisher = await Publisher.fromConfig(config, {
@@ -11,8 +40,8 @@ const publisher = await Publisher.fromConfig(config, {
   discovery,
 });
 
-const validation = await publisher.validateConfiguration();
-if (!validation.isValid) {
+const ready = await publisher.getReadiness();
+if (!ready.isAvailable) {
   throw new Error('Invalid TechDocs publisher configuration');
 }
 ```
diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts
index 9883f2afbc..d0387cd15d 100644
--- a/packages/backend/src/plugins/techdocs.ts
+++ b/packages/backend/src/plugins/techdocs.ts
@@ -48,6 +48,9 @@ export default async function createPlugin({
     discovery,
   });
 
+  // checks if the publisher is working and logs the result
+  await publisher.getReadiness();
+
   // Docker client (conditionally) used by the generators, based on techdocs.generators config.
   const dockerClient = new Docker();
 
diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts
index 0ba08cefaf..231a7e7fd7 100644
--- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts
+++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts
@@ -34,6 +34,9 @@ export default async function createPlugin({
     discovery,
   });
 
+  // checks if the publisher is working and logs the result
+  await publisher.getReadiness();
+
   // Docker client (conditionally) used by the generators, based on techdocs.generators config.
   const dockerClient = new Docker();
 
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
index b0000b6464..15da3b7cfc 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts
@@ -85,10 +85,10 @@ beforeEach(() => {
 });
 
 describe('AwsS3Publish', () => {
-  describe('validateConfiguration', () => {
+  describe('getReadiness', () => {
     it('should validate correct config', async () => {
-      expect(await publisher.validateConfiguration()).toEqual({
-        isValid: true,
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
       });
     });
 
@@ -112,8 +112,8 @@ describe('AwsS3Publish', () => {
 
       const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger);
 
-      expect(await errorPublisher.validateConfiguration()).toEqual({
-        isValid: false,
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
       });
     });
   });
diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts
index 77bb771fb6..9c6684fdca 100644
--- a/packages/techdocs-common/src/stages/publish/awsS3.ts
+++ b/packages/techdocs-common/src/stages/publish/awsS3.ts
@@ -27,9 +27,9 @@ import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
 import {
-  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
+  ReadinessResponse,
   TechDocsMetadata,
 } from './types';
 
@@ -134,7 +134,7 @@ export class AwsS3Publish implements PublisherBase {
    * Check if the defined bucket exists. Being able to connect means the configuration is good
    * and the storage client will work.
    */
-  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+  async getReadiness(): Promise<ReadinessResponse> {
     try {
       await this.storageClient
         .headBucket({ Bucket: this.bucketName })
@@ -144,7 +144,7 @@ export class AwsS3Publish implements PublisherBase {
         `Successfully connected to the AWS S3 bucket ${this.bucketName}.`,
       );
 
-      return { isValid: true };
+      return { isAvailable: true };
     } catch (error) {
       this.logger.error(
         `Could not retrieve metadata about the AWS S3 bucket ${this.bucketName}. ` +
@@ -154,7 +154,7 @@ export class AwsS3Publish implements PublisherBase {
       );
       this.logger.error(`from AWS client library`, error);
       return {
-        isValid: false,
+        isAvailable: false,
       };
     }
   }
diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
index b67e47c675..c431e924a8 100644
--- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
@@ -86,10 +86,10 @@ beforeEach(async () => {
 });
 
 describe('publishing with valid credentials', () => {
-  describe('validateConfiguration', () => {
+  describe('getReadiness', () => {
     it('should validate correct config', async () => {
-      expect(await publisher.validateConfiguration()).toEqual({
-        isValid: true,
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
       });
     });
 
@@ -115,8 +115,8 @@ describe('publishing with valid credentials', () => {
         logger,
       );
 
-      expect(await errorPublisher.validateConfiguration()).toEqual({
-        isValid: false,
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
       });
 
       expect(logger.error).toHaveBeenCalledWith(
diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
index 6ecc2b76cc..1a823c3d19 100644
--- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
@@ -27,9 +27,9 @@ import { default as path, default as platformPath } from 'path';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
 import {
-  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
+  ReadinessResponse,
   TechDocsMetadata,
 } from './types';
 
@@ -93,7 +93,7 @@ export class AzureBlobStoragePublish implements PublisherBase {
     this.logger = logger;
   }
 
-  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+  async getReadiness(): Promise<ReadinessResponse> {
     try {
       const response = await this.storageClient
         .getContainerClient(this.containerName)
@@ -101,7 +101,7 @@ export class AzureBlobStoragePublish implements PublisherBase {
 
       if (response._response.status === 200) {
         return {
-          isValid: true,
+          isAvailable: true,
         };
       }
 
@@ -121,7 +121,7 @@ export class AzureBlobStoragePublish implements PublisherBase {
         'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
     );
 
-    return { isValid: false };
+    return { isAvailable: false };
   }
 
   /**
diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
index c8cbcb0fda..7e580b16bc 100644
--- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts
@@ -83,10 +83,10 @@ beforeEach(async () => {
 });
 
 describe('GoogleGCSPublish', () => {
-  describe('validateConfiguration', () => {
+  describe('getReadiness', () => {
     it('should validate correct config', async () => {
-      expect(await publisher.validateConfiguration()).toEqual({
-        isValid: true,
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
       });
     });
 
@@ -106,8 +106,8 @@ describe('GoogleGCSPublish', () => {
 
       const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger);
 
-      expect(await errorPublisher.validateConfiguration()).toEqual({
-        isValid: false,
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
       });
     });
   });
diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts
index 047518d43f..28fbb42ece 100644
--- a/packages/techdocs-common/src/stages/publish/googleStorage.ts
+++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts
@@ -27,9 +27,9 @@ import path from 'path';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
 import {
-  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
+  ReadinessResponse,
   TechDocsMetadata,
 } from './types';
 
@@ -84,7 +84,7 @@ export class GoogleGCSPublish implements PublisherBase {
    * Check if the defined bucket exists. Being able to connect means the configuration is good
    * and the storage client will work.
    */
-  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+  async getReadiness(): Promise<ReadinessResponse> {
     try {
       await this.storageClient.bucket(this.bucketName).getMetadata();
       this.logger.info(
@@ -92,7 +92,7 @@ export class GoogleGCSPublish implements PublisherBase {
       );
 
       return {
-        isValid: true,
+        isAvailable: true,
       };
     } catch (err) {
       this.logger.error(
@@ -103,7 +103,7 @@ export class GoogleGCSPublish implements PublisherBase {
       );
       this.logger.error(`from GCS client library: ${err.message}`);
 
-      return { isValid: false };
+      return { isAvailable: false };
     }
   }
 
diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts
index f641c63031..e09473cf60 100644
--- a/packages/techdocs-common/src/stages/publish/local.ts
+++ b/packages/techdocs-common/src/stages/publish/local.ts
@@ -25,10 +25,10 @@ import os from 'os';
 import path from 'path';
 import { Logger } from 'winston';
 import {
-  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
   PublishResponse,
+  ReadinessResponse,
   TechDocsMetadata,
 } from './types';
 
@@ -66,9 +66,9 @@ export class LocalPublish implements PublisherBase {
     this.discovery = discovery;
   }
 
-  async validateConfiguration(): Promise<ConfigurationValidationResponse> {
+  async getReadiness(): Promise<ReadinessResponse> {
     return {
-      isValid: true,
+      isAvailable: true,
     };
   }
 
diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
index 67de30245b..6f0aacede0 100644
--- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
+++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
@@ -87,10 +87,10 @@ beforeEach(() => {
 });
 
 describe('OpenStackSwiftPublish', () => {
-  describe('validateConfiguration', () => {
+  describe('getReadiness', () => {
     it('should validate correct config', async () => {
-      expect(await publisher.validateConfiguration()).toEqual({
-        isValid: true,
+      expect(await publisher.getReadiness()).toEqual({
+        isAvailable: true,
       });
     });
 
@@ -118,8 +118,8 @@ describe('OpenStackSwiftPublish', () => {
         logger,
       );
 
-      expect(await errorPublisher.validateConfiguration()).toEqual({
-        isValid: false,
+      expect(await errorPublisher.getReadiness()).toEqual({
+        isAvailable: false,
       });
     });
   });
diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts
index ca05cf2ce5..bff0055402 100644
--- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts
+++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts
@@ -25,9 +25,9 @@ import { Readable } from 'stream';
 import { Logger } from 'winston';
 import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers';
 import {
-  ConfigurationValidationResponse,
   PublisherBase,
   PublishRequest,
+  ReadinessResponse,
   TechDocsMetadata,
 } from './types';
 
@@ -92,7 +92,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
    * Check if the defined container exists. Being able to connect means the configuration is good
    * and the storage client will work.
    */
-  validateConfiguration(): Promise<ConfigurationValidationResponse> {
+  getReadiness(): Promise<ReadinessResponse> {
     return new Promise(resolve => {
       this.storageClient.getContainer(this.containerName, (err, container) => {
         if (container) {
@@ -100,7 +100,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
             `Successfully connected to the OpenStack Swift container ${this.containerName}.`,
           );
           resolve({
-            isValid: true,
+            isAvailable: true,
           });
         } else {
           this.logger.error(
@@ -112,7 +112,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
 
           this.logger.error(`from OpenStack client library: ${err.message}`);
           resolve({
-            isValid: false,
+            isAvailable: false,
           });
         }
       });
diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts
index a4a02ae285..c2849b8e35 100644
--- a/packages/techdocs-common/src/stages/publish/types.ts
+++ b/packages/techdocs-common/src/stages/publish/types.ts
@@ -40,9 +40,9 @@ export type PublishResponse = {
 /**
  * Result for the validation check.
  */
-export type ConfigurationValidationResponse = {
-  /** Tells whether the configuration is valid.   */
-  isValid: boolean;
+export type ReadinessResponse = {
+  /** If true, the publisher is able to interact with the backing storage. */
+  isAvailable: boolean;
 };
 
 /**
@@ -62,12 +62,12 @@ export type TechDocsMetadata = {
  */
 export interface PublisherBase {
   /**
-   * Check if the configuration is valid. This check tries to perform certain checks to see if the
+   * Check if the publisher is ready. This check tries to perform certain checks to see if the
    * publisher is configured correctly and can be used to publish or read documentations.
    * The different implementations might e.g. use the provided service credentials to access the
    * target or check if a folder/bucket is available.
    */
-  validateConfiguration(): Promise<ConfigurationValidationResponse>;
+  getReadiness(): Promise<ReadinessResponse>;
 
   /**
    * Store the generated static files onto a storage service (either local filesystem or external service).

From f1952337ce3577912a1bcb45cc01f8b1e58b4fc3 Mon Sep 17 00:00:00 2001
From: Dominik Henneke <dominik.henneke@sda-se.com>
Date: Tue, 13 Apr 2021 10:14:00 +0200
Subject: [PATCH 03/23] Add changeset for create-app

Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
---
 .changeset/yellow-snails-hang.md | 33 ++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 .changeset/yellow-snails-hang.md

diff --git a/.changeset/yellow-snails-hang.md b/.changeset/yellow-snails-hang.md
new file mode 100644
index 0000000000..e908a98aee
--- /dev/null
+++ b/.changeset/yellow-snails-hang.md
@@ -0,0 +1,33 @@
+---
+'@backstage/create-app': patch
+---
+
+Due to a change in the techdocs publishers, they don't check if they are able to reach e.g. the configured S3 bucket anymore.
+This can be added again by the following change. Note that the backend process will no longer exit when it is not reachable but will only emit an error log message.
+You should include the check when your backend to get early feedback about a potential misconfiguration:
+
+```diff
+  // packages/backend/src/plugins/techdocs.ts
+
+  export default async function createPlugin({
+    logger,
+    config,
+    discovery,
+    reader,
+  }: PluginEnvironment): Promise<Router> {
+    // ...
+
+    const publisher = await Publisher.fromConfig(config, {
+      logger,
+      discovery,
+    })
+
++   // checks if the publisher is working and logs the result
++   await publisher.getReadiness();
+
+    // Docker client (conditionally) used by the generators, based on techdocs.generators config.
+    const dockerClient = new Docker();
+
+    // ...
+}
+```

From 5abace95ac82c9cc866412ea43b7326971456c2e Mon Sep 17 00:00:00 2001
From: Frank Showalter <fshowalter@gmail.com>
Date: Tue, 13 Apr 2021 10:45:52 -0400
Subject: [PATCH 04/23] Disable hot-reloading for e2e CI tests

Signed-off-by: Frank Showalter <fshowalter@gmail.com>
---
 packages/cli/src/lib/bundler/server.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/lib/bundler/server.ts b/packages/cli/src/lib/bundler/server.ts
index 5fc3a7b5de..24a24d25d2 100644
--- a/packages/cli/src/lib/bundler/server.ts
+++ b/packages/cli/src/lib/bundler/server.ts
@@ -43,7 +43,7 @@ export async function serveBundle(options: ServeOptions) {
   const compiler = webpack(config);
 
   const server = new WebpackDevServer(compiler, {
-    hot: true,
+    hot: !process.env.CI,
     contentBase: paths.targetPublic,
     contentBasePublicPath: config.output?.publicPath,
     publicPath: config.output?.publicPath,

From 60ce64aa20cf7a8a870bbe7a922cfd28224bf9ee Mon Sep 17 00:00:00 2001
From: Frank Showalter <fshowalter@gmail.com>
Date: Tue, 13 Apr 2021 10:49:37 -0400
Subject: [PATCH 05/23] add changeset

Signed-off-by: Frank Showalter <fshowalter@gmail.com>
---
 .changeset/young-kids-remember.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/young-kids-remember.md

diff --git a/.changeset/young-kids-remember.md b/.changeset/young-kids-remember.md
new file mode 100644
index 0000000000..c07097234f
--- /dev/null
+++ b/.changeset/young-kids-remember.md
@@ -0,0 +1,5 @@
+---
+'@backstage/cli': patch
+---
+
+Disable hot reloading in CI environments.

From 78cad7589aea83a0be3b64fed25befbc5e1240d5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 14 Apr 2021 04:15:33 +0000
Subject: [PATCH 06/23] chore(deps): bump @rjsf/material-ui from 2.4.1 to 2.5.1

Bumps [@rjsf/material-ui](https://github.com/rjsf-team/react-jsonschema-form) from 2.4.1 to 2.5.1.
- [Release notes](https://github.com/rjsf-team/react-jsonschema-form/releases)
- [Commits](https://github.com/rjsf-team/react-jsonschema-form/compare/v2.4.1...v2.5.1)

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 6f7e171b2c..ac4cf4c08e 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4239,9 +4239,9 @@
     shortid "^2.2.14"
 
 "@rjsf/material-ui@^2.4.0":
-  version "2.4.1"
-  resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.4.1.tgz#b0dedff8877b114147e298966ca3faba895a7a62"
-  integrity sha512-pZaWL5Dw+km8S0hFLIK1lRHaeNtheMxTF2mZrWhf6HlGWCTGkQJnXta2UgshJN/nKtZPgO1L4FKz42Eun9nnhg==
+  version "2.5.1"
+  resolved "https://registry.npmjs.org/@rjsf/material-ui/-/material-ui-2.5.1.tgz#b93ac9f1f4a909e2aae729616859c2d72557e53e"
+  integrity sha512-ooKxQJO12+i1xCGtknMZDxWSbVlSEgQ5U1I7lJ+uI/MvwAg3Rz9wb2cTEF3ErBczT7EEW+j1lR19rxBjFd78MA==
 
 "@roadiehq/backstage-plugin-buildkite@^1.0.0":
   version "1.0.0"

From 4592d8ae198d2731bc540aac9317c64bf36c22e2 Mon Sep 17 00:00:00 2001
From: Oliver Sand <oliver.sand@sda-se.com>
Date: Wed, 14 Apr 2021 09:39:58 +0200
Subject: [PATCH 07/23]  Remove unused stories from storybook

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
---
 .../core/src/layout/Header/Header.stories.tsx | 32 -------------------
 1 file changed, 32 deletions(-)

diff --git a/packages/core/src/layout/Header/Header.stories.tsx b/packages/core/src/layout/Header/Header.stories.tsx
index 3deb980dfd..db4ddac8ec 100644
--- a/packages/core/src/layout/Header/Header.stories.tsx
+++ b/packages/core/src/layout/Header/Header.stories.tsx
@@ -53,38 +53,6 @@ export const Apis = () => (
   </Page>
 );
 
-export const Grpc = () => (
-  <Page themeId="grpc">
-    <Header title="Grpc catalogue" type="tool">
-      {labels}
-    </Header>
-  </Page>
-);
-
-export const AsyncApi = () => (
-  <Page themeId="asyncapi">
-    <Header title="Async API catalogue" type="tool">
-      {labels}
-    </Header>
-  </Page>
-);
-
-export const Graphql = () => (
-  <Page themeId="graphql">
-    <Header title="GraphQL API catalogue" type="tool">
-      {labels}
-    </Header>
-  </Page>
-);
-
-export const OpenApi = () => (
-  <Page themeId="openapi">
-    <Header title="OpenAPI catalogue" type="tool">
-      {labels}
-    </Header>
-  </Page>
-);
-
 export const Tool = () => (
   <Page themeId="tool">
     <Header title="Stand-alone tool" type="tool">

From 0962565da9af884acef2cc4e6e7e336a2a64e15f Mon Sep 17 00:00:00 2001
From: Oliver Sand <oliver.sand@sda-se.com>
Date: Wed, 14 Apr 2021 09:40:15 +0200
Subject: [PATCH 08/23] Restore page theme for API docs plugin to previous
 color

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
---
 packages/theme/src/pageTheme.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/theme/src/pageTheme.ts b/packages/theme/src/pageTheme.ts
index c9a61bd95b..36f49bcdf9 100644
--- a/packages/theme/src/pageTheme.ts
+++ b/packages/theme/src/pageTheme.ts
@@ -63,5 +63,5 @@ export const pageTheme: Record<string, PageTheme> = {
   library: genPageTheme(colorVariants.rubyRed, shapes.wave),
   other: genPageTheme(colorVariants.darkGrey, shapes.wave),
   app: genPageTheme(colorVariants.toastyOrange, shapes.wave),
-  apis: genPageTheme(colorVariants.eveningSea, shapes.wave2),
+  apis: genPageTheme(colorVariants.teal, shapes.wave2),
 };

From bb5055aeea70f4463c00392e57a3f2465541959b Mon Sep 17 00:00:00 2001
From: Johan Haals <johan.haals@gmail.com>
Date: Wed, 14 Apr 2021 09:59:00 +0200
Subject: [PATCH 09/23] Catalog-model: Add getEntitySourceLocation helper

Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>
---
 .changeset/metal-foxes-speak.md               |  5 ++
 .../src/location/helpers.test.ts              | 51 ++++++++++++++++++-
 .../catalog-model/src/location/helpers.ts     | 26 ++++++++++
 packages/catalog-model/src/location/index.ts  |  6 ++-
 4 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 .changeset/metal-foxes-speak.md

diff --git a/.changeset/metal-foxes-speak.md b/.changeset/metal-foxes-speak.md
new file mode 100644
index 0000000000..45b8ae3c7c
--- /dev/null
+++ b/.changeset/metal-foxes-speak.md
@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-model': patch
+---
+
+Add getEntitySourceLocation helper
diff --git a/packages/catalog-model/src/location/helpers.test.ts b/packages/catalog-model/src/location/helpers.test.ts
index 888c79b328..3b5994b956 100644
--- a/packages/catalog-model/src/location/helpers.test.ts
+++ b/packages/catalog-model/src/location/helpers.test.ts
@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
-import { parseLocationReference, stringifyLocationReference } from './helpers';
+import {
+  getEntitySourceLocation,
+  parseLocationReference,
+  stringifyLocationReference,
+} from './helpers';
 
 describe('parseLocationReference', () => {
   it('works for the simple case', () => {
@@ -68,3 +72,48 @@ describe('stringifyLocationReference', () => {
     ).toThrow('Unable to stringify location reference, empty target');
   });
 });
+
+describe('getEntitySourceLocation', () => {
+  it('returns the source-location', () => {
+    expect(
+      getEntitySourceLocation({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Location',
+        metadata: {
+          name: 'test',
+          namespace: 'default',
+          annotations: {
+            'backstage.io/source-location': 'url:https://backstage.io/foo.yaml',
+            'backstage.io/managed-by-location': 'url:https://spotify.com',
+          },
+        },
+      }),
+    ).toEqual({ target: 'https://backstage.io/foo.yaml', type: 'url' });
+  });
+
+  it('returns the managed-by-location', () => {
+    expect(
+      getEntitySourceLocation({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Location',
+        metadata: {
+          name: 'test',
+          namespace: 'default',
+          annotations: {
+            'backstage.io/managed-by-location': 'url:https://spotify.com',
+          },
+        },
+      }),
+    ).toEqual({ target: 'https://spotify.com', type: 'url' });
+  });
+
+  it('rejects missing location annotation', () => {
+    expect(() =>
+      getEntitySourceLocation({
+        apiVersion: 'backstage.io/v1alpha1',
+        kind: 'Location',
+        metadata: { name: 'test', namespace: 'default' },
+      }),
+    ).toThrow(`Entity 'location:default/test' is missing location`);
+  });
+});
diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts
index fb1be5abad..9dce67d054 100644
--- a/packages/catalog-model/src/location/helpers.ts
+++ b/packages/catalog-model/src/location/helpers.ts
@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+import { Entity, stringifyEntityRef } from '../entity';
+import { LOCATION_ANNOTATION, SOURCE_LOCATION_ANNOTATION } from './annotation';
+
 /**
  * Parses a string form location reference.
  *
@@ -80,3 +83,26 @@ export function stringifyLocationReference(ref: {
 
   return `${type}:${target}`;
 }
+
+/**
+ * Returns the source code location of the Entity, to the extend which one exists.
+ *
+ * If the returned location type is of type 'url', the target should be readable at least
+ * using the UrlReader from @backstage/backend-common. If it is not of type 'url', the caller
+ * needs to have explicit handling of each location type or signal that it is not supported.
+ */
+export function getEntitySourceLocation(
+  entity: Entity,
+): { type: string; target: string } {
+  const locationRef =
+    entity.metadata?.annotations?.[SOURCE_LOCATION_ANNOTATION] ??
+    entity.metadata?.annotations?.[LOCATION_ANNOTATION];
+
+  if (!locationRef) {
+    throw new Error(
+      `Entity '${stringifyEntityRef(entity)}' is missing location`,
+    );
+  }
+
+  return parseLocationReference(locationRef);
+}
diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts
index fddc8bde37..751172c6ba 100644
--- a/packages/catalog-model/src/location/index.ts
+++ b/packages/catalog-model/src/location/index.ts
@@ -19,7 +19,11 @@ export {
   ORIGIN_LOCATION_ANNOTATION,
   SOURCE_LOCATION_ANNOTATION,
 } from './annotation';
-export { parseLocationReference, stringifyLocationReference } from './helpers';
+export {
+  parseLocationReference,
+  stringifyLocationReference,
+  getEntitySourceLocation,
+} from './helpers';
 export type { Location, LocationSpec } from './types';
 export {
   analyzeLocationSchema,

From e44574d2147544aebf45f6d83c26d26048aba13f Mon Sep 17 00:00:00 2001
From: Johan Haals <johan.haals@gmail.com>
Date: Wed, 14 Apr 2021 10:20:58 +0200
Subject: [PATCH 10/23] Update packages/catalog-model/src/location/helpers.ts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Johan Haals johan.haals@gmail.com

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>
---
 packages/catalog-model/src/location/helpers.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts
index 9dce67d054..5eff598c87 100644
--- a/packages/catalog-model/src/location/helpers.ts
+++ b/packages/catalog-model/src/location/helpers.ts
@@ -85,7 +85,7 @@ export function stringifyLocationReference(ref: {
 }
 
 /**
- * Returns the source code location of the Entity, to the extend which one exists.
+ * Returns the source code location of the Entity, to the extent that one exists.
  *
  * If the returned location type is of type 'url', the target should be readable at least
  * using the UrlReader from @backstage/backend-common. If it is not of type 'url', the caller

From f361f9e9dd1805aa27922aae835ced05d05a175b Mon Sep 17 00:00:00 2001
From: Patrik Oldsberg <poldsberg@gmail.com>
Date: Wed, 14 Apr 2021 10:46:47 +0200
Subject: [PATCH 11/23] docs: tweak docs for backstage.io/source-location

Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
---
 docs/features/software-catalog/well-known-annotations.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/features/software-catalog/well-known-annotations.md b/docs/features/software-catalog/well-known-annotations.md
index 8ec4813799..0053c14b79 100644
--- a/docs/features/software-catalog/well-known-annotations.md
+++ b/docs/features/software-catalog/well-known-annotations.md
@@ -92,12 +92,14 @@ view and edit links need changing.
 # Example:
 metadata:
   annotations:
-    backstage.io/source-location: github:https://github.com/my-org/my-service
+    backstage.io/source-location: url:https://github.com/my-org/my-service/
 ```
 
 A `Location` reference that points to the source code of the entity (typically a
 `Component`). Useful when catalog files do not get ingested from the source code
-repository itself.
+repository itself. If the URL points to a folder, it is important that it is
+suffixed with a `'/'` in order for relative path resolution to work
+consistently.
 
 ### jenkins.io/github-folder
 

From f4f9fd04aa5b7e9ce9105c83365f99e248651a0c Mon Sep 17 00:00:00 2001
From: blam <ben@blam.sh>
Date: Wed, 14 Apr 2021 11:47:01 +0200
Subject: [PATCH 12/23] bump(rjsf): bump core too

Signed-off-by: blam <ben@blam.sh>
---
 yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index ac4cf4c08e..f44faae442 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4222,9 +4222,9 @@
     react-lifecycles-compat "^3.0.4"
 
 "@rjsf/core@^2.4.0":
-  version "2.4.0"
-  resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.4.0.tgz#c50bcff0d8178948ce08123177399d8816d51929"
-  integrity sha512-8zlydBkGldOxGXFEwNGFa1gzTxpcxaYn7ofegcu8XHJ7IKMCfpnU3ABg+H3eml1KZCX3FODmj1tHFJKuTmfynw==
+  version "2.5.1"
+  resolved "https://registry.npmjs.org/@rjsf/core/-/core-2.5.1.tgz#95a842d22bab5f83929662fcd73739108e9f5cbb"
+  integrity sha512-km8NYScXNONaL5BiSLS6wyDj49pOLZtn0iXg7Zxlm921uuf3o2AAX5SuZS5kB4Zj2zlrVMrXESexfX6bxdDYHw==
   dependencies:
     "@babel/runtime-corejs2" "^7.8.7"
     "@types/json-schema" "^7.0.4"

From 7fd46f26dc264fc480ad207ec10cf00c9900807e Mon Sep 17 00:00:00 2001
From: Oliver Sand <oliver.sand@sda-se.com>
Date: Wed, 14 Apr 2021 14:03:17 +0200
Subject: [PATCH 13/23] Use `string` TypeScript type instead of `String`

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
---
 .changeset/old-clouds-whisper.md                             | 5 +++++
 .../KubernetesAuthTranslatorGenerator.ts                     | 2 +-
 .../src/service/KubernetesFanOutHandler.test.ts              | 4 ++--
 3 files changed, 8 insertions(+), 3 deletions(-)
 create mode 100644 .changeset/old-clouds-whisper.md

diff --git a/.changeset/old-clouds-whisper.md b/.changeset/old-clouds-whisper.md
new file mode 100644
index 0000000000..edaba13ad7
--- /dev/null
+++ b/.changeset/old-clouds-whisper.md
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-kubernetes-backend': patch
+---
+
+Use `string` TypeScript type instead of `String`.
diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts
index 4ae2aff35a..d222bdbce1 100644
--- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts
+++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts
@@ -21,7 +21,7 @@ import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator
 
 export class KubernetesAuthTranslatorGenerator {
   static getKubernetesAuthTranslatorInstance(
-    authProvider: String,
+    authProvider: string,
   ): KubernetesAuthTranslator {
     switch (authProvider) {
       case 'google': {
diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts
index e76a114d54..7ec1a317d8 100644
--- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts
+++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts
@@ -34,8 +34,8 @@ const mockFetch = (mock: jest.Mock) => {
 };
 
 function generateMockResourcesAndErrors(
-  serviceId: String,
-  clusterName: String,
+  serviceId: string,
+  clusterName: string,
 ) {
   if (clusterName === 'empty-cluster') {
     return {

From 442f34b87f96b1fefd3a57da9ceae7a36fcc4122 Mon Sep 17 00:00:00 2001
From: Oliver Sand <oliver.sand@sda-se.com>
Date: Wed, 14 Apr 2021 14:06:09 +0200
Subject: [PATCH 14/23] Make sure the `CatalogClient` escapes URL parameters
 correctly

Signed-off-by: Oliver Sand <oliver.sand@sda-se.com>
---
 .changeset/tiny-games-hide.md                |  5 ++++
 packages/catalog-client/src/CatalogClient.ts | 24 ++++++++++++++++----
 packages/catalog-client/src/types.ts         |  2 +-
 plugins/catalog/src/CatalogClientWrapper.ts  |  2 +-
 4 files changed, 26 insertions(+), 7 deletions(-)
 create mode 100644 .changeset/tiny-games-hide.md

diff --git a/.changeset/tiny-games-hide.md b/.changeset/tiny-games-hide.md
new file mode 100644
index 0000000000..2687174060
--- /dev/null
+++ b/.changeset/tiny-games-hide.md
@@ -0,0 +1,5 @@
+---
+'@backstage/catalog-client': patch
+---
+
+Make sure the `CatalogClient` escapes URL parameters correctly.
diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts
index 7426aabaac..3de25e9808 100644
--- a/packages/catalog-client/src/CatalogClient.ts
+++ b/packages/catalog-client/src/CatalogClient.ts
@@ -42,10 +42,14 @@ export class CatalogClient implements CatalogApi {
   }
 
   async getLocationById(
-    id: String,
+    id: string,
     options?: CatalogRequestOptions,
   ): Promise<Location | undefined> {
-    return await this.requestOptional('GET', `/locations/${id}`, options);
+    return await this.requestOptional(
+      'GET',
+      `/locations/${encodeURIComponent(id)}`,
+      options,
+    );
   }
 
   async getEntities(
@@ -86,7 +90,9 @@ export class CatalogClient implements CatalogApi {
     const { kind, namespace = 'default', name } = compoundName;
     return this.requestOptional(
       'GET',
-      `/entities/by-name/${kind}/${namespace}/${name}`,
+      `/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent(
+        namespace,
+      )}/${encodeURIComponent(name)}`,
       options,
     );
   }
@@ -171,14 +177,22 @@ export class CatalogClient implements CatalogApi {
     id: string,
     options?: CatalogRequestOptions,
   ): Promise<void> {
-    await this.requestIgnored('DELETE', `/locations/${id}`, options);
+    await this.requestIgnored(
+      'DELETE',
+      `/locations/${encodeURIComponent(id)}`,
+      options,
+    );
   }
 
   async removeEntityByUid(
     uid: string,
     options?: CatalogRequestOptions,
   ): Promise<void> {
-    await this.requestIgnored('DELETE', `/entities/by-uid/${uid}`, options);
+    await this.requestIgnored(
+      'DELETE',
+      `/entities/by-uid/${encodeURIComponent(uid)}`,
+      options,
+    );
   }
 
   //
diff --git a/packages/catalog-client/src/types.ts b/packages/catalog-client/src/types.ts
index 04cbb2b689..0d25bf7483 100644
--- a/packages/catalog-client/src/types.ts
+++ b/packages/catalog-client/src/types.ts
@@ -46,7 +46,7 @@ export interface CatalogApi {
 
   // Locations
   getLocationById(
-    id: String,
+    id: string,
     options?: CatalogRequestOptions,
   ): Promise<Location | undefined>;
   getOriginLocationByEntity(
diff --git a/plugins/catalog/src/CatalogClientWrapper.ts b/plugins/catalog/src/CatalogClientWrapper.ts
index 32dff8a0a1..4966d9c13f 100644
--- a/plugins/catalog/src/CatalogClientWrapper.ts
+++ b/plugins/catalog/src/CatalogClientWrapper.ts
@@ -42,7 +42,7 @@ export class CatalogClientWrapper implements CatalogApi {
   }
 
   async getLocationById(
-    id: String,
+    id: string,
     options?: CatalogRequestOptions,
   ): Promise<Location | undefined> {
     return await this.client.getLocationById(id, {

From fef852ecd3ccf2ab136342b3b38bca034c583814 Mon Sep 17 00:00:00 2001
From: Jonah Grimes <jgrimes@appriss.com>
Date: Wed, 14 Apr 2021 10:45:20 -0400
Subject: [PATCH 15/23] Reworked TechDocs page subtitle to reflect the
 company/organization name (#5317)

* reworked page subtitle to reflect the company/organization name instead of 'Backstage'

Signed-off-by: Jonah Grimes <jgrimes@appriss.com>

* ran prettier to reformat code

Signed-off-by: Jonah Grimes <jgrimes@appriss.com>

* marked changeset as techdocs only

Signed-off-by: Jonah Grimes <jgrimes@appriss.com>
---
 .changeset/techdocs-chatty-impalas-compare.md |  6 +++++
 .../src/home/components/TechDocsHome.test.tsx | 21 +++++++++++++++---
 .../src/home/components/TechDocsHome.tsx      | 22 +++++++++----------
 3 files changed, 34 insertions(+), 15 deletions(-)
 create mode 100644 .changeset/techdocs-chatty-impalas-compare.md

diff --git a/.changeset/techdocs-chatty-impalas-compare.md b/.changeset/techdocs-chatty-impalas-compare.md
new file mode 100644
index 0000000000..26e7fd571d
--- /dev/null
+++ b/.changeset/techdocs-chatty-impalas-compare.md
@@ -0,0 +1,6 @@
+---
+'@backstage/plugin-techdocs': patch
+---
+
+Reworked the TechDocs plugin to support using the configured company name instead of
+'Backstage' in the page title.
diff --git a/plugins/techdocs/src/home/components/TechDocsHome.test.tsx b/plugins/techdocs/src/home/components/TechDocsHome.test.tsx
index 9783c98f27..8067767d39 100644
--- a/plugins/techdocs/src/home/components/TechDocsHome.test.tsx
+++ b/plugins/techdocs/src/home/components/TechDocsHome.test.tsx
@@ -14,7 +14,13 @@
  * limitations under the License.
  */
 
-import { ApiProvider, ApiRegistry } from '@backstage/core';
+import {
+  ApiProvider,
+  ApiRegistry,
+  ConfigApi,
+  configApiRef,
+  ConfigReader,
+} from '@backstage/core';
 import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react';
 import { renderInTestApp } from '@backstage/test-utils';
 import { screen } from '@testing-library/react';
@@ -26,7 +32,16 @@ describe('TechDocs Home', () => {
     getEntities: async () => ({ items: [] }),
   };
 
-  const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi);
+  const configApi: ConfigApi = new ConfigReader({
+    organization: {
+      name: 'My Company',
+    },
+  });
+
+  const apiRegistry = ApiRegistry.with(catalogApiRef, catalogApi).with(
+    configApiRef,
+    configApi,
+  );
 
   it('should render a TechDocs home page', async () => {
     await renderInTestApp(
@@ -38,7 +53,7 @@ describe('TechDocs Home', () => {
     // Header
     expect(await screen.findByText('Documentation')).toBeInTheDocument();
     expect(
-      await screen.findByText(/Documentation available in Backstage/i),
+      await screen.findByText(/Documentation available in My Company/i),
     ).toBeInTheDocument();
 
     // Explore Content
diff --git a/plugins/techdocs/src/home/components/TechDocsHome.tsx b/plugins/techdocs/src/home/components/TechDocsHome.tsx
index 916a46ab7a..5e9e25e7a0 100644
--- a/plugins/techdocs/src/home/components/TechDocsHome.tsx
+++ b/plugins/techdocs/src/home/components/TechDocsHome.tsx
@@ -21,6 +21,8 @@ import { catalogApiRef, CatalogApi } from '@backstage/plugin-catalog-react';
 import { Entity } from '@backstage/catalog-model';
 import {
   CodeSnippet,
+  ConfigApi,
+  configApiRef,
   Content,
   Header,
   HeaderTabs,
@@ -36,6 +38,7 @@ import { OwnedContent } from './OwnedContent';
 export const TechDocsHome = () => {
   const [selectedTab, setSelectedTab] = useState<number>(0);
   const catalogApi: CatalogApi = useApi(catalogApiRef);
+  const configApi: ConfigApi = useApi(configApiRef);
 
   const tabs = [{ label: 'Overview' }, { label: 'Owned Documents' }];
 
@@ -46,13 +49,14 @@ export const TechDocsHome = () => {
     });
   });
 
+  const generatedSubtitle = `Documentation available in ${
+    configApi.getOptionalString('organization.name') ?? 'Backstage'
+  }`;
+
   if (loading) {
     return (
       <Page themeId="documentation">
-        <Header
-          title="Documentation"
-          subtitle="Documentation available in Backstage"
-        />
+        <Header title="Documentation" subtitle={generatedSubtitle} />
         <Content>
           <Progress />
         </Content>
@@ -63,10 +67,7 @@ export const TechDocsHome = () => {
   if (error) {
     return (
       <Page themeId="documentation">
-        <Header
-          title="Documentation"
-          subtitle="Documentation available in Backstage"
-        />
+        <Header title="Documentation" subtitle={generatedSubtitle} />
         <Content>
           <WarningPanel
             severity="error"
@@ -81,10 +82,7 @@ export const TechDocsHome = () => {
 
   return (
     <Page themeId="documentation">
-      <Header
-        title="Documentation"
-        subtitle="Documentation available in Backstage"
-      />
+      <Header title="Documentation" subtitle={generatedSubtitle} />
       <HeaderTabs
         selectedIndex={selectedTab}
         onChange={index => setSelectedTab(index)}

From 5ce9744d0bff011bc5ee200bc3bbae1ac97598fb Mon Sep 17 00:00:00 2001
From: Tim Hansen <timbonicus@gmail.com>
Date: Wed, 14 Apr 2021 21:00:23 -0600
Subject: [PATCH 16/23] Add `addProcessor` code sample to external-integrations

Signed-off-by: Tim Hansen <timbonicus@gmail.com>
---
 .../software-catalog/external-integrations.md      | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md
index e278bc7a1a..247324a46c 100644
--- a/docs/features/software-catalog/external-integrations.md
+++ b/docs/features/software-catalog/external-integrations.md
@@ -156,8 +156,18 @@ The key points to note are:
 - Call `emit` any number of times with the results of that process
 - Finally return `true`
 
-You should now be able to instantiate this class in your backend, and add it to
-the `CatalogBuilder` using the `addProcessors` method.
+You should now be able to add this class to your backend in
+`packages/backend/src/plugins/catalog.ts`:
+
+```diff
++ import { SystemXReaderProcessor } from '../path/to/class';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const builder = new CatalogBuilder(env);
++  builder.addProcessor(new SystemXReaderProcessor(env.reader));
+```
 
 Start up the backend - it should now start reading from the previously
 registered location and you'll see your entities start to appear in Backstage.

From 3f3a735f55b146246fe94bc89d115c9c6385bef8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 15 Apr 2021 04:19:28 +0000
Subject: [PATCH 17/23] chore(deps-dev): bump @spotify/prettier-config in
 /microsite

Bumps [@spotify/prettier-config](https://github.com/spotify/web-scripts) from 9.0.0 to 10.0.0.
- [Release notes](https://github.com/spotify/web-scripts/releases)
- [Changelog](https://github.com/spotify/web-scripts/blob/master/CHANGELOG.md)
- [Commits](https://github.com/spotify/web-scripts/compare/v9.0.0...v10.0.0)

Signed-off-by: dependabot[bot] <support@github.com>
---
 microsite/package.json | 2 +-
 microsite/yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/microsite/package.json b/microsite/package.json
index ea16fd8a32..9caafbd95b 100644
--- a/microsite/package.json
+++ b/microsite/package.json
@@ -15,7 +15,7 @@
     "verify:sidebars": "node ./scripts/verify-sidebars"
   },
   "devDependencies": {
-    "@spotify/prettier-config": "^9.0.0",
+    "@spotify/prettier-config": "^10.0.0",
     "docusaurus": "^2.0.0-alpha.70",
     "js-yaml": "^4.0.0",
     "prettier": "^2.2.1"
diff --git a/microsite/yarn.lock b/microsite/yarn.lock
index 898fd69ad2..5618a5ee4b 100644
--- a/microsite/yarn.lock
+++ b/microsite/yarn.lock
@@ -909,10 +909,10 @@
   resolved "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
   integrity sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==
 
-"@spotify/prettier-config@^9.0.0":
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-9.0.0.tgz#7b562d56573c6fc0094446fbc92b22bc318945dc"
-  integrity sha512-In1q0tIiqTYKAGe3KOHDcFDdZRFISyQeSeipeTHGfki23ebHRZcjxvqj5SSdBkw65D4VpSREMi0s9i5iJiMcTw==
+"@spotify/prettier-config@^10.0.0":
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/@spotify/prettier-config/-/prettier-config-10.0.0.tgz#fa076d98d2e7e6c53dd3d86a696307a7010bd056"
+  integrity sha512-VYOdo8P7lIScAkl02nB9KpUAuOYMManryBIBuKJkAw5D3aVtLobfmdIKvdV6MqEmGMEQPbn7w/UpnjJYhUH+IA==
 
 "@types/cheerio@^0.22.8":
   version "0.22.23"

From 22f01501ca16a23bbf72a28ea54c904c91ef9c82 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 15 Apr 2021 04:19:38 +0000
Subject: [PATCH 18/23] chore(deps): bump semver from 7.3.4 to 7.3.5

Bumps [semver](https://github.com/npm/node-semver) from 7.3.4 to 7.3.5.
- [Release notes](https://github.com/npm/node-semver/releases)
- [Changelog](https://github.com/npm/node-semver/blob/master/CHANGELOG.md)
- [Commits](https://github.com/npm/node-semver/compare/v7.3.4...v7.3.5)

Signed-off-by: dependabot[bot] <support@github.com>
---
 yarn.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index f44faae442..1f76456d00 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -23398,9 +23398,9 @@ semver@7.0.0:
   integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==
 
 semver@7.x, semver@^7.0.0, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4:
-  version "7.3.4"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.4.tgz#27aaa7d2e4ca76452f98d3add093a72c943edc97"
-  integrity sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==
+  version "7.3.5"
+  resolved "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7"
+  integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==
   dependencies:
     lru-cache "^6.0.0"
 

From c2a685ddb1217541547713ae3f2d257217c4be84 Mon Sep 17 00:00:00 2001
From: blam <ben@blam.sh>
Date: Tue, 13 Apr 2021 19:38:44 +0200
Subject: [PATCH 19/23] feat(scaffolder): add support for parsing the valuues
 as a json object in the parameters

Signed-off-by: blam <ben@blam.sh>
---
 .../src/scaffolder/tasks/TaskWorker.test.ts   | 97 +++++++++++++++++--
 .../src/scaffolder/tasks/TaskWorker.ts        | 14 ++-
 2 files changed, 103 insertions(+), 8 deletions(-)

diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
index b598b91036..014fdf2d99 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
@@ -41,20 +41,24 @@ async function createStore(): Promise<DatabaseTaskStore> {
 
 describe('TaskWorker', () => {
   let storage: DatabaseTaskStore;
+  let actionRegistry = new TemplateActionRegistry();
 
   beforeAll(async () => {
     storage = await createStore();
   });
 
-  const logger = getVoidLogger();
-  const actionRegistry = new TemplateActionRegistry();
-  actionRegistry.register({
-    id: 'test-action',
-    handler: async ctx => {
-      ctx.output('testOutput', 'winning');
-    },
+  beforeEach(() => {
+    actionRegistry = new TemplateActionRegistry();
+    actionRegistry.register({
+      id: 'test-action',
+      handler: async ctx => {
+        ctx.output('testOutput', 'winning');
+      },
+    });
   });
 
+  const logger = getVoidLogger();
+
   it('should fail when action does not exist', async () => {
     const broker = new StorageTaskBroker(storage, logger);
     const taskWorker = new TaskWorker({
@@ -166,4 +170,83 @@ describe('TaskWorker', () => {
     const event = events.find(e => e.type === 'completion');
     expect((event?.body?.output as JsonObject).result).toBe('winning');
   });
+
+  it('should parse strings as objects if possible', async () => {
+    const inputAction = createTemplateAction<{
+      address: { line1: string };
+      address2: string;
+    }>({
+      id: 'test-input',
+      schema: {
+        input: {
+          type: 'object',
+          required: ['address'],
+          properties: {
+            address: {
+              title: 'address',
+              description: 'Enter name',
+              type: 'object',
+              properties: {
+                line1: {
+                  type: 'string',
+                },
+              },
+            },
+            address2: {
+              type: 'string',
+            },
+          },
+        },
+      },
+      async handler(ctx) {
+        if (ctx.input.address.line1 !== 'line 1') {
+          throw new Error(
+            `expected address.line1 to be "line 1" got ${ctx.input.address.line1}`,
+          );
+        }
+
+        if (ctx.input.address2 !== '{"not valid"}') {
+          throw new Error(
+            `expected address2 to be "{"not valid"}" got ${ctx.input.address2}`,
+          );
+        }
+        ctx.output('address', ctx.input.address.line1);
+      },
+    });
+    actionRegistry.register(inputAction);
+
+    const broker = new StorageTaskBroker(storage, logger);
+    const taskWorker = new TaskWorker({
+      logger,
+      workingDirectory: os.tmpdir(),
+      actionRegistry,
+      taskBroker: broker,
+    });
+
+    const { taskId } = await broker.dispatch({
+      steps: [
+        {
+          id: 'test-input',
+          name: 'test-input',
+          action: 'test-input',
+          input: {
+            address: JSON.stringify({ line1: 'line 1' }),
+            address2: '{"not valid"}',
+          },
+        },
+      ],
+      output: {
+        result: '{{ steps.test-input.output.address }}',
+      },
+      values: {},
+    });
+
+    const task = await broker.claim();
+    await taskWorker.runOneTask(task);
+
+    const { events } = await storage.listEvents({ taskId });
+    const event = events.find(e => e.type === 'completion');
+
+    expect((event?.body?.output as JsonObject).result).toBe('line 1');
+  });
 });
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
index 3a846f79a5..aef9e22ada 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
@@ -102,13 +102,25 @@ export class TaskWorker {
             step.input &&
             JSON.parse(JSON.stringify(step.input), (_key, value) => {
               if (typeof value === 'string') {
-                return handlebars.compile(value, {
+                const templated = handlebars.compile(value, {
                   noEscape: true,
                   strict: true,
                   data: false,
                   preventIndent: true,
                 })(templateCtx);
+
+                // If it smells like a JSON object then give it a parse as an object and if it fails return the string
+                if (templated.startsWith('{') && templated.endsWith('}')) {
+                  try {
+                    // Don't recursively JSON parse the values of this string. Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
+                    return JSON.parse(templated);
+                  } catch {
+                    return templated;
+                  }
+                }
+                return templated;
               }
+
               return value;
             });
 

From 096c0d85e06a654ac734e2bf98acc2342acdaee8 Mon Sep 17 00:00:00 2001
From: blam <ben@blam.sh>
Date: Thu, 15 Apr 2021 11:59:09 +0200
Subject: [PATCH 20/23] feat(scaffolder): added parseRepoUrl helper to parse to
 json object

Signed-off-by: blam <ben@blam.sh>
---
 .../actions/builtin/publish/util.ts           |  10 +-
 .../src/scaffolder/tasks/TaskWorker.test.ts   | 112 ++++++++++++++++++
 .../src/scaffolder/tasks/TaskWorker.ts        |  30 ++++-
 3 files changed, 144 insertions(+), 8 deletions(-)

diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts
index 07b3823e7b..acf1351004 100644
--- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts
@@ -30,8 +30,14 @@ export const getRepoSourceDirectory = (
   }
   return workspacePath;
 };
+export type RepoSpec = {
+  repo: string;
+  host: string;
+  owner: string;
+  organization?: string;
+};
 
-export const parseRepoUrl = (repoUrl: string) => {
+export const parseRepoUrl = (repoUrl: string): RepoSpec => {
   let parsed;
   try {
     parsed = new URL(`https://${repoUrl}`);
@@ -55,7 +61,7 @@ export const parseRepoUrl = (repoUrl: string) => {
     );
   }
 
-  const organization = parsed.searchParams.get('organization');
+  const organization = parsed.searchParams.get('organization') ?? undefined;
 
   return { host, owner, repo, organization };
 };
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
index 014fdf2d99..edef02064f 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
@@ -24,6 +24,7 @@ import { ConfigReader, JsonObject } from '@backstage/config';
 import { StorageTaskBroker } from './StorageTaskBroker';
 import { DatabaseTaskStore } from './DatabaseTaskStore';
 import { createTemplateAction, TemplateActionRegistry } from '../actions';
+import { RepoSpec } from '../actions/builtin/publish/util';
 
 async function createStore(): Promise<DatabaseTaskStore> {
   const manager = SingleConnectionDatabaseManager.fromConfig(
@@ -174,6 +175,7 @@ describe('TaskWorker', () => {
   it('should parse strings as objects if possible', async () => {
     const inputAction = createTemplateAction<{
       address: { line1: string };
+      list: string[];
       address2: string;
     }>({
       id: 'test-input',
@@ -195,10 +197,21 @@ describe('TaskWorker', () => {
             address2: {
               type: 'string',
             },
+            list: {
+              type: 'array',
+              items: {
+                type: 'string',
+              },
+            },
           },
         },
       },
       async handler(ctx) {
+        if (ctx.input.list.length !== 1) {
+          throw new Error(
+            `expected list to have length "1" got ${ctx.input.list.length}`,
+          );
+        }
         if (ctx.input.address.line1 !== 'line 1') {
           throw new Error(
             `expected address.line1 to be "line 1" got ${ctx.input.address.line1}`,
@@ -231,6 +244,7 @@ describe('TaskWorker', () => {
           action: 'test-input',
           input: {
             address: JSON.stringify({ line1: 'line 1' }),
+            list: JSON.stringify(['hey!']),
             address2: '{"not valid"}',
           },
         },
@@ -249,4 +263,102 @@ describe('TaskWorker', () => {
 
     expect((event?.body?.output as JsonObject).result).toBe('line 1');
   });
+
+  // TODO(blam): Can delete this test when we make the helpers a public API
+  it('should provide a repoUrlParse helper for the templates', async () => {
+    const inputAction = createTemplateAction<{
+      destination: RepoSpec;
+    }>({
+      id: 'test-input',
+      schema: {
+        input: {
+          type: 'object',
+          required: ['destination'],
+          properties: {
+            destination: {
+              title: 'destination',
+              type: 'object',
+              properties: {
+                repo: {
+                  type: 'string',
+                },
+                host: {
+                  type: 'string',
+                },
+                owner: {
+                  type: 'string',
+                },
+                organization: {
+                  type: 'string',
+                },
+              },
+            },
+          },
+        },
+      },
+      async handler(ctx) {
+        ctx.output('host', ctx.input.destination.host);
+        ctx.output('repo', ctx.input.destination.repo);
+        ctx.output('owner', ctx.input.destination.owner);
+
+        if (ctx.input.destination.host !== 'github.com') {
+          throw new Error(
+            `expected host to be "github.com" got ${ctx.input.destination.host}`,
+          );
+        }
+
+        if (ctx.input.destination.repo !== 'repo') {
+          throw new Error(
+            `expected repo to be "repo" got ${ctx.input.destination.repo}`,
+          );
+        }
+
+        if (ctx.input.destination.owner !== 'owner') {
+          throw new Error(
+            `expected repo to be "owner" got ${ctx.input.destination.owner}`,
+          );
+        }
+      },
+    });
+    actionRegistry.register(inputAction);
+
+    const broker = new StorageTaskBroker(storage, logger);
+    const taskWorker = new TaskWorker({
+      logger,
+      workingDirectory: os.tmpdir(),
+      actionRegistry,
+      taskBroker: broker,
+    });
+
+    const { taskId } = await broker.dispatch({
+      steps: [
+        {
+          id: 'test-input',
+          name: 'test-input',
+          action: 'test-input',
+          input: {
+            destination: '{{ parseRepoUrl parameters.repoUrl }}',
+          },
+        },
+      ],
+      output: {
+        host: '{{ steps.test-input.output.host }}',
+        repo: '{{ steps.test-input.output.repo }}',
+        owner: '{{ steps.test-input.output.owner }}',
+      },
+      values: {
+        repoUrl: 'github.com?repo=repo&owner=owner',
+      },
+    });
+
+    const task = await broker.claim();
+    await taskWorker.runOneTask(task);
+
+    const { events } = await storage.listEvents({ taskId });
+    const event = events.find(e => e.type === 'completion');
+
+    expect((event?.body?.output as JsonObject).host).toBe('github.com');
+    expect((event?.body?.output as JsonObject).repo).toBe('repo');
+    expect((event?.body?.output as JsonObject).owner).toBe('owner');
+  });
 });
diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
index aef9e22ada..14752e1988 100644
--- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
+++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
@@ -23,8 +23,9 @@ import { TaskBroker, Task } from './types';
 import fs from 'fs-extra';
 import path from 'path';
 import { TemplateActionRegistry } from '../actions/TemplateActionRegistry';
-import * as handlebars from 'handlebars';
+import * as Handlebars from 'handlebars';
 import { InputError } from '@backstage/errors';
+import { parseRepoUrl } from '../actions/builtin/publish/util';
 
 type Options = {
   logger: Logger;
@@ -34,7 +35,20 @@ type Options = {
 };
 
 export class TaskWorker {
-  constructor(private readonly options: Options) {}
+  private readonly handlebars: typeof Handlebars;
+
+  constructor(private readonly options: Options) {
+    this.handlebars = Handlebars.create();
+
+    // TODO(blam): this should be a public facing API but it's a little
+    // scary right now, so we're going to lock it off like the component API is
+    // in the frontend until we can work out a nice way to do it.
+    this.handlebars.registerHelper('parseRepoUrl', repoUrl => {
+      return JSON.stringify(parseRepoUrl(repoUrl));
+    });
+
+    this.handlebars.registerHelper('json', obj => JSON.stringify(obj));
+  }
 
   start() {
     (async () => {
@@ -102,7 +116,7 @@ export class TaskWorker {
             step.input &&
             JSON.parse(JSON.stringify(step.input), (_key, value) => {
               if (typeof value === 'string') {
-                const templated = handlebars.compile(value, {
+                const templated = this.handlebars.compile(value, {
                   noEscape: true,
                   strict: true,
                   data: false,
@@ -110,9 +124,13 @@ export class TaskWorker {
                 })(templateCtx);
 
                 // If it smells like a JSON object then give it a parse as an object and if it fails return the string
-                if (templated.startsWith('{') && templated.endsWith('}')) {
+                if (
+                  (templated.startsWith('{') && templated.endsWith('}')) ||
+                  (templated.startsWith('[') && templated.endsWith(']'))
+                ) {
                   try {
-                    // Don't recursively JSON parse the values of this string. Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
+                    // Don't recursively JSON parse the values of this string.
+                    // Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
                     return JSON.parse(templated);
                   } catch {
                     return templated;
@@ -183,7 +201,7 @@ export class TaskWorker {
         JSON.stringify(task.spec.output),
         (_key, value) => {
           if (typeof value === 'string') {
-            return handlebars.compile(value, {
+            return this.handlebars.compile(value, {
               noEscape: true,
               strict: true,
               data: false,

From 4de6b74cc0be6f332eb0869c421da18163df4ffb Mon Sep 17 00:00:00 2001
From: blam <ben@blam.sh>
Date: Thu, 15 Apr 2021 12:08:31 +0200
Subject: [PATCH 21/23] chore: update sample template to show how to

Signed-off-by: blam <ben@blam.sh>
---
 .../sample-templates/v1beta2-demo/template.yaml                  | 1 +
 .../sample-templates/v1beta2-demo/template/catalog-info.yaml     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml
index e90b43b7f3..1a01f7998c 100644
--- a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml
+++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml
@@ -50,6 +50,7 @@ spec:
         values:
           name: '{{ parameters.name }}'
           owner: '{{ parameters.owner }}'
+          destination: '{{ parseRepoUrl parameters.repoUrl }}'
 
     - id: fetch-docs
       name: Fetch Docs
diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml
index 875664d2a8..6519d68402 100644
--- a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml
+++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml
@@ -2,6 +2,7 @@ apiVersion: backstage.io/v1alpha1
 kind: Component
 metadata:
   name: {{cookiecutter.name | jsonify}}
+  github.com/project-slug: {{cookiecutter.destination.owner + "/" + cookiecutter.destination.repo}}
 spec:
   type: website
   lifecycle: experimental

From b25846562a69a67a3272ce19ddffdadf27dcfc25 Mon Sep 17 00:00:00 2001
From: blam <ben@blam.sh>
Date: Thu, 15 Apr 2021 12:16:38 +0200
Subject: [PATCH 22/23] chore:  add changeset

Signed-off-by: blam <ben@blam.sh>
---
 .changeset/silent-papayas-dress.md | 38 ++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 .changeset/silent-papayas-dress.md

diff --git a/.changeset/silent-papayas-dress.md b/.changeset/silent-papayas-dress.md
new file mode 100644
index 0000000000..c3c05971fe
--- /dev/null
+++ b/.changeset/silent-papayas-dress.md
@@ -0,0 +1,38 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Enable the JSON parsing of the response from templated variables in the `v2beta1` syntax. Previously if template parameters json strings they were left as strings, they are now parsed as JSON objects.
+
+Before:
+
+```yaml
+- id: test
+  name: test-action
+  action: custom:run
+  input:
+    input: '{"hello":"ben"}'
+```
+
+Now:
+
+```yaml
+- id: test
+  name: test-action
+  action: custom:run
+  input:
+    input:
+      hello: ben
+```
+
+Also added the `parseRepoUrl` and `json` helpers to the parameters syntax. You can now use these helpers to parse work with some `json` or `repoUrl` strings in templates.
+
+```yaml
+- id: test
+  name: test-action
+  action: cookiecutter:fetch
+  input:
+    destination: '{{ parseRepoUrl parameters.repoUrl }}'
+```
+
+Will produce a parsed version of the `repoUrl` of type `{ repo: string, owner: string, host: string }` that you can use in your actions. Specifically `cookiecutter` with `{{ cookiecutter.destination.owner }}` like the `plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml` example.

From c28689c68685e4d490825065c396cbd2bcd2871a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= <freben@gmail.com>
Date: Thu, 15 Apr 2021 13:25:48 +0200
Subject: [PATCH 23/23] change all `$env` to `${}`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
---
 .tugboat/tugboat.app-config.production.yaml   |   9 +-
 app-config.yaml                               | 168 ++++++------------
 contrib/chart/backstage/values.yaml           |  63 +++----
 docs/auth/auth-backend-classes.md             |  41 ++---
 docs/conf/writing.md                          |   3 +
 docs/features/kubernetes/configuration.md     |   3 +-
 .../software-templates/installation.md        |  18 +-
 docs/features/techdocs/configuration.md       |  18 +-
 docs/features/techdocs/using-cloud-storage.md |  45 ++---
 .../configure-app-with-plugins.md             |   3 +-
 .../google-cloud-storage/locations.md         |   6 +-
 docs/plugins/proxying.md                      |   5 +-
 docs/tutorials/quickstart-app-auth.md         |  41 ++---
 docs/tutorials/switching-sqlite-postgres.md   |  18 +-
 .../templates/default-app/app-config.yaml.hbs |  21 +--
 plugins/bitrise/README.md                     |   3 +-
 plugins/circleci/README.md                    |   3 +-
 plugins/fossa/README.md                       |   4 +-
 plugins/jenkins/README.md                     |   6 +-
 plugins/newrelic/README.md                    |   3 +-
 plugins/rollbar-backend/README.md             |   3 +-
 plugins/rollbar/README.md                     |   3 +-
 plugins/sentry/README.md                      |   4 +-
 plugins/sonarqube/README.md                   |  24 ++-
 plugins/splunk-on-call/README.md              |   8 +-
 .../AuthProviders/EmptyProviders.tsx          |   6 +-
 26 files changed, 184 insertions(+), 345 deletions(-)

diff --git a/.tugboat/tugboat.app-config.production.yaml b/.tugboat/tugboat.app-config.production.yaml
index f606574d89..b264d53f3f 100644
--- a/.tugboat/tugboat.app-config.production.yaml
+++ b/.tugboat/tugboat.app-config.production.yaml
@@ -1,13 +1,10 @@
 app:
   title: Backstage Tugboat Preview
-  baseUrl:
-    $env: TUGBOAT_DEFAULT_SERVICE_URL
+  baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL}
 
 backend:
-  baseUrl:
-    $env: TUGBOAT_DEFAULT_SERVICE_URL
+  baseUrl: ${TUGBOAT_DEFAULT_SERVICE_URL}
   cors:
-    origin:
-      $env: TUGBOAT_DEFAULT_SERVICE_URL
+    origin: ${TUGBOAT_DEFAULT_SERVICE_URL}
     methods: [GET, POST, PUT, DELETE]
     credentials: true
diff --git a/app-config.yaml b/app-config.yaml
index 309fa28234..afbda40852 100644
--- a/app-config.yaml
+++ b/app-config.yaml
@@ -40,47 +40,40 @@ proxy:
   '/circleci/api':
     target: https://circleci.com/api/v1.1
     headers:
-      Circle-Token:
-        $env: CIRCLECI_AUTH_TOKEN
+      Circle-Token: ${CIRCLECI_AUTH_TOKEN}
 
   '/jenkins/api':
     target: http://localhost:8080
     headers:
-      Authorization:
-        $env: JENKINS_BASIC_AUTH_HEADER
+      Authorization: ${JENKINS_BASIC_AUTH_HEADER}
 
   '/travisci/api':
     target: https://api.travis-ci.com
     changeOrigin: true
     headers:
-      Authorization:
-        $env: TRAVISCI_AUTH_TOKEN
+      Authorization: ${TRAVISCI_AUTH_TOKEN}
       travis-api-version: '3'
 
   '/newrelic/apm/api':
     target: https://api.newrelic.com/v2
     headers:
-      X-Api-Key:
-        $env: NEW_RELIC_REST_API_KEY
+      X-Api-Key: ${NEW_RELIC_REST_API_KEY}
 
   '/pagerduty':
     target: https://api.pagerduty.com
     headers:
-      Authorization:
-        $env: PAGERDUTY_TOKEN
+      Authorization: ${PAGERDUTY_TOKEN}
 
   '/buildkite/api':
     target: https://api.buildkite.com/v2/
     headers:
-      Authorization:
-        $env: BUILDKITE_TOKEN
+      Authorization: ${BUILDKITE_TOKEN}
 
   '/sentry/api':
     target: https://sentry.io/api/
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        $env: SENTRY_TOKEN
+      Authorization: ${SENTRY_TOKEN}
 
 organization:
   name: My Company
@@ -124,36 +117,28 @@ kafka:
 integrations:
   github:
     - host: github.com
-      token:
-        $env: GITHUB_TOKEN
+      token: ${GITHUB_TOKEN}
     ### Example for how to add your GitHub Enterprise instance using the API:
     # - host: ghe.example.net
     #   apiBaseUrl: https://ghe.example.net/api/v3
-    #   token:
-    #     $env: GHE_TOKEN
+    #   token: ${GHE_TOKEN}
     ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional):
     # - host: ghe.example.net
     #   rawBaseUrl: https://ghe.example.net/raw
-    #   token:
-    #     $env: GHE_TOKEN
+    #   token: ${GHE_TOKEN}
   gitlab:
     - host: gitlab.com
-      token:
-        $env: GITLAB_TOKEN
+      token: ${GITLAB_TOKEN}
   bitbucket:
     - host: bitbucket.org
-      username:
-        $env: BITBUCKET_USERNAME
-      appPassword:
-        $env: BITBUCKET_APP_PASSWORD
+      username: ${BITBUCKET_USERNAME}
+      appPassword: ${BITBUCKET_APP_PASSWORD}
   azure:
     - host: dev.azure.com
-      token:
-        $env: AZURE_TOKEN
+      token: ${AZURE_TOKEN}
 #  googleGcs:
 #    clientEmail: 'example@example.com'
-#    privateKey:
-#      $env: GCS_PRIVATE_KEY
+#    privateKey: ${GCS_PRIVATE_KEY}
 
 catalog:
   rules:
@@ -172,21 +157,18 @@ catalog:
     githubOrg:
       providers:
         - target: https://github.com
-          token:
-            $env: GITHUB_TOKEN
+          token: ${GITHUB_TOKEN}
         #### Example for how to add your GitHub Enterprise instance using the API:
         # - target: https://ghe.example.net
         #   apiBaseUrl: https://ghe.example.net/api
-        #   token:
-        #     $env: GHE_TOKEN
+        #   token: ${GHE_TOKEN}
     ldapOrg:
       ### Example for how to add your enterprise LDAP server
       # providers:
       #   - target: ldaps://ds.example.net
       #     bind:
       #       dn: uid=ldap-reader-user,ou=people,ou=example,dc=example,dc=net
-      #       secret:
-      #         $env: LDAP_SECRET
+      #       secret: ${LDAP_SECRET}
       #     users:
       #       dn: ou=people,ou=example,dc=example,dc=net
       #       options:
@@ -202,12 +184,9 @@ catalog:
       #providers:
       #  - target: https://graph.microsoft.com/v1.0
       #    authority: https://login.microsoftonline.com
-      #    tenantId:
-      #      $env: MICROSOFT_GRAPH_TENANT_ID
-      #    clientId:
-      #      $env: MICROSOFT_GRAPH_CLIENT_ID
-      #    clientSecret:
-      #      $env: MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN
+      #    tenantId: ${MICROSOFT_GRAPH_TENANT_ID}
+      #    clientId: ${MICROSOFT_GRAPH_CLIENT_ID}
+      #    clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN}
       #    userFilter: accountEnabled eq true and userType eq 'member'
       #    groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified')
 
@@ -255,27 +234,22 @@ catalog:
 
 scaffolder:
   github:
-    token:
-      $env: GITHUB_TOKEN
+    token: ${GITHUB_TOKEN}
     visibility: public # or 'internal' or 'private'
   gitlab:
     api:
       baseUrl: https://gitlab.com
-      token:
-        $env: GITLAB_TOKEN
+      token: ${GITLAB_TOKEN}
     visibility: public # or 'internal' or 'private'
   azure:
     baseUrl: https://dev.azure.com/{your-organization}
     api:
-      token:
-        $env: AZURE_TOKEN
+      token: ${AZURE_TOKEN}
   bitbucket:
     api:
       host: https://bitbucket.org
-      username:
-        $env: BITBUCKET_USERNAME
-      token:
-        $env: BITBUCKET_TOKEN
+      username: ${BITBUCKET_USERNAME}
+      token: ${BITBUCKET_TOKEN}
     visibility: public # or or 'private'
 
 auth:
@@ -286,89 +260,59 @@ auth:
   providers:
     google:
       development:
-        clientId:
-          $env: AUTH_GOOGLE_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GOOGLE_CLIENT_SECRET
+        clientId: ${AUTH_GOOGLE_CLIENT_ID}
+        clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
     github:
       development:
-        clientId:
-          $env: AUTH_GITHUB_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GITHUB_CLIENT_SECRET
-        enterpriseInstanceUrl:
-          $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
+        clientId: ${AUTH_GITHUB_CLIENT_ID}
+        clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
+        enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
     gitlab:
       development:
-        clientId:
-          $env: AUTH_GITLAB_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GITLAB_CLIENT_SECRET
-        audience:
-          $env: GITLAB_BASE_URL
+        clientId: ${AUTH_GITLAB_CLIENT_ID}
+        clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
+        audience: ${GITLAB_BASE_URL}
     saml:
       entryPoint: 'http://localhost:7001/'
       issuer: 'passport-saml'
     okta:
       development:
-        clientId:
-          $env: AUTH_OKTA_CLIENT_ID
-        clientSecret:
-          $env: AUTH_OKTA_CLIENT_SECRET
-        audience:
-          $env: AUTH_OKTA_AUDIENCE
+        clientId: ${AUTH_OKTA_CLIENT_ID}
+        clientSecret: ${AUTH_OKTA_CLIENT_SECRET}
+        audience: ${AUTH_OKTA_AUDIENCE}
     oauth2:
       development:
-        clientId:
-          $env: AUTH_OAUTH2_CLIENT_ID
-        clientSecret:
-          $env: AUTH_OAUTH2_CLIENT_SECRET
-        authorizationUrl:
-          $env: AUTH_OAUTH2_AUTH_URL
-        tokenUrl:
-          $env: AUTH_OAUTH2_TOKEN_URL
+        clientId: ${AUTH_OAUTH2_CLIENT_ID}
+        clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
+        authorizationUrl: ${AUTH_OAUTH2_AUTH_URL}
+        tokenUrl: ${AUTH_OAUTH2_TOKEN_URL}
         ###
         # provide a list of scopes as needed for your OAuth2 Server:
         #
         # scope: saml-login-selector openid profile email
     oidc:
       development:
-        metadataUrl:
-          $env: AUTH_OIDC_METADATA_URL
-        clientId:
-          $env: AUTH_OIDC_CLIENT_ID
-        clientSecret:
-          $env: AUTH_OIDC_CLIENT_SECRET
-        authorizationUrl:
-          $env: AUTH_OIDC_AUTH_URL
-        tokenUrl:
-          $env: AUTH_OIDC_TOKEN_URL
-        tokenSignedResponseAlg:
-          $env: AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG
+        metadataUrl: ${AUTH_OIDC_METADATA_URL}
+        clientId: ${AUTH_OIDC_CLIENT_ID}
+        clientSecret: ${AUTH_OIDC_CLIENT_SECRET}
+        authorizationUrl: ${AUTH_OIDC_AUTH_URL}
+        tokenUrl: ${AUTH_OIDC_TOKEN_URL}
+        tokenSignedResponseAlg: ${AUTH_OIDC_TOKEN_SIGNED_RESPONSE_ALG}
     auth0:
       development:
-        clientId:
-          $env: AUTH_AUTH0_CLIENT_ID
-        clientSecret:
-          $env: AUTH_AUTH0_CLIENT_SECRET
-        domain:
-          $env: AUTH_AUTH0_DOMAIN
+        clientId: ${AUTH_AUTH0_CLIENT_ID}
+        clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
+        domain: ${AUTH_AUTH0_DOMAIN}
     microsoft:
       development:
-        clientId:
-          $env: AUTH_MICROSOFT_CLIENT_ID
-        clientSecret:
-          $env: AUTH_MICROSOFT_CLIENT_SECRET
-        tenantId:
-          $env: AUTH_MICROSOFT_TENANT_ID
+        clientId: ${AUTH_MICROSOFT_CLIENT_ID}
+        clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
+        tenantId: ${AUTH_MICROSOFT_TENANT_ID}
     onelogin:
       development:
-        clientId:
-          $env: AUTH_ONELOGIN_CLIENT_ID
-        clientSecret:
-          $env: AUTH_ONELOGIN_CLIENT_SECRET
-        issuer:
-          $env: AUTH_ONELOGIN_ISSUER
+        clientId: ${AUTH_ONELOGIN_CLIENT_ID}
+        clientSecret: ${AUTH_ONELOGIN_CLIENT_SECRET}
+        issuer: ${AUTH_ONELOGIN_ISSUER}
 costInsights:
   engineerCost: 200000
   products:
diff --git a/contrib/chart/backstage/values.yaml b/contrib/chart/backstage/values.yaml
index 7b26d2bccd..bd80bc22b6 100644
--- a/contrib/chart/backstage/values.yaml
+++ b/contrib/chart/backstage/values.yaml
@@ -127,68 +127,47 @@ appConfig:
         development:
           appOrigin: 'http://localhost:3000/'
           secure: false
-          clientId:
-            $env: AUTH_GOOGLE_CLIENT_ID
-          clientSecret:
-            $env: AUTH_GOOGLE_CLIENT_SECRET
+          clientId: ${AUTH_GOOGLE_CLIENT_ID}
+          clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
       github:
         development:
           appOrigin: 'http://localhost:3000/'
           secure: false
-          clientId:
-            $env: AUTH_GITHUB_CLIENT_ID
-          clientSecret:
-            $env: AUTH_GITHUB_CLIENT_SECRET
-          enterpriseInstanceUrl:
-            $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
+          clientId: ${AUTH_GITHUB_CLIENT_ID}
+          clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
+          enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
       gitlab:
         development:
           appOrigin: 'http://localhost:3000/'
           secure: false
-          clientId:
-            $env: AUTH_GITLAB_CLIENT_ID
-          clientSecret:
-            $env: AUTH_GITLAB_CLIENT_SECRET
-          audience:
-            $env: GITLAB_BASE_URL
+          clientId: ${AUTH_GITLAB_CLIENT_ID}
+          clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
+          audience: ${GITLAB_BASE_URL}
       okta:
         development:
           appOrigin: 'http://localhost:3000/'
           secure: false
-          clientId:
-            $env: AUTH_OKTA_CLIENT_ID
-          clientSecret:
-            $env: AUTH_OKTA_CLIENT_SECRET
-          audience:
-            $env: AUTH_OKTA_AUDIENCE
+          clientId: ${AUTH_OKTA_CLIENT_ID}
+          clientSecret: ${AUTH_OKTA_CLIENT_SECRET}
+          audience: ${AUTH_OKTA_AUDIENCE}
       oauth2:
         development:
           appOrigin: 'http://localhost:3000/'
           secure: false
-          clientId:
-            $env: AUTH_OAUTH2_CLIENT_ID
-          clientSecret:
-            $env: AUTH_OAUTH2_CLIENT_SECRET
-          authorizationURL:
-            $env: AUTH_OAUTH2_AUTH_URL
-          tokenURL:
-            $env: AUTH_OAUTH2_TOKEN_URL
+          clientId: ${AUTH_OAUTH2_CLIENT_ID}
+          clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
+          authorizationURL: ${AUTH_OAUTH2_AUTH_URL}
+          tokenURL: ${AUTH_OAUTH2_TOKEN_URL}
       auth0:
         development:
-          clientId:
-            $env: AUTH_AUTH0_CLIENT_ID
-          clientSecret:
-            $env: AUTH_AUTH0_CLIENT_SECRET
-          domain:
-            $env: AUTH_AUTH0_DOMAIN
+          clientId: ${AUTH_AUTH0_CLIENT_ID}
+          clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
+          domain: ${AUTH_AUTH0_DOMAIN}
       microsoft:
         development:
-          clientId:
-            $env: AUTH_MICROSOFT_CLIENT_ID
-          clientSecret:
-            $env: AUTH_MICROSOFT_CLIENT_SECRET
-          tenantId:
-            $env: AUTH_MICROSOFT_TENANT_ID
+          clientId: ${AUTH_MICROSOFT_CLIENT_ID}
+          clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
+          tenantId: ${AUTH_MICROSOFT_TENANT_ID}
 
 auth:
   google:
diff --git a/docs/auth/auth-backend-classes.md b/docs/auth/auth-backend-classes.md
index a99ca119f0..5b5950f6c3 100644
--- a/docs/auth/auth-backend-classes.md
+++ b/docs/auth/auth-backend-classes.md
@@ -95,40 +95,27 @@ auth:
   providers:
     google:
       development:
-        clientId:
-          $env: AUTH_GOOGLE_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GOOGLE_CLIENT_SECRET
+        clientId: ${AUTH_GOOGLE_CLIENT_ID}
+        clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
     github:
       development:
-        clientId:
-          $env: AUTH_GITHUB_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GITHUB_CLIENT_SECRET
-        enterpriseInstanceUrl:
-          $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
+        clientId: ${AUTH_GITHUB_CLIENT_ID}
+        clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
+        enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
     gitlab:
       development:
-        clientId:
-          $env:
+        clientId: ${AUTH_GITLAB_CLIENT_ID}
     oauth2:
       development:
-        clientId:
-          $env: AUTH_OAUTH2_CLIENT_ID
-        clientSecret:
-          $env: AUTH_OAUTH2_CLIENT_SECRET
-        authorizationUrl:
-          $env: AUTH_OAUTH2_AUTH_URL
-        tokenUrl:
-          $env: AUTH_OAUTH2_TOKEN_URL
-        scope:
-          $env: AUTH_OAUTH2_SCOPE
+        clientId: ${AUTH_OAUTH2_CLIENT_ID}
+        clientSecret: ${AUTH_OAUTH2_CLIENT_SECRET}
+        authorizationUrl: ${AUTH_OAUTH2_AUTH_URL}
+        tokenUrl: ${AUTH_OAUTH2_TOKEN_URL}
+        scope: ${AUTH_OAUTH2_SCOPE}
     saml:
-      entryPoint:
-        $env: AUTH_SAML_ENTRY_POINT
-      issuer:
-        $env: AUTH_SAML_ISSUER
-            ...
+      entryPoint: ${AUTH_SAML_ENTRY_POINT}
+      issuer: ${AUTH_SAML_ISSUER}
+    ...
 ```
 
 ## Implementing Your Own Auth Wrapper
diff --git a/docs/conf/writing.md b/docs/conf/writing.md
index ac8509db0a..c7b3bfe296 100644
--- a/docs/conf/writing.md
+++ b/docs/conf/writing.md
@@ -129,6 +129,9 @@ variable.
 $env: MY_SECRET
 ```
 
+Note however, that it's often more convenient to use
+[environment variable substitution](#environment-variable-substitution) instead.
+
 ### File Includes
 
 This reads a string value from the entire contents of a text file. The file path
diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md
index c21ac7e18a..2ecd7d42be 100644
--- a/docs/features/kubernetes/configuration.md
+++ b/docs/features/kubernetes/configuration.md
@@ -25,8 +25,7 @@ kubernetes:
         - url: http://127.0.0.1:9999
           name: minikube
           authProvider: 'serviceAccount'
-          serviceAccountToken:
-            $env: K8S_MINIKUBE_TOKEN
+          serviceAccountToken: ${K8S_MINIKUBE_TOKEN}
         - url: http://127.0.0.2:9999
           name: aws-cluster-1
           authProvider: 'aws'
diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md
index b3c9f878d7..105b7e5f96 100644
--- a/docs/features/software-templates/installation.md
+++ b/docs/features/software-templates/installation.md
@@ -189,8 +189,7 @@ public within the enterprise.
 integrations:
   github:
     - host: github.com
-      token:
-        $env: GITHUB_TOKEN
+      token: ${GITHUB_TOKEN}
 
 scaffolder:
   github:
@@ -207,8 +206,7 @@ instance:
 integrations:
   gitlab:
     - host: gitlab.com
-      token:
-        $env: GITLAB_TOKEN
+      token: ${GITLAB_TOKEN}
 ```
 
 #### Bitbucket
@@ -221,8 +219,7 @@ following:
 integrations:
   bitbucket:
     - host: bitbucket.org
-      token:
-        $env: BITBUCKET_TOKEN
+      token: ${BITBUCKET_TOKEN}
 ```
 
 or
@@ -231,10 +228,8 @@ or
 integrations:
   bitbucket:
     - host: bitbucket.org
-      appPassword:
-        $env: BITBUCKET_APP_PASSWORD
-      username:
-        $env: BITBUCKET_USERNAME
+      appPassword: ${BITBUCKET_APP_PASSWORD}
+      username: ${BITBUCKET_USERNAME}
 ```
 
 #### Azure DevOps
@@ -249,8 +244,7 @@ verified.
 integrations:
   azure:
     - host: dev.azure.com
-      token:
-        $env: AZURE_TOKEN
+      token: ${AZURE_TOKEN}
 ```
 
 ### Running the Backend
diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md
index 9bf2c25926..c7876f3874 100644
--- a/docs/features/techdocs/configuration.md
+++ b/docs/features/techdocs/configuration.md
@@ -65,22 +65,18 @@ techdocs:
       # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
       # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
       credentials:
-        accessKeyId:
-          $env: TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL
-        secretAccessKey:
-          $env: TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL
+        accessKeyId: ${TECHDOCS_AWSS3_ACCESS_KEY_ID_CREDENTIAL}
+        secretAccessKey: ${TECHDOCS_AWSS3_SECRET_ACCESS_KEY_CREDENTIAL}
 
       # (Optional) AWS Region of the bucket.
       # If not set, AWS_REGION environment variable or aws config file will be used.
       # https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
-      region:
-        $env: AWS_REGION
+      region: ${AWS_REGION}
 
       # (Optional) Endpoint URI to send requests to.
       # If not set, the default endpoint is built from the configured region.
       # https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property
-      endpoint:
-        $env: AWS_ENDPOINT
+      endpoint: ${AWS_ENDPOINT}
 
     # Required when techdocs.publisher.type is set to 'azureBlobStorage'. Skip otherwise.
 
@@ -91,13 +87,11 @@ techdocs:
       # (Required) An account name is required to write to a storage blob container.
       # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key
       credentials:
-        accountName:
-          $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
+        accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
         # (Optional) An account key is required to write to a storage container.
         # If missing,AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET environment variable will be used.
         # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json
-        accountKey:
-          $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
+        accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
 
   # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc.
   # You don't have to specify this anymore.
diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md
index fd8f320859..d5a3a68425 100644
--- a/docs/features/techdocs/using-cloud-storage.md
+++ b/docs/features/techdocs/using-cloud-storage.md
@@ -95,8 +95,7 @@ techdocs:
     type: 'googleGcs'
     googleGcs:
       bucketName: 'name-of-techdocs-storage-bucket'
-      credentials:
-        $env: GOOGLE_APPLICATION_CREDENTIALS
+      credentials: ${GOOGLE_APPLICATION_CREDENTIALS}
 ```
 
 **4. That's it!**
@@ -179,13 +178,10 @@ techdocs:
     type: 'awsS3'
     awsS3:
       bucketName: 'name-of-techdocs-storage-bucket'
-      region:
-        $env: AWS_REGION
+      region: ${AWS_REGION}
       credentials:
-        accessKeyId:
-          $env: AWS_ACCESS_KEY_ID
-        secretAccessKey:
-          $env: AWS_SECRET_ACCESS_KEY
+        accessKeyId: ${AWS_ACCESS_KEY_ID}
+        secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
 ```
 
 Refer to the
@@ -202,8 +198,7 @@ techdocs:
     type: 'awsS3'
     awsS3:
       bucketName: 'name-of-techdocs-storage-bucket'
-      region:
-        $env: AWS_REGION
+      region: ${AWS_REGION}
       credentials:
         roleArn: arn:aws:iam::123456789012:role/my-backstage-role
 ```
@@ -276,8 +271,7 @@ techdocs:
     azureBlobStorage:
       containerName: 'name-of-techdocs-storage-bucket'
       credentials:
-        accountName:
-          $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
+        accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
 ```
 
 **3b. Authentication using app-config.yaml**
@@ -297,10 +291,8 @@ techdocs:
     azureBlobStorage:
       containerName: 'name-of-techdocs-storage-bucket'
       credentials:
-        accountName:
-          $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME
-        accountKey:
-          $env: TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY
+        accountName: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_NAME}
+        accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY}
 ```
 
 **4. That's it!**
@@ -361,20 +353,13 @@ techdocs:
     openStackSwift:
       containerName: 'name-of-techdocs-storage-bucket'
       credentials:
-        userName:
-          $env: OPENSTACK_SWIFT_STORAGE_USERNAME
-        password:
-          $env: OPENSTACK_SWIFT_STORAGE_PASSWORD
-      authUrl:
-        $env: OPENSTACK_SWIFT_STORAGE_AUTH_URL
-      keystoneAuthVersion:
-        $env: OPENSTACK_SWIFT_STORAGE_AUTH_VERSION
-      domainId:
-        $env: OPENSTACK_SWIFT_STORAGE_DOMAIN_ID
-      domainName:
-        $env: OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME
-      region:
-        $env: OPENSTACK_SWIFT_STORAGE_REGION
+        userName: ${OPENSTACK_SWIFT_STORAGE_USERNAME}
+        password: ${OPENSTACK_SWIFT_STORAGE_PASSWORD}
+      authUrl: ${OPENSTACK_SWIFT_STORAGE_AUTH_URL}
+      keystoneAuthVersion: ${OPENSTACK_SWIFT_STORAGE_AUTH_VERSION}
+      domainId: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_ID}
+      domainName: ${OPENSTACK_SWIFT_STORAGE_DOMAIN_NAME}
+      region: ${OPENSTACK_SWIFT_STORAGE_REGION}
 ```
 
 **4. That's it!**
diff --git a/docs/getting-started/configure-app-with-plugins.md b/docs/getting-started/configure-app-with-plugins.md
index e5b4d08a3f..cdef932127 100644
--- a/docs/getting-started/configure-app-with-plugins.md
+++ b/docs/getting-started/configure-app-with-plugins.md
@@ -60,8 +60,7 @@ proxy:
   '/circleci/api':
     target: https://circleci.com/api/v1.1
     headers:
-      Circle-Token:
-        $env: CIRCLECI_AUTH_TOKEN
+      Circle-Token: ${CIRCLECI_AUTH_TOKEN}
 ```
 
 ### Adding a plugin page to the Sidebar
diff --git a/docs/integrations/google-cloud-storage/locations.md b/docs/integrations/google-cloud-storage/locations.md
index e3f6b72847..bcdf612b9a 100644
--- a/docs/integrations/google-cloud-storage/locations.md
+++ b/docs/integrations/google-cloud-storage/locations.md
@@ -24,10 +24,8 @@ Explicit credentials can be set in the following format:
 ```yaml
 integrations:
   googleGcs:
-    clientEmail:
-      $env: GCS_CLIENT_EMAIL
-    privateKey:
-      $env: GCS_PRIVATE_KEY
+    clientEmail: ${GCS_CLIENT_EMAIL}
+    privateKey: ${GCS_PRIVATE_KEY}
 ```
 
 Then make sure the environment variables `GCS_CLIENT_EMAIL` and
diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md
index f2d7d3c12a..50c56af7a6 100644
--- a/docs/plugins/proxying.md
+++ b/docs/plugins/proxying.md
@@ -40,8 +40,9 @@ proxy:
   '/larger-example/v1':
     target: http://larger.example.com:8080/svc.v1
     headers:
-      Authorization:
-        $env: EXAMPLE_AUTH_HEADER
+      Authorization: ${EXAMPLE_AUTH_HEADER}
+      # ...or interpolating a value into part of a string,
+      # Authorization: Bearer ${EXAMPLE_AUTH_TOKEN}
 ```
 
 Each key under the proxy configuration entry is a route to match, below the
diff --git a/docs/tutorials/quickstart-app-auth.md b/docs/tutorials/quickstart-app-auth.md
index d322e2ec3c..510b697221 100644
--- a/docs/tutorials/quickstart-app-auth.md
+++ b/docs/tutorials/quickstart-app-auth.md
@@ -79,13 +79,10 @@ auth:
   providers:
     github:
       development:
-        clientId:
-          $env: AUTH_GITHUB_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GITHUB_CLIENT_SECRET
-        ## uncomment the following two lines if using enterprise
-        # enterpriseInstanceUrl:
-        #  $env: AUTH_GITHUB_ENTERPRISE_INSTANCE_URL
+        clientId: ${AUTH_GITHUB_CLIENT_ID}
+        clientSecret: ${AUTH_GITHUB_CLIENT_SECRET}
+        ## uncomment the following line if using enterprise
+        # enterpriseInstanceUrl: ${AUTH_GITHUB_ENTERPRISE_INSTANCE_URL}
 ```
 
 ### 2. Generate a GitHub client ID and secret
@@ -122,10 +119,8 @@ auth:
   providers:
     gitlab:
       development:
-        clientId:
-          $env: AUTH_GITLAB_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GITLAB_CLIENT_SECRET
+        clientId: ${AUTH_GITLAB_CLIENT_ID}
+        clientSecret: ${AUTH_GITLAB_CLIENT_SECRET}
         audience: https://gitlab.com # Or your self-hosted GitLab instance URL
 ```
 
@@ -172,10 +167,8 @@ auth:
   providers:
     google:
       development:
-        clientId:
-          $env: AUTH_GOOGLE_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GOOGLE_CLIENT_SECRET
+        clientId: ${AUTH_GOOGLE_CLIENT_ID}
+        clientSecret: ${AUTH_GOOGLE_CLIENT_SECRET}
 ```
 
 ### 2. Generate Google Credentials in Google Cloud console
@@ -216,12 +209,9 @@ auth:
   providers:
     microsoft:
       development:
-        clientId:
-          $env: AUTH_MICROSOFT_CLIENT_ID
-        clientSecret:
-          $env: AUTH_MICROSOFT_CLIENT_SECRET
-        tenantId:
-          $env: AUTH_MICROSOFT_TENANT_ID
+        clientId: ${AUTH_MICROSOFT_CLIENT_ID}
+        clientSecret: ${AUTH_MICROSOFT_CLIENT_SECRET}
+        tenantId: ${AUTH_MICROSOFT_TENANT_ID}
 ```
 
 ### 2. Create a Microsoft App Registration in Microsoft Portal
@@ -264,12 +254,9 @@ auth:
   providers:
     auth0:
       development:
-        clientId:
-          $env: AUTH_AUTH0_CLIENT_ID
-        clientSecret:
-          $env: AUTH_AUTH0_CLIENT_SECRET
-        domain:
-          $env: AUTH_AUTH0_DOMAIN_ID
+        clientId: ${AUTH_AUTH0_CLIENT_ID}
+        clientSecret: ${AUTH_AUTH0_CLIENT_SECRET}
+        domain: ${AUTH_AUTH0_DOMAIN_ID}
 ```
 
 ### 2. Create an Auth0 application in the Auth0 management console
diff --git a/docs/tutorials/switching-sqlite-postgres.md b/docs/tutorials/switching-sqlite-postgres.md
index 72e5589baf..a759b7dc49 100644
--- a/docs/tutorials/switching-sqlite-postgres.md
+++ b/docs/tutorials/switching-sqlite-postgres.md
@@ -38,14 +38,10 @@ backend:
 +    # config options: https://node-postgres.com/api/client
 +    client: pg
 +    connection:
-+      host:
-+        $env: POSTGRES_HOST
-+      port:
-+        $env: POSTGRES_PORT
-+      user:
-+        $env: POSTGRES_USER
-+      password:
-+        $env: POSTGRES_PASSWORD
++      host: ${POSTGRES_HOST}
++      port: ${POSTGRES_PORT}
++      user: ${POSTGRES_USER}
++      password: ${POSTGRES_PASSWORD}
 +      # https://node-postgres.com/features/ssl
 +      #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
 +        #ca: # if you have a CA file and want to verify it you can uncomment this section
@@ -53,9 +49,9 @@ backend:
 
 ```
 
-If you have a `app-config.local.yaml` for local development, a similar update
+If you have an `app-config.local.yaml` for local development, a similar update
 should be made there. You can set the `POSTGRES_` environment variables prior to
-launching Backstage, or remove the $env keys and simply set values directly for
-development.
+launching Backstage, or remove the `${...}` values and simply set actual values
+directly for development.
 
 The Backstage App is now ready to start up with a PostgreSQL backing database.
diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs
index e7e44c9961..442e90944e 100644
--- a/packages/create-app/templates/default-app/app-config.yaml.hbs
+++ b/packages/create-app/templates/default-app/app-config.yaml.hbs
@@ -25,14 +25,10 @@ backend:
   database:
     client: pg
     connection:
-      host:
-        $env: POSTGRES_HOST
-      port:
-        $env: POSTGRES_PORT
-      user:
-        $env: POSTGRES_USER
-      password:
-        $env: POSTGRES_PASSWORD
+      host: ${POSTGRES_HOST}
+      port: ${POSTGRES_PORT}
+      user: ${POSTGRES_USER}
+      password: ${POSTGRES_PASSWORD}
       # https://node-postgres.com/features/ssl
       #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require)
         #ca: # if you have a CA file and want to verify it you can uncomment this section
@@ -43,13 +39,11 @@ backend:
 integrations:
   github:
     - host: github.com
-      token:
-        $env: GITHUB_TOKEN
+      token: ${GITHUB_TOKEN}
     ### Example for how to add your GitHub Enterprise instance using the API:
     # - host: ghe.example.net
     #   apiBaseUrl: https://ghe.example.net/api/v3
-    #   token:
-    #     $env: GHE_TOKEN
+    #   token: ${GHE_TOKEN}
 
 proxy:
   '/test':
@@ -73,8 +67,7 @@ auth:
 
 scaffolder:
   github:
-    token:
-      $env: GITHUB_TOKEN
+    token: ${GITHUB_TOKEN}
     visibility: public # or 'internal' or 'private'
 
 catalog:
diff --git a/plugins/bitrise/README.md b/plugins/bitrise/README.md
index 2bff1f0d74..59d85b0a98 100644
--- a/plugins/bitrise/README.md
+++ b/plugins/bitrise/README.md
@@ -55,8 +55,7 @@ proxy:
     target: 'https://api.bitrise.io/v0.1'
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        $env: BITRISE_AUTH_TOKEN
+      Authorization: ${BITRISE_AUTH_TOKEN}
 ```
 
 Learn on https://devcenter.bitrise.io/api/authentication how to create a new Bitrise token.
diff --git a/plugins/circleci/README.md b/plugins/circleci/README.md
index 35fed49827..d30e950aa9 100644
--- a/plugins/circleci/README.md
+++ b/plugins/circleci/README.md
@@ -43,8 +43,7 @@ proxy:
   '/circleci/api':
     target: https://circleci.com/api/v1.1
     headers:
-      Circle-Token:
-        $env: CIRCLECI_AUTH_TOKEN
+      Circle-Token: ${CIRCLECI_AUTH_TOKEN}
 ```
 
 5. Get and provide `CIRCLECI_AUTH_TOKEN` as env variable (https://circleci.com/docs/api/#add-an-api-token)
diff --git a/plugins/fossa/README.md b/plugins/fossa/README.md
index 6e53ced0cf..722d8d2cf3 100644
--- a/plugins/fossa/README.md
+++ b/plugins/fossa/README.md
@@ -50,9 +50,7 @@ proxy:
     target: https://app.fossa.io/api
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        # Content: 'token <your-fossa-api-token>'
-        $env: FOSSA_AUTH_HEADER
+      Authorization: token ${FOSSA_API_TOKEN}
 
 # if you have a fossa organization, configure your id here
 fossa:
diff --git a/plugins/jenkins/README.md b/plugins/jenkins/README.md
index 13ff9b3547..b3bc21b7d5 100644
--- a/plugins/jenkins/README.md
+++ b/plugins/jenkins/README.md
@@ -29,15 +29,13 @@ proxy:
     target: 'http://localhost:8080' # your Jenkins URL
     changeOrigin: true
     headers:
-      Authorization:
-        $env: JENKINS_BASIC_AUTH_HEADER
+      Authorization: Basic ${JENKINS_BASIC_AUTH_HEADER}
 ```
 
 4. Add an environment variable which contains the Jenkins credentials, (note: use an API token not your password). Here user is the name of the user created in Jenkins.
 
 ```shell
-HEADER=$(echo -n user:api-token | base64)
-export JENKINS_BASIC_AUTH_HEADER="Basic $HEADER"
+export JENKINS_BASIC_AUTH_HEADER=$(echo -n user:api-token | base64)
 ```
 
 5. Run app with `yarn start`
diff --git a/plugins/newrelic/README.md b/plugins/newrelic/README.md
index e14acedef2..e7cdd0df4a 100644
--- a/plugins/newrelic/README.md
+++ b/plugins/newrelic/README.md
@@ -15,8 +15,7 @@ proxy:
   '/newrelic/apm/api':
     target: https://api.newrelic.com/v2
     headers:
-      X-Api-Key:
-        $env: NEW_RELIC_REST_API_KEY
+      X-Api-Key: ${NEW_RELIC_REST_API_KEY}
 ```
 
 In your production deployment of Backstage, you would also need to ensure that
diff --git a/plugins/rollbar-backend/README.md b/plugins/rollbar-backend/README.md
index 6cf3b55948..2dcf6495dd 100644
--- a/plugins/rollbar-backend/README.md
+++ b/plugins/rollbar-backend/README.md
@@ -8,8 +8,7 @@ The following values are read from the configuration file.
 
 ```yaml
 rollbar:
-  accountToken:
-    $env: ROLLBAR_ACCOUNT_TOKEN
+  accountToken: ${ROLLBAR_ACCOUNT_TOKEN}
 ```
 
 _NOTE: The `ROLLBAR_ACCOUNT_TOKEN` environment variable must be set to a read
diff --git a/plugins/rollbar/README.md b/plugins/rollbar/README.md
index 6337d45ffe..993b93d3a0 100644
--- a/plugins/rollbar/README.md
+++ b/plugins/rollbar/README.md
@@ -45,8 +45,7 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => (
 rollbar:
   organization: organization-name
   # used by rollbar-backend
-  accountToken:
-    $env: ROLLBAR_ACCOUNT_TOKEN
+  accountToken: ${ROLLBAR_ACCOUNT_TOKEN}
 ```
 
 6. Annotate entities with the rollbar project slug
diff --git a/plugins/sentry/README.md b/plugins/sentry/README.md
index 5d280de249..52cef9cbe6 100644
--- a/plugins/sentry/README.md
+++ b/plugins/sentry/README.md
@@ -70,9 +70,7 @@ proxy:
     target: https://sentry.io/api/
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        # Content: 'Bearer <your-sentry-token>'
-        $env: SENTRY_TOKEN
+      Authorization: Bearer ${SENTRY_TOKEN}
 
 sentry:
   organization: <your-organization>
diff --git a/plugins/sonarqube/README.md b/plugins/sonarqube/README.md
index f95a2c6151..269fb7c859 100644
--- a/plugins/sonarqube/README.md
+++ b/plugins/sonarqube/README.md
@@ -54,10 +54,9 @@ proxy:
     target: https://sonarcloud.io/api
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        # Content: 'Basic base64("<api-key>:")' <-- note the trailing ':'
-        # Example: Basic bXktYXBpLWtleTo=
-        $env: SONARQUBE_AUTH_HEADER
+      Authorization: Basic ${SONARQUBE_AUTH}
+      # Content: 'base64("<api-key>:")' <-- note the trailing ':'
+      # Example: bXktYXBpLWtleTo=
 ```
 
 **SonarQube**
@@ -70,20 +69,19 @@ proxy:
     target: https://your.sonarqube.instance.com/api
     allowedMethods: ['GET']
     headers:
-      Authorization:
-        # Environmental variable: SONARQUBE_AUTH_HEADER
-        # Value: 'Basic base64("<sonar-auth-token>:")'
-        # Encode the "<sonar-auth-token>:" string using base64 encoder.
-        # Note the trailing colon (:) at the end of the token.
-        # Example environmental config: SONARQUBE_AUTH_HEADER=Basic bXktYXBpLWtleTo=
-        # Fetch the sonar-auth-token from https://sonarcloud.io/account/security/
-        $env: SONARQUBE_AUTH_HEADER
+      Authorization: Basic ${SONARQUBE_AUTH}
+      # Environmental variable: SONARQUBE_AUTH
+      # Value: 'base64("<sonar-auth-token>:")'
+      # Encode the "<sonar-auth-token>:" string using base64 encoder.
+      # Note the trailing colon (:) at the end of the token.
+      # Example environmental config: SONARQUBE_AUTH=bXktYXBpLWtleTo=
+      # Fetch the sonar-auth-token from https://sonarcloud.io/account/security/
 
 sonarQube:
   baseUrl: https://your.sonarqube.instance.com
 ```
 
-5. Get and provide `SONARQUBE_AUTH_HEADER` as env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/)
+5. Get and provide `SONARQUBE_AUTH` as an env variable (https://sonarcloud.io/account/security or https://docs.sonarqube.org/latest/user-guide/user-token/)
 
 6. Run the following commands in the root folder of the project to install and compile the changes.
 
diff --git a/plugins/splunk-on-call/README.md b/plugins/splunk-on-call/README.md
index 78313d3329..5c1dda33fc 100644
--- a/plugins/splunk-on-call/README.md
+++ b/plugins/splunk-on-call/README.md
@@ -50,7 +50,7 @@ import {
 
 In order to be able to perform certain action (create-acknowledge-resolve an action), you need to provide a REST Endpoint.
 
-To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST – Generic.  
+To enable the REST Endpoint integration you can go on https://portal.victorops.com/ inside Integrations > 3rd Party Integrations > REST – Generic.
 You can now copy the URL to notify: `<SPLUNK_ON_CALL_REST_ENDPOINT>/$routing_key`
 
 In `app-config.yaml`:
@@ -69,10 +69,8 @@ proxy:
   '/splunk-on-call':
     target: https://api.victorops.com/api-public
     headers:
-      X-VO-Api-Id:
-        $env: SPLUNK_ON_CALL_API_ID
-      X-VO-Api-Key:
-        $env: SPLUNK_ON_CALL_API_KEY
+      X-VO-Api-Id: ${SPLUNK_ON_CALL_API_ID}
+      X-VO-Api-Key: ${SPLUNK_ON_CALL_API_KEY}
 ```
 
 In addition, to make certain API calls (trigger-resolve-acknowledge an incident) you need to add the `PATCH` method to the backend `cors` methods list: `[GET, POST, PUT, DELETE, PATCH]`.
diff --git a/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx b/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx
index 653f449e92..55c83eff3e 100644
--- a/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx
+++ b/plugins/user-settings/src/components/AuthProviders/EmptyProviders.tsx
@@ -22,10 +22,8 @@ const EXAMPLE = `auth:
   providers:
     google:
       development:
-        clientId:
-          $env: AUTH_GOOGLE_CLIENT_ID
-        clientSecret:
-          $env: AUTH_GOOGLE_CLIENT_SECRET
+        clientId: \${AUTH_GOOGLE_CLIENT_ID}
+        clientSecret: \${AUTH_GOOGLE_CLIENT_SECRET}
 `;
 
 export const EmptyProviders = () => (