Merge branch 'master' of github.com:backstage/backstage into readersprocessorstasks

packages/backend-common/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @backstage/backend-common
 
+## 0.10.2
+
+### Patch Changes
+
+- 21ae56168e: Updated the Git class with the following:
+
+  - Added `depth` and `noCheckout` options to Git clone, using these you can create a bare clone that includes just the git history
+  - New `log` function which you can use to view the commit history of a git repo
+
+- eacc582473: Reverted the default CSP configuration to include `'unsafe-eval'` again, which was mistakenly removed in the previous version.
+
+## 0.10.1
+
+### Patch Changes
+
+- 94cdf5d1bd: In-memory cache clients instantiated from the same cache manager now share the same memory space.
+- 916b2f1f3e: Use the default CSP policy provided by `helmet` directly rather than a copy.
+- 7d4b4e937c: Uptake changes to the GitHub Credentials Provider interface.
+- 995e4c7d9d: Added support for non-"amazonaws.com" hosts (for example when testing with LocalStack) in AwsS3UrlReader.
+- Updated dependencies
+  - @backstage/integration@0.7.0
+  - @backstage/config-loader@0.9.1
+
 ## 0.10.0
 
 ### Minor Changes

packages/backend-common/api-report.md

@@ -34,9 +34,7 @@ import { Server } from 'http';
 import * as winston from 'winston';
 import { Writable } from 'stream';
 
-// Warning: (ae-missing-release-tag) "AwsS3UrlReader" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @public
 export class AwsS3UrlReader implements UrlReader {
   constructor(
     integration: AwsS3Integration,
@@ -59,7 +57,7 @@ export class AwsS3UrlReader implements UrlReader {
   toString(): string;
 }
 
-// @public (undocumented)
+// @public
 export class AzureUrlReader implements UrlReader {
   constructor(
     integration: AzureIntegration,
@@ -114,12 +112,12 @@ export interface CacheClient {
   ): Promise<void>;
 }
 
-// @public (undocumented)
+// @public
 export type CacheClientOptions = {
   defaultTtl?: number;
 };
 
-// @public (undocumented)
+// @public
 export type CacheClientSetOptions = {
   ttl?: number;
 };
@@ -133,18 +131,17 @@ export class CacheManager {
   ): CacheManager;
 }
 
-// @public (undocumented)
+// @public
 export type CacheManagerOptions = {
   logger?: Logger_2;
   onError?: (err: Error) => void;
 };
 
-// @public (undocumented)
+// @public
 export const coloredFormat: winston.Logform.Format;
 
-// @public (undocumented)
+// @public
 export interface ContainerRunner {
-  // (undocumented)
   runContainer(opts: RunContainerOptions): Promise<void>;
 }
 
@@ -157,7 +154,7 @@ export function createDatabaseClient(
   overrides?: Partial<Knex.Config>,
 ): Knex<any, unknown[]>;
 
-// @public (undocumented)
+// @public
 export function createRootLogger(
   options?: winston.LoggerOptions,
   env?: NodeJS.ProcessEnv,
@@ -166,14 +163,14 @@ export function createRootLogger(
 // @public
 export function createServiceBuilder(_module: NodeModule): ServiceBuilder;
 
-// @public (undocumented)
+// @public
 export function createStatusCheckRouter(options: {
   logger: Logger_2;
   path?: string;
   statusCheck?: StatusCheck;
 }): Promise<express.Router>;
 
-// @public (undocumented)
+// @public
 export class DatabaseManager {
   forPlugin(pluginId: string): PluginDatabaseManager;
   static fromConfig(
@@ -187,7 +184,7 @@ export type DatabaseManagerOptions = {
   migrations?: PluginDatabaseManager['migrations'];
 };
 
-// @public (undocumented)
+// @public
 export class DockerContainerRunner implements ContainerRunner {
   constructor(options: { dockerClient: Docker });
   // (undocumented)
@@ -205,7 +202,7 @@ export function errorHandler(
   options?: ErrorHandlerOptions,
 ): ErrorRequestHandler;
 
-// @public (undocumented)
+// @public
 export type ErrorHandlerOptions = {
   showStackTraces?: boolean;
   logger?: Logger_2;
@@ -218,13 +215,13 @@ export type FromReadableArrayOptions = Array<{
   path: string;
 }>;
 
-// @public (undocumented)
+// @public
 export function getRootLogger(): winston.Logger;
 
 // @public
 export function getVoidLogger(): winston.Logger;
 
-// @public (undocumented)
+// @public
 export class Git {
   // (undocumented)
   add(options: { dir: string; filepath: string }): Promise<void>;
@@ -234,8 +231,13 @@ export class Git {
     remote: string;
     url: string;
   }): Promise<void>;
-  // (undocumented)
-  clone(options: { url: string; dir: string; ref?: string }): Promise<void>;
+  clone(options: {
+    url: string;
+    dir: string;
+    ref?: string;
+    depth?: number;
+    noCheckout?: boolean;
+  }): Promise<void>;
   // (undocumented)
   commit(options: {
     dir: string;
@@ -249,12 +251,10 @@ export class Git {
       email: string;
     };
   }): Promise<string>;
-  // (undocumented)
   currentBranch(options: {
     dir: string;
     fullName?: boolean;
   }): Promise<string | undefined>;
-  // (undocumented)
   fetch(options: { dir: string; remote?: string }): Promise<void>;
   // (undocumented)
   static fromAuth: (options: {
@@ -264,7 +264,7 @@ export class Git {
   }) => Git;
   // (undocumented)
   init(options: { dir: string; defaultBranch?: string }): Promise<void>;
-  // (undocumented)
+  log(options: { dir: string; ref?: string }): Promise<ReadCommitResult[]>;
   merge(options: {
     dir: string;
     theirs: string;
@@ -280,9 +280,7 @@ export class Git {
   }): Promise<MergeResult>;
   // (undocumented)
   push(options: { dir: string; remote: string }): Promise<PushResult>;
-  // (undocumented)
   readCommit(options: { dir: string; sha: string }): Promise<ReadCommitResult>;
-  // (undocumented)
   resolveRef(options: { dir: string; ref: string }): Promise<string>;
 }
 
@@ -309,7 +307,7 @@ export class GithubUrlReader implements UrlReader {
   toString(): string;
 }
 
-// @public (undocumented)
+// @public
 export class GitlabUrlReader implements UrlReader {
   constructor(
     integration: GitLabIntegration,
@@ -397,7 +395,7 @@ export type ReadTreeResponseDirOptions = {
   targetDir?: string;
 };
 
-// @public (undocumented)
+// @public
 export interface ReadTreeResponseFactory {
   // (undocumented)
   fromReadableArray(
@@ -447,7 +445,7 @@ export type ReadUrlResponse = {
 // @public
 export function requestLoggingHandler(logger?: Logger_2): RequestHandler;
 
-// @public (undocumented)
+// @public
 export type RequestLoggingHandlerFactory = (
   logger?: Logger_2,
 ) => RequestHandler;
@@ -458,7 +456,7 @@ export function resolvePackagePath(name: string, ...paths: string[]): string;
 // @public
 export function resolveSafeChildPath(base: string, path: string): string;
 
-// @public (undocumented)
+// @public
 export type RunContainerOptions = {
   imageName: string;
   command?: string | string[];
@@ -507,7 +505,7 @@ export class ServerTokenManager implements TokenManager {
   static noop(): TokenManager;
 }
 
-// @public (undocumented)
+// @public
 export type ServiceBuilder = {
   loadConfig(config: Config): ServiceBuilder;
   setPort(port: number): ServiceBuilder;
@@ -533,7 +531,7 @@ export type ServiceBuilder = {
   start(): Promise<Server>;
 };
 
-// @public (undocumented)
+// @public
 export function setRootLogger(newLogger: winston.Logger): void;
 
 // @public @deprecated
@@ -553,7 +551,7 @@ export class SingleHostDiscovery implements PluginEndpointDiscovery {
   getExternalBaseUrl(pluginId: string): Promise<string>;
 }
 
-// @public (undocumented)
+// @public
 export type StatusCheck = () => Promise<any>;
 
 // @public
@@ -561,7 +559,7 @@ export function statusCheckHandler(
   options?: StatusCheckHandlerOptions,
 ): Promise<RequestHandler>;
 
-// @public (undocumented)
+// @public
 export interface StatusCheckHandlerOptions {
   statusCheck?: StatusCheck;
 }
@@ -596,7 +594,7 @@ export class UrlReaders {
   static default(options: UrlReadersOptions): UrlReader;
 }
 
-// @public (undocumented)
+// @public
 export type UrlReadersOptions = {
   config: Config;
   logger: Logger_2;
@@ -611,8 +609,4 @@ export function useHotCleanup(
 
 // @public
 export function useHotMemoize<T>(_module: NodeModule, valueFactory: () => T): T;
-
-// Warnings were encountered during analysis:
-//
-// src/database/types.d.ts:23:12 - (tsdoc-undefined-tag) The TSDoc tag "@default" is not defined in this configuration
 ```

packages/backend-common/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@backstage/backend-common",
   "description": "Common functionality library for Backstage backends",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "main": "src/index.ts",
   "types": "src/index.ts",
   "private": false,
@@ -31,12 +31,12 @@
   "dependencies": {
     "@backstage/cli-common": "^0.1.6",
     "@backstage/config": "^0.1.11",
-    "@backstage/config-loader": "^0.9.0",
+    "@backstage/config-loader": "^0.9.1",
     "@backstage/errors": "^0.1.5",
-    "@backstage/integration": "^0.6.10",
+    "@backstage/integration": "^0.7.0",
     "@backstage/types": "^0.1.1",
     "@google-cloud/storage": "^5.8.0",
-    "@lerna/project": "^4.0.0",
+    "@manypkg/get-packages": "^1.1.3",
     "@octokit/rest": "^18.5.3",
     "@types/cors": "^2.8.6",
     "@types/dockerode": "^3.3.0",
@@ -58,7 +58,7 @@
     "keyv-memcache": "^1.2.5",
     "knex": "^0.95.1",
     "lodash": "^4.17.21",
-    "logform": "^2.1.1",
+    "logform": "^2.3.2",
     "minimatch": "^3.0.4",
     "minimist": "^1.2.5",
     "morgan": "^1.10.0",
@@ -81,8 +81,8 @@
     }
   },
   "devDependencies": {
-    "@backstage/cli": "^0.10.3",
-    "@backstage/test-utils": "^0.2.0",
+    "@backstage/cli": "^0.10.5",
+    "@backstage/test-utils": "^0.2.1",
     "@types/archiver": "^5.1.0",
     "@types/compression": "^1.7.0",
     "@types/concat-stream": "^1.6.0",

packages/backend-common/src/cache/CacheClient.ts

@@ -22,7 +22,11 @@ type CacheClientArgs = {
   client: Keyv;
 };
 
-/** @public */
+/**
+ * Options passed to {@link CacheClient.set}.
+ *
+ * @public
+ */
 export type CacheClientSetOptions = {
   /**
    * Optional TTL in milliseconds. Defaults to the TTL provided when the client

packages/backend-common/src/cache/CacheManager.test.ts

@@ -147,6 +147,22 @@ describe('CacheManager', () => {
       });
     });
 
+    it('shares memory across multiple instances of the memory client', () => {
+      const manager = CacheManager.fromConfig(defaultConfig());
+      const plugin = 'test-plugin';
+
+      // Instantiate two in-memory clients.
+      manager.forPlugin(plugin).getClient({ defaultTtl: 10 });
+      manager.forPlugin(plugin).getClient({ defaultTtl: 10 });
+
+      const cache = Keyv as unknown as jest.Mock;
+      const mockCall2 = cache.mock.calls.splice(-1)[0][0];
+      const mockCall1 = cache.mock.calls.splice(-1)[0][0];
+
+      // Note: .toBe() checks referential identity of object instances.
+      expect(mockCall1.store).toBe(mockCall2.store);
+    });
+
     it('returns a memcache client when configured', () => {
       const expectedHost = '127.0.0.1:11211';
       const manager = CacheManager.fromConfig(

packages/backend-common/src/cache/CacheManager.ts

@@ -42,14 +42,21 @@ export class CacheManager {
     none: this.getNoneClient,
   };
 
+  /**
+   * Shared memory store for the in-memory cache client. Sharing the same Map
+   * instance ensures get/set/delete operations hit the same store, regardless
+   * of where/when a client is instantiated.
+   */
+  private readonly memoryStore = new Map();
+
   private readonly logger: Logger;
   private readonly store: keyof CacheManager['storeFactories'];
   private readonly connection: string;
   private readonly errorHandler: CacheManagerOptions['onError'];
 
   /**
-   * Creates a new CacheManager instance by reading from the `backend` config
-   * section, specifically the `.cache` key.
+   * Creates a new {@link CacheManager} instance by reading from the `backend`
+   * config section, specifically the `.cache` key.
    *
    * @param config - The loaded application configuration.
    */
@@ -86,7 +93,8 @@ export class CacheManager {
   /**
    * Generates a PluginCacheManager for consumption by plugins.
    *
-   * @param pluginId - The plugin that the cache manager should be created for. Plugin names should be unique.
+   * @param pluginId - The plugin that the cache manager should be created for.
+   *        Plugin names should be unique.
    */
   forPlugin(pluginId: string): PluginCacheManager {
     return {
@@ -133,6 +141,7 @@ export class CacheManager {
     return new Keyv({
       namespace: pluginId,
       ttl: defaultTtl,
+      store: this.memoryStore,
     });
   }
 

packages/backend-common/src/cache/types.ts

@@ -17,7 +17,11 @@
 import { Logger } from 'winston';
 import { CacheClient } from './CacheClient';
 
-/** @public */
+/**
+ * Options given when constructing a {@link CacheClient}.
+ *
+ * @public
+ */
 export type CacheClientOptions = {
   /**
    * An optional default TTL (in milliseconds) to be set when getting a client
@@ -27,7 +31,11 @@ export type CacheClientOptions = {
   defaultTtl?: number;
 };
 
-/** @public */
+/**
+ * Options given when constructing a {@link CacheManager}.
+ *
+ * @public
+ */
 export type CacheManagerOptions = {
   /**
    * An optional logger for use by the PluginCacheManager.
@@ -42,17 +50,19 @@ export type CacheManagerOptions = {
 };
 
 /**
- * The PluginCacheManager manages access to cache stores that Plugins get.
+ * Manages access to cache stores that plugins get.
  *
  * @public
  */
 export type PluginCacheManager = {
   /**
-   * getClient provides backend plugins cache connections for itself.
+   * Provides backend plugins cache connections for themselves.
    *
-   * The purpose of this method is to allow plugins to get isolated data
-   * stores so that plugins are discouraged from cache-level integration
-   * and/or cache key collisions.
+   * @remarks
+   *
+   * The purpose of this method is to allow plugins to get isolated data stores
+   * so that plugins are discouraged from cache-level integration and/or cache
+   * key collisions.
    */
   getClient: (options?: CacheClientOptions) => CacheClient;
 };

packages/backend-common/src/config.ts

@@ -27,6 +27,7 @@ import {
 } from '@backstage/config-loader';
 import { AppConfig, Config, ConfigReader } from '@backstage/config';
 import { JsonValue } from '@backstage/types';
+import { getPackages } from '@manypkg/get-packages';
 
 import { isValidUrl } from './urls';
 
@@ -194,11 +195,9 @@ export async function loadBackendConfig(options: {
   // TODO(hhogg): This is fetching _all_ of the packages of the monorepo
   // in order to find the secrets for redactions, however we only care about
   // the backend ones, we need to find a way to exclude the frontend packages.
-  const { Project } = require('@lerna/project');
-  const project = new Project(paths.targetDir);
-  const packages = await project.getPackages();
+  const { packages } = await getPackages(paths.targetDir);
   const schema = await loadConfigSchema({
-    dependencies: packages.map((p: any) => p.name),
+    dependencies: packages.map(p => p.packageJson.name),
   });
 
   const config = new ObservableConfigProxy(options.logger);

packages/backend-common/src/database/DatabaseManager.ts

@@ -38,7 +38,7 @@ function pluginPath(pluginId: string): string {
 }
 
 /**
- * Configuration options object.
+ * Creation options for {@link DatabaseManager}.
  *
  * @public
  */
@@ -46,15 +46,20 @@ export type DatabaseManagerOptions = {
   migrations?: PluginDatabaseManager['migrations'];
 };
 
-/** @public */
+/**
+ * Manages database connections for Backstage backend plugins.
+ *
+ * The database manager allows the user to set connection and client settings on
+ * a per pluginId basis by defining a database config block under
+ * `plugin.<pluginId>` in addition to top level defaults. Optionally, a user may
+ * set `prefix` which is used to prefix generated database names if config is
+ * not provided.
+ *
+ * @public
+ */
 export class DatabaseManager {
   /**
-   * Creates a DatabaseManager from `backend.database` config.
-   *
-   * The database manager allows the user to set connection and client settings on a per pluginId
-   * basis by defining a database config block under `plugin.<pluginId>` in addition to top level
-   * defaults. Optionally, a user may set `prefix` which is used to prefix generated database
-   * names if config is not provided.
+   * Creates a {@link DatabaseManager} from `backend.database` config.
    *
    * @param config - The loaded application configuration.
    * @param options - An optional configuration object.
@@ -108,7 +113,7 @@ export class DatabaseManager {
    * which is the pluginId prefixed with 'backstage_plugin_'. If `pluginDivisionMode` is
    * `schema`, it will fallback to using the default database for the knex instance.
    *
-   * @param pluginId Lookup the database name for given plugin
+   * @param pluginId - Lookup the database name for given plugin
    * @returns String representing the plugin's database name
    */
   private getDatabaseName(pluginId: string): string | undefined {
@@ -143,12 +148,13 @@ export class DatabaseManager {
   /**
    * Provides the client type which should be used for a given plugin.
    *
-   * The client type is determined by plugin specific config if present. Otherwise the base
-   * client is used as the fallback.
+   * The client type is determined by plugin specific config if present.
+   * Otherwise the base client is used as the fallback.
    *
-   * @param pluginId Plugin to get the client type for
-   * @returns Object with client type returned as `client` and boolean representing whether
-   * or not the client was overridden as `overridden`
+   * @param pluginId - Plugin to get the client type for
+   * @returns Object with client type returned as `client` and boolean
+   *          representing whether or not the client was overridden as
+   *          `overridden`
    */
   private getClientType(pluginId: string): {
     client: string;
@@ -169,8 +175,8 @@ export class DatabaseManager {
   /**
    * Provides the knexConfig which should be used for a given plugin.
    *
-   * @param pluginId Plugin to get the knexConfig for
-   * @returns the merged kexConfig value or undefined if it isn't specified
+   * @param pluginId - Plugin to get the knexConfig for
+   * @returns The merged knexConfig value or undefined if it isn't specified
    */
   private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
     const pluginConfig = this.config
@@ -197,13 +203,15 @@ export class DatabaseManager {
   }
 
   /**
-   * Provides a Knex connection plugin config by combining base and plugin config.
+   * Provides a Knex connection plugin config by combining base and plugin
+   * config.
    *
-   * This method provides a baseConfig for a plugin database connector. If the client type
-   * has not been overridden, the global connection config will be included with plugin
-   * specific config as the base. Values from the plugin connection take precedence over the
-   * base. Base database name is omitted for all supported databases excluding SQLite unless
-   * `pluginDivisionMode` is set to `schema`.
+   * This method provides a baseConfig for a plugin database connector. If the
+   * client type has not been overridden, the global connection config will be
+   * included with plugin specific config as the base. Values from the plugin
+   * connection take precedence over the base. Base database name is omitted for
+   * all supported databases excluding SQLite unless `pluginDivisionMode` is set
+   * to `schema`.
    */
   private getConnectionConfig(
     pluginId: string,
@@ -249,9 +257,10 @@ export class DatabaseManager {
   /**
    * Provides a Knex database config for a given plugin.
    *
-   * This method provides a Knex configuration object along with the plugin's client type.
+   * This method provides a Knex configuration object along with the plugin's
+   * client type.
    *
-   * @param pluginId The plugin that the database config should correspond with
+   * @param pluginId - The plugin that the database config should correspond with
    */
   private getConfigForPlugin(pluginId: string): Knex.Config {
     const { client } = this.getClientType(pluginId);
@@ -264,20 +273,21 @@ export class DatabaseManager {
   }
 
   /**
-   * Provides a partial Knex.Config database schema override for a given plugin.
+   * Provides a partial `Knex.Config` database schema override for a given
+   * plugin.
    *
-   * @param pluginId Target plugin to get database schema override
-   * @returns Partial Knex.Config with database schema override
+   * @param pluginId - Target plugin to get database schema override
+   * @returns Partial `Knex.Config` with database schema override
    */
   private getSchemaOverrides(pluginId: string): Knex.Config | undefined {
     return createSchemaOverride(this.getClientType(pluginId).client, pluginId);
   }
 
   /**
-   * Provides a partial Knex.Config database name override for a given plugin.
+   * Provides a partial `Knex.Config`• database name override for a given plugin.
    *
-   * @param pluginId Target plugin to get database name override
-   * @returns Partial Knex.Config with database name override
+   * @param pluginId - Target plugin to get database name override
+   * @returns Partial `Knex.Config` with database name override
    */
   private getDatabaseOverrides(pluginId: string): Knex.Config {
     const databaseName = this.getDatabaseName(pluginId);
@@ -289,8 +299,9 @@ export class DatabaseManager {
   /**
    * Provides a scoped Knex client for a plugin as per application config.
    *
-   *  @param pluginId Plugin to get a Knex client for
-   *  @returns Promise which resolves to a scoped Knex database client for a plugin
+   * @param pluginId - Plugin to get a Knex client for
+   * @returns Promise which resolves to a scoped Knex database client for a
+   *          plugin
    */
   private async getDatabase(pluginId: string): Promise<Knex> {
     const pluginConfig = new ConfigReader(

packages/backend-common/src/database/config.ts

@@ -20,8 +20,8 @@ import { merge } from 'lodash';
  * Merges database objects together
  *
  * @public
- * @param config The base config. The input is not modified
- * @param overrides Any additional overrides
+ * @param config - The base config. The input is not modified
+ * @param overrides - Any additional overrides
  */
 export function mergeDatabaseConfig(config: any, ...overrides: any[]) {
   return merge({}, config, ...overrides);

packages/backend-common/src/database/connection.ts

@@ -58,7 +58,7 @@ export function createDatabaseClient(
 }
 
 /**
- * Alias for createDatabaseClient
+ * Alias for {@link createDatabaseClient}
  *
  * @public
  * @deprecated Use createDatabaseClient instead
@@ -100,7 +100,8 @@ export async function ensureSchemaExists(
 }
 
 /**
- * Provides a Knex.Config object with the provided database name for a given client.
+ * Provides a `Knex.Config` object with the provided database name for a given
+ * client.
  */
 export function createNameOverride(
   client: string,
@@ -117,7 +118,8 @@ export function createNameOverride(
 }
 
 /**
- * Provides a Knex.Config object with the provided database schema for a given client. Currently only supported by `pg`.
+ * Provides a `Knex.Config` object with the provided database schema for a given
+ * client. Currently only supported by `pg`.
  */
 export function createSchemaOverride(
   client: string,
@@ -156,7 +158,8 @@ export function parseConnectionString(
 }
 
 /**
- * Normalizes a connection config or string into an object which can be passed to Knex.
+ * Normalizes a connection config or string into an object which can be passed
+ * to Knex.
  */
 export function normalizeConnection(
   connection: Knex.StaticConnectionConfig | JsonObject | string | undefined,

packages/backend-common/src/database/connectors/defaultNameOverride.ts

@@ -21,7 +21,7 @@ import { Knex } from 'knex';
  * Default override for knex database drivers which accept ConnectionConfig
  * with `connection.database` as the database name field.
  *
- * @param name database name to get config override for
+ * @param name - database name to get config override for
  */
 export default function defaultNameOverride(
   name: string,

packages/backend-common/src/database/connectors/defaultSchemaOverride.ts

@@ -18,7 +18,7 @@ import { Knex } from 'knex';
 /**
  * Provides a partial knex config with schema name override.
  *
- * @param name schema name to get config override for
+ * @param name - schema name to get config override for
  */
 export default function defaultSchemaOverride(
   name: string,

packages/backend-common/src/database/connectors/mysql.ts

@@ -26,8 +26,8 @@ import defaultNameOverride from './defaultNameOverride';
 /**
  * Creates a knex mysql database connection
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function createMysqlDatabaseClient(
   dbConfig: Config,
@@ -41,8 +41,8 @@ export function createMysqlDatabaseClient(
 /**
  * Builds a knex mysql database connection
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function buildMysqlDatabaseConfig(
   dbConfig: Config,
@@ -61,8 +61,8 @@ export function buildMysqlDatabaseConfig(
 /**
  * Gets the mysql connection config
  *
- * @param dbConfig The database config
- * @param parseConnectionString Flag to explicitly control connection string parsing
+ * @param dbConfig - The database config
+ * @param parseConnectionString - Flag to explicitly control connection string parsing
  */
 export function getMysqlConnectionConfig(
   dbConfig: Config,
@@ -86,7 +86,7 @@ export function getMysqlConnectionConfig(
  * Parses a mysql connection string.
  *
  * e.g. mysql://examplename:somepassword@examplehost:3306/dbname
- * @param connectionString The mysql connection string
+ * @param connectionString - The mysql connection string
  */
 export function parseMysqlConnectionString(
   connectionString: string,
@@ -140,8 +140,8 @@ export function parseMysqlConnectionString(
 /**
  * Creates the missing mysql database if it does not exist
  *
- * @param dbConfig The database config
- * @param databases The names of the databases to create
+ * @param dbConfig - The database config
+ * @param databases - The names of the databases to create
  */
 export async function ensureMysqlDatabaseExists(
   dbConfig: Config,

packages/backend-common/src/database/connectors/postgres.ts

@@ -26,8 +26,8 @@ import defaultSchemaOverride from './defaultSchemaOverride';
 /**
  * Creates a knex postgres database connection
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function createPgDatabaseClient(
   dbConfig: Config,
@@ -41,8 +41,8 @@ export function createPgDatabaseClient(
 /**
  * Builds a knex postgres database connection
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function buildPgDatabaseConfig(
   dbConfig: Config,
@@ -61,8 +61,8 @@ export function buildPgDatabaseConfig(
 /**
  * Gets the postgres connection config
  *
- * @param dbConfig The database config
- * @param parseConnectionString Flag to explicitly control connection string parsing
+ * @param dbConfig - The database config
+ * @param parseConnectionString - Flag to explicitly control connection string parsing
  */
 export function getPgConnectionConfig(
   dbConfig: Config,
@@ -85,7 +85,7 @@ export function getPgConnectionConfig(
 /**
  * Parses a connection string using pg-connection-string
  *
- * @param connectionString The postgres connection string
+ * @param connectionString - The postgres connection string
  */
 export function parsePgConnectionString(connectionString: string) {
   const parse = requirePgConnectionString();
@@ -103,8 +103,8 @@ function requirePgConnectionString() {
 /**
  * Creates the missing Postgres database if it does not exist
  *
- * @param dbConfig The database config
- * @param databases The name of the databases to create
+ * @param dbConfig - The database config
+ * @param databases - The name of the databases to create
  */
 export async function ensurePgDatabaseExists(
   dbConfig: Config,
@@ -139,8 +139,8 @@ export async function ensurePgDatabaseExists(
 /**
  * Creates the missing Postgres schema if it does not exist
  *
- * @param dbConfig The database config
- * @param schemas The name of the schemas to create
+ * @param dbConfig - The database config
+ * @param schemas - The name of the schemas to create
  */
 export async function ensurePgSchemaExists(
   dbConfig: Config,

packages/backend-common/src/database/connectors/sqlite3.ts

@@ -25,8 +25,8 @@ import { DatabaseConnector } from '../types';
 /**
  * Creates a knex SQLite3 database connection
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function createSqliteDatabaseClient(
   dbConfig: Config,
@@ -58,8 +58,8 @@ export function createSqliteDatabaseClient(
 /**
  * Builds a knex SQLite3 connection config
  *
- * @param dbConfig The database config
- * @param overrides Additional options to merge with the config
+ * @param dbConfig - The database config
+ * @param overrides - Additional options to merge with the config
  */
 export function buildSqliteDatabaseConfig(
   dbConfig: Config,

packages/backend-common/src/database/types.ts

@@ -38,7 +38,7 @@ export interface PluginDatabaseManager {
     /**
      * skip database migrations. Useful if connecting to a read-only database.
      *
-     * @default false
+     * @defaultValue false
      */
     skip?: boolean;
   };

packages/backend-common/src/logging/formats.ts

@@ -31,7 +31,11 @@ const coloredTemplate = (info: TransformableInfo) => {
   return `${timestampColor} ${prefixColor} ${level} ${message} ${extraFields}`;
 };
 
-/** @public */
+/**
+ * A logging format that adds coloring to console output.
+ *
+ * @public
+ */
 export const coloredFormat = winston.format.combine(
   winston.format.timestamp(),
   winston.format.colorize({

packages/backend-common/src/logging/rootLogger.ts

@@ -23,12 +23,29 @@ import { escapeRegExp } from '../util/escapeRegExp';
 let rootLogger: winston.Logger;
 let redactionRegExp: RegExp | undefined;
 
-/** @public */
+/**
+ * Gets the current root logger.
+ *
+ * @public
+ */
 export function getRootLogger(): winston.Logger {
   return rootLogger;
 }
 
-/** @public */
+/**
+ * Sets a completely custom default "root" logger.
+ *
+ * @remarks
+ *
+ * This is the logger instance that will be the foundation for all other logger
+ * instances passed to plugins etc, in a given backend.
+ *
+ * Only use this if you absolutely need to make a completely custom logger.
+ * Normally if you want to make light adaptations to the default logger
+ * behavior, you would instead call {@link createRootLogger}.
+ *
+ * @public
+ */
 export function setRootLogger(newLogger: winston.Logger) {
   rootLogger = newLogger;
 }
@@ -67,7 +84,17 @@ function redactLogLine(info: winston.Logform.TransformableInfo) {
   return info;
 }
 
-/** @public */
+/**
+ * Creates a default "root" logger. This also calls {@link setRootLogger} under
+ * the hood.
+ *
+ * @remarks
+ *
+ * This is the logger instance that will be the foundation for all other logger
+ * instances passed to plugins etc, in a given backend.
+ *
+ * @public
+ */
 export function createRootLogger(
   options: winston.LoggerOptions = {},
   env = process.env,

packages/backend-common/src/middleware/errorHandler.ts

@@ -17,7 +17,7 @@
 import {
   AuthenticationError,
   ConflictError,
-  ErrorResponse,
+  ErrorResponseBody,
   InputError,
   NotAllowedError,
   NotFoundError,
@@ -28,7 +28,11 @@ import { ErrorRequestHandler, NextFunction, Request, Response } from 'express';
 import { Logger } from 'winston';
 import { getRootLogger } from '../logging';
 
-/** @public */
+/**
+ * Options passed to the {@link errorHandler} middleware.
+ *
+ * @public
+ */
 export type ErrorHandlerOptions = {
   /**
    * Whether error response bodies should show error stack traces or not.
@@ -89,7 +93,7 @@ export function errorHandler(
       return;
     }
 
-    const body: ErrorResponse = {
+    const body: ErrorResponseBody = {
       error: serializeError(error, { includeStack: showStackTraces }),
       request: { method: req.method, url: req.url },
       response: { statusCode },

packages/backend-common/src/middleware/statusCheckHandler.ts

@@ -16,10 +16,19 @@
 
 import { NextFunction, Request, Response, RequestHandler } from 'express';
 
-/** @public */
+/**
+ * A custom status checking function, passed to {@link statusCheckHandler} and
+ * {@link createStatusCheckRouter}.
+ *
+ * @public
+ */
 export type StatusCheck = () => Promise<any>;
 
-/** @public */
+/**
+ * Options passed to {@link statusCheckHandler}.
+ *
+ * @public
+ */
 export interface StatusCheckHandlerOptions {
   /**
    * Optional status function which returns a message.

packages/backend-common/src/reading/AwsS3UrlReader.test.ts

@@ -178,7 +178,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ).rejects.toThrow(
         Error(
-          `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+          `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
         ),
       );
     });
@@ -234,7 +234,7 @@ describe('AwsS3UrlReader', () => {
         ),
       ).rejects.toThrow(
         Error(
-          `Could not retrieve file from S3; caused by Error: not a valid AWS S3 URL: https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
+          `Could not retrieve file from S3; caused by Error: invalid AWS S3 URL, cannot parse region from host in https://test-bucket.s3.us-east-2.NOTamazonaws.com/file.yaml`,
         ),
       );
     });
@@ -332,4 +332,47 @@ describe('AwsS3UrlReader', () => {
       expect(body.toString().trim()).toBe('site_name: Test');
     });
   });
+
+  describe('readNonAwsHost', () => {
+    let awsS3UrlReader: AwsS3UrlReader;
+
+    beforeAll(() => {
+      AWSMock.setSDKInstance(aws);
+      AWSMock.mock(
+        'S3',
+        'getObject',
+        Buffer.from(
+          require('fs').readFileSync(
+            path.resolve(
+              __dirname,
+              '__fixtures__/awsS3/awsS3-mock-object.yaml',
+            ),
+          ),
+        ),
+      );
+
+      const s3 = new aws.S3();
+      awsS3UrlReader = new AwsS3UrlReader(
+        new AwsS3Integration(
+          readAwsS3IntegrationConfig(
+            new ConfigReader({
+              host: 'localhost:4566',
+              accessKeyId: 'fake-access-key',
+              secretAccessKey: 'fake-secret-key',
+              endpoint: 'http://localhost:4566',
+              s3ForcePathStyle: true,
+            }),
+          ),
+        ),
+        { s3, treeResponseFactory },
+      );
+    });
+
+    it('returns contents of an object in a bucket', async () => {
+      const response = await awsS3UrlReader.read(
+        'http://localhost:4566/test-bucket/awsS3-mock-object.yaml',
+      );
+      expect(response.toString().trim()).toBe('site_name: Test');
+    });
+  });
 });

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -27,12 +27,17 @@ import {
   UrlReader,
 } from './types';
 import getRawBody from 'raw-body';
-import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
+import {
+  AwsS3Integration,
+  ScmIntegrations,
+  AwsS3IntegrationConfig,
+} from '@backstage/integration';
 import { ForwardedError, NotModifiedError } from '@backstage/errors';
 import { ListObjectsV2Output, ObjectList } from 'aws-sdk/clients/s3';
 
 const parseURL = (
   url: string,
+  config: AwsS3IntegrationConfig,
 ): { path: string; bucket: string; region: string } => {
   let { host, pathname } = new URL(url);
 
@@ -42,20 +47,45 @@ const parseURL = (
    */
   pathname = pathname.substr(1);
 
+  let bucket;
+  let region;
+
   /**
-   * Checks that the given URL is a valid S3 object url.
-   * Format of a Valid S3 URL: https://bucket-name.s3.Region.amazonaws.com/keyname
+   * Path style URLs: https://s3.Region.amazonaws.com/bucket-name/key-name
+   * Virtual hosted style URLs: https://bucket-name.s3.Region.amazonaws.com/key-name
+   * See https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access
    */
-  const validHost = new RegExp(
-    /^[a-z\d][a-z\d\.-]{1,61}[a-z\d]\.s3\.[a-z\d-]+\.amazonaws.com$/,
-  );
-  if (!validHost.test(host)) {
-    throw new Error(`not a valid AWS S3 URL: ${url}`);
+  if (config.s3ForcePathStyle) {
+    if (pathname.indexOf('/') < 0) {
+      throw new Error(
+        `invalid path-style AWS S3 URL, ${url} does not contain bucket in the path`,
+      );
+    }
+    [bucket] = pathname.split('/');
+    pathname = pathname.substr(bucket.length + 1);
+  } else {
+    if (host.indexOf('.') < 0) {
+      throw new Error(
+        `invalid virtual hosted-style AWS S3 URL, ${url} does not contain bucket prefix in the host`,
+      );
+    }
+    [bucket] = host.split('.');
+    host = host.substr(bucket.length + 1);
   }
 
-  const [bucket] = host.split(/\.s3\.[a-z\d-]+\.amazonaws.com/);
-  host = host.substring(bucket.length);
-  const [, , region, ,] = host.split('.');
+  // Only extract region from *.amazonaws.com hosts
+  if (config.host === 'amazonaws.com') {
+    // At this point bucket prefix is removed from host for virtual hosted URLs
+    const match = host.match(/^s3\.([a-z\d-]+)\.amazonaws\.com$/);
+    if (!match) {
+      throw new Error(
+        `invalid AWS S3 URL, cannot parse region from host in ${url}`,
+      );
+    }
+    region = match[1];
+  } else {
+    region = '';
+  }
 
   return {
     path: pathname,
@@ -64,15 +94,23 @@ const parseURL = (
   };
 };
 
+/**
+ * Implements a {@link UrlReader} for AWS S3 buckets.
+ *
+ * @public
+ */
 export class AwsS3UrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
 
     return integrations.awsS3.list().map(integration => {
       const creds = AwsS3UrlReader.buildCredentials(integration);
+
       const s3 = new S3({
         apiVersion: '2006-03-01',
         credentials: creds,
+        endpoint: integration.config.endpoint,
+        s3ForcePathStyle: integration.config.s3ForcePathStyle,
       });
       const reader = new AwsS3UrlReader(integration, {
         s3,
@@ -138,7 +176,7 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadUrlOptions,
   ): Promise<ReadUrlResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(url, this.integration.config);
       aws.config.update({ region: region });
 
       let params;
@@ -178,7 +216,7 @@ export class AwsS3UrlReader implements UrlReader {
     options?: ReadTreeOptions,
   ): Promise<ReadTreeResponse> {
     try {
-      const { path, bucket, region } = parseURL(url);
+      const { path, bucket, region } = parseURL(url, this.integration.config);
       const allObjects: ObjectList = [];
       const responses = [];
       let continuationToken: string | undefined;

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -38,7 +38,11 @@ import {
   ReadUrlResponse,
 } from './types';
 
-/** @public */
+/**
+ * Implements a {@link UrlReader} for Azure repos.
+ *
+ * @public
+ */
 export class AzureUrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);

packages/backend-common/src/reading/BitbucketUrlReader.ts

@@ -41,8 +41,8 @@ import {
 } from './types';
 
 /**
- * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as
- * the one exposed by Bitbucket Cloud itself.
+ * Implements a {@link UrlReader} for files from Bitbucket v1 and v2 APIs, such
+ * as the one exposed by Bitbucket Cloud itself.
  *
  * @public
  */

packages/backend-common/src/reading/FetchUrlReader.ts

@@ -27,7 +27,7 @@ import {
 import path from 'path';
 
 /**
- * A UrlReader that does a plain fetch of the URL.
+ * A {@link UrlReader} that does a plain fetch of the URL.
  *
  * @public
  */

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -50,7 +50,7 @@ export type GhBlobResponse =
   RestEndpointMethodTypes['git']['getBlob']['response']['data'];
 
 /**
- * A processor that adds the ability to read files from GitHub v3 APIs, such as
+ * Implements a {@link UrlReader} for files through the GitHub v3 APIs, such as
  * the one exposed by GitHub itself.
  *
  * @public

packages/backend-common/src/reading/GitlabUrlReader.ts

@@ -39,7 +39,11 @@ import {
 } from './types';
 import { trimEnd } from 'lodash';
 
-/** @public */
+/**
+ * Implements a {@link UrlReader} for files on GitLab.
+ *
+ * @public
+ */
 export class GitlabUrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);

packages/backend-common/src/reading/GoogleGcsUrlReader.ts

@@ -48,7 +48,11 @@ const parseURL = (
   };
 };
 
-/** @public */
+/**
+ * Implements a {@link UrlReader} for files on Google GCS.
+ *
+ * @public
+ */
 export class GoogleGcsUrlReader implements UrlReader {
   static factory: ReaderFactory = ({ config, logger }) => {
     if (!config.has('integrations.googleGcs')) {

packages/backend-common/src/reading/UrlReaders.ts

@@ -27,7 +27,11 @@ import { FetchUrlReader } from './FetchUrlReader';
 import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 import { AwsS3UrlReader } from './AwsS3UrlReader';
 
-/** @public */
+/**
+ * Creation options for {@link UrlReaders}.
+ *
+ * @public
+ */
 export type UrlReadersOptions = {
   /** Root config object */
   config: Config;
@@ -38,13 +42,13 @@ export type UrlReadersOptions = {
 };
 
 /**
- * UrlReaders provide various utilities related to the UrlReader interface.
+ * Helps construct {@link UrlReader}s.
  *
  * @public
  */
 export class UrlReaders {
   /**
-   * Creates a UrlReader without any known types.
+   * Creates a custom {@link UrlReader} wrapper for your own set of factories.
    */
   static create(options: UrlReadersOptions): UrlReader {
     const { logger, config, factories } = options;
@@ -65,7 +69,8 @@ export class UrlReaders {
   }
 
   /**
-   * Creates a UrlReader that includes all the default factories from this package.
+   * Creates a {@link UrlReader} wrapper that includes all the default factories
+   * from this package.
    *
    * Any additional factories passed will be loaded before the default ones.
    */

packages/backend-common/src/reading/types.ts

@@ -281,7 +281,12 @@ export type FromReadableArrayOptions = Array<{
   path: string;
 }>;
 
-/** @public */
+/**
+ * A factory for response factories that handle the unpacking and inspection of
+ * complex responses such as archive data.
+ *
+ * @public
+ */
 export interface ReadTreeResponseFactory {
   fromTarArchive(
     options: ReadTreeResponseFactoryOptions,

packages/backend-common/src/scm/git.test.ts

@@ -320,4 +320,21 @@ describe('Git', () => {
       });
     });
   });
+
+  describe('log', () => {
+    it('should call isomorphic-git with the correct arguments', async () => {
+      const dir = '/some/mock/dir';
+      const ref = 'as43bd7';
+
+      const git = Git.fromAuth({});
+
+      await git.log({ dir, ref });
+
+      expect(isomorphic.log).toHaveBeenCalledWith({
+        fs,
+        dir,
+        ref,
+      });
+    });
+  });
 });

packages/backend-common/src/scm/git.ts

@@ -33,7 +33,11 @@ From : https://isomorphic-git.org/docs/en/onAuth with fix for GitHub
 Azure       'notempty'      token
 */
 
-/** @public */
+/**
+ * A convenience wrapper around the `isomorphic-git` library.
+ *
+ * @public
+ */
 export class Git {
   private constructor(
     private readonly config: {
@@ -76,12 +80,15 @@ export class Git {
     return git.commit({ fs, dir, message, author, committer });
   }
 
+  /** https://isomorphic-git.org/docs/en/clone */
   async clone(options: {
     url: string;
     dir: string;
     ref?: string;
+    depth?: number;
+    noCheckout?: boolean;
   }): Promise<void> {
-    const { url, dir, ref } = options;
+    const { url, dir, ref, depth, noCheckout } = options;
     this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`);
     return git.clone({
       fs,
@@ -90,7 +97,8 @@ export class Git {
       dir,
       ref,
       singleBranch: true,
-      depth: 1,
+      depth: depth ?? 1,
+      noCheckout,
       onProgress: this.onProgressHandler(),
       headers: {
         'user-agent': 'git/@isomorphic-git',
@@ -99,7 +107,7 @@ export class Git {
     });
   }
 
-  // https://isomorphic-git.org/docs/en/currentBranch
+  /** https://isomorphic-git.org/docs/en/currentBranch */
   async currentBranch(options: {
     dir: string;
     fullName?: boolean;
@@ -110,7 +118,7 @@ export class Git {
     >;
   }
 
-  // https://isomorphic-git.org/docs/en/fetch
+  /** https://isomorphic-git.org/docs/en/fetch */
   async fetch(options: { dir: string; remote?: string }): Promise<void> {
     const { dir, remote = 'origin' } = options;
     this.config.logger?.info(
@@ -138,7 +146,7 @@ export class Git {
     });
   }
 
-  // https://isomorphic-git.org/docs/en/merge
+  /** https://isomorphic-git.org/docs/en/merge */
   async merge(options: {
     dir: string;
     theirs: string;
@@ -180,7 +188,7 @@ export class Git {
     });
   }
 
-  // https://isomorphic-git.org/docs/en/readCommit
+  /** https://isomorphic-git.org/docs/en/readCommit */
   async readCommit(options: {
     dir: string;
     sha: string;
@@ -189,12 +197,25 @@ export class Git {
     return git.readCommit({ fs, dir, oid: sha });
   }
 
-  // https://isomorphic-git.org/docs/en/resolveRef
+  /** https://isomorphic-git.org/docs/en/resolveRef */
   async resolveRef(options: { dir: string; ref: string }): Promise<string> {
     const { dir, ref } = options;
     return git.resolveRef({ fs, dir, ref });
   }
 
+  /** https://isomorphic-git.org/docs/en/log */
+  async log(options: {
+    dir: string;
+    ref?: string;
+  }): Promise<ReadCommitResult[]> {
+    const { dir, ref } = options;
+    return git.log({
+      fs,
+      dir,
+      ref: ref ?? 'HEAD',
+    });
+  }
+
   private onAuth = () => ({
     username: this.config.username,
     password: this.config.password,

packages/backend-common/src/service/createStatusCheckRouter.ts

@@ -19,9 +19,26 @@ import Router from 'express-promise-router';
 import express from 'express';
 import { errorHandler, statusCheckHandler, StatusCheck } from '../middleware';
 
-/** @public */
+/**
+ * Creates a default status checking router, that you can add to your express
+ * app.
+ *
+ * @remarks
+ *
+ * This adds a `/healthcheck` route (or any other path, if given as an
+ * argument), which your infra can call to see if the service is ready to serve
+ * requests.
+ *
+ * @public
+ */
 export async function createStatusCheckRouter(options: {
   logger: Logger;
+  /**
+   * The path (including a leading slash) that the health check should be
+   * mounted on.
+   *
+   * @defaultValue '/healthcheck'
+   */
   path?: string;
   /**
    * If not implemented, the default express middleware always returns 200.

packages/backend-common/src/service/lib/ServiceBuilderImpl.ts

@@ -19,6 +19,7 @@ import compression from 'compression';
 import cors from 'cors';
 import express, { Router, ErrorRequestHandler } from 'express';
 import helmet from 'helmet';
+import { ContentSecurityPolicyOptions } from 'helmet/dist/middlewares/content-security-policy';
 import * as http from 'http';
 import stoppable from 'stoppable';
 import { Logger } from 'winston';
@@ -43,19 +44,6 @@ import { createHttpServer, createHttpsServer } from './hostFactory';
 export const DEFAULT_PORT = 7007;
 // '' is express default, which listens to all interfaces
 const DEFAULT_HOST = '';
-// taken from the helmet source code - don't seem to be exported
-const DEFAULT_CSP = {
-  'default-src': ["'self'"],
-  'base-uri': ["'self'"],
-  'block-all-mixed-content': [],
-  'font-src': ["'self'", 'https:', 'data:'],
-  'frame-ancestors': ["'self'"],
-  'img-src': ["'self'", 'data:'],
-  'object-src': ["'none'"],
-  'script-src': ["'self'", "'unsafe-eval'"],
-  'script-src-attr': ["'none'"],
-  'style-src': ["'self'", 'https:', "'unsafe-inline'"],
-};
 
 export class ServiceBuilderImpl implements ServiceBuilder {
   private port: number | undefined;
@@ -236,8 +224,13 @@ export class ServiceBuilderImpl implements ServiceBuilder {
 
 export function applyCspDirectives(
   directives: Record<string, string[] | false> | undefined,
-): CspOptions | undefined {
-  const result: CspOptions = { ...DEFAULT_CSP };
+): ContentSecurityPolicyOptions['directives'] {
+  const result: ContentSecurityPolicyOptions['directives'] =
+    helmet.contentSecurityPolicy.getDefaultDirectives();
+
+  // TODO(Rugvip): We currently use non-precompiled AJV for validation in the frontend, which uses eval.
+  //               It should be replaced by any other solution that doesn't require unsafe-eval.
+  result['script-src'] = ["'self'", "'unsafe-eval'"];
 
   if (directives) {
     for (const [key, value] of Object.entries(directives)) {

packages/backend-common/src/service/lib/config.ts

@@ -42,8 +42,6 @@ export type CertificateAttributes = {
 
 /**
  * A map from CSP directive names to their values.
- *
- * Added here since helmet doesn't export this type publicly.
  */
 export type CspOptions = Record<string, string[]>;
 
@@ -57,7 +55,7 @@ type CustomOrigin = (
 /**
  * Reads some base options out of a config object.
  *
- * @param config The root of a backend config object
+ * @param config - The root of a backend config object
  * @returns A base options object
  *
  * @example
@@ -100,7 +98,7 @@ export function readBaseOptions(config: Config): BaseOptions {
 /**
  * Attempts to read a CORS options object from the root of a config object.
  *
- * @param config The root of a backend config object
+ * @param config - The root of a backend config object
  * @returns A CORS options object, or undefined if not specified
  *
  * @example
@@ -134,7 +132,7 @@ export function readCorsOptions(config: Config): CorsOptions | undefined {
 /**
  * Attempts to read a CSP options object from the root of a config object.
  *
- * @param config The root of a backend config object
+ * @param config - The root of a backend config object
  * @returns A CSP options object, or undefined if not specified. Values can be
  *          false as well, which means to remove the default behavior for that
  *          key.
@@ -170,7 +168,7 @@ export function readCspOptions(
 /**
  * Attempts to read a https settings object from the root of a config object.
  *
- * @param config The root of a backend config object
+ * @param config - The root of a backend config object
  * @returns A https settings object, or undefined if not specified
  *
  * @example

packages/backend-common/src/service/lib/hostFactory.ts

@@ -29,8 +29,8 @@ const IP_HOSTNAME_REGEX = /:|^\d+\.\d+\.\d+\.\d+$/;
 /**
  * Creates a Http server instance based on an Express application.
  *
- * @param app The Express application object
- * @param logger Optional Winston logger object
+ * @param app - The Express application object
+ * @param logger - Optional Winston logger object
  * @returns A Http server instance
  *
  */
@@ -46,9 +46,9 @@ export function createHttpServer(
 /**
  * Creates a Https server instance based on an Express application.
  *
- * @param app The Express application object
- * @param httpsSettings HttpsSettings for self-signed certificate generation
- * @param logger Optional Winston logger object
+ * @param app - The Express application object
+ * @param httpsSettings - HttpsSettings for self-signed certificate generation
+ * @param logger - Optional Winston logger object
  * @returns A Https server instance
  *
  */

packages/backend-common/src/service/types.ts

@@ -20,7 +20,11 @@ import { Router, RequestHandler, ErrorRequestHandler } from 'express';
 import { Server } from 'http';
 import { Logger } from 'winston';
 
-/** @public */
+/**
+ * A helper for building backend service instances.
+ *
+ * @public
+ */
 export type ServiceBuilder = {
   /**
    * Sets the service parameters based on configuration.
@@ -119,5 +123,9 @@ export type ServiceBuilder = {
   start(): Promise<Server>;
 };
 
-/** @public */
+/**
+ * A factory for request loggers.
+ *
+ * @public
+ */
 export type RequestLoggingHandlerFactory = (logger?: Logger) => RequestHandler;

packages/backend-common/src/util/ContainerRunner.ts

@@ -16,7 +16,11 @@
 
 import { Writable } from 'stream';
 
-/** @public */
+/**
+ * Options passed to the {@link ContainerRunner.runContainer} method.
+ *
+ * @public
+ */
 export type RunContainerOptions = {
   imageName: string;
   command?: string | string[];
@@ -28,7 +32,14 @@ export type RunContainerOptions = {
   pullImage?: boolean;
 };
 
-/** @public */
+/**
+ * Handles the running of containers, on behalf of others.
+ *
+ * @public
+ */
 export interface ContainerRunner {
+  /**
+   * Runs a container image to completion.
+   */
   runContainer(opts: RunContainerOptions): Promise<void>;
 }

packages/backend-common/src/util/DockerContainerRunner.ts

@@ -24,7 +24,11 @@ export type UserOptions = {
   User?: string;
 };
 
-/** @public */
+/**
+ * A {@link ContainerRunner} for Docker containers.
+ *
+ * @public
+ */
 export class DockerContainerRunner implements ContainerRunner {
   private readonly dockerClient: Docker;