backend-common: refactor UrlReaders and factories to be centered around predicates

app-config.yaml

@@ -67,53 +67,42 @@ kubernetes:
   clusterLocatorMethod: 'configMultiTenant'
   clusters: []
 
-# This sections configures the URL readers used by various parts of
-# Backstage to read data from remote URLs
-readers:
-  byHost:
-    - type: github
-      host: github.com
-      config:
-        token:
-          $secret:
-            env: GITHUB_PRIVATE_TOKEN
+integrations:
+  github:
+    - host: github.com
+      token:
+        $secret:
+          env: GITHUB_PRIVATE_TOKEN
     ### Example for how to add your GitHub Enterprise instance using the API:
-    # - type: github
-    #   host: ghe.example.net
-    #   config:
-    #     apiBaseUrl: https://ghe.example.net/api/v3
-    #     token:
-    #       $secret:
-    #         env: GHE_PRIVATE_TOKEN
+    # - host: ghe.example.net
+    #   apiBaseUrl: https://ghe.example.net/api/v3
+    #   token:
+    #     $secret:
+    #       env: GHE_PRIVATE_TOKEN
     ### Example for how to add your GitHub Enterprise instance using raw HTTP fetches (token is optional):
-    # - type: github
-    #   host: ghe.example.net
-    #   config:
-    #     rawBaseUrl: https://ghe.example.net/raw
-    #     token:
-    #       $secret:
-    #         env: GHE_PRIVATE_TOKEN
-    - type: gitlab
-      host: gitlab.com
-      config:
-        token:
-          $secret:
-            env: GITLAB_PRIVATE_TOKEN
-    - type: bitbucket
-      host: bitbucket.org
-      config:
-        username:
-          $secret:
-            env: BITBUCKET_USERNAME
-        appPassword:
-          $secret:
-            env: BITBUCKET_APP_PASSWORD
-    - type: azure
-      host: dev.azure.com
-      config:
-        privateToken:
-          $secret:
-            env: AZURE_PRIVATE_TOKEN
+    # - host: ghe.example.net
+    #   rawBaseUrl: https://ghe.example.net/raw
+    #   token:
+    #     $secret:
+    #       env: GHE_PRIVATE_TOKEN
+  gitlab:
+    - host: gitlab.com
+      token:
+        $secret:
+          env: GITLAB_PRIVATE_TOKEN
+  bitbucket:
+    - host: bitbucket.org
+      username:
+        $secret:
+          env: BITBUCKET_USERNAME
+      appPassword:
+        $secret:
+          env: BITBUCKET_APP_PASSWORD
+  azure:
+    - host: dev.azure.com
+      token:
+        $secret:
+          env: AZURE_PRIVATE_TOKEN
 
 catalog:
   rules:

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -17,19 +17,18 @@
 import { Config } from '@backstage/config';
 import parseGitUri from 'git-url-parse';
 import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch';
-import { Logger } from 'winston';
 import { NotFoundError } from '../errors';
 import { UrlReader } from './types';
+import { ReaderFactory } from './UrlReaders';
 
 /**
  * The configuration parameters for a single GitHub API provider.
  */
 export type ProviderConfig = {
   /**
-   * The prefix of the target that this matches on, e.g. "https://github.com",
-   * with no trailing slash.
+   * The host of the target that this matches on, e.g. "github.com"
    */
-  target: string;
+  host: string;
 
   /**
    * The base URL of the API of this provider, e.g. "https://api.github.com",
@@ -137,63 +136,47 @@ export function getRawUrl(target: string, provider: ProviderConfig): URL {
   }
 }
 
-export function readConfig(config: Config, logger: Logger): ProviderConfig[] {
+export function readConfig(config: Config): ProviderConfig[] {
   const providers: ProviderConfig[] = [];
 
-  // TODO(freben): Deprecate the old config root entirely in a later release
-  if (config.has('catalog.processors.githubApi')) {
-    logger.warn(
-      'The catalog.processors.githubApi configuration key has been deprecated, please use catalog.processors.github instead',
-    );
-  }
-
-  // In a previous version of the configuration, we only supported github,
-  // and the "privateToken" key held the token to use for it. The new
-  // configuration method is to use the "providers" key instead.
   const providerConfigs =
-    config.getOptionalConfigArray('catalog.processors.github.providers') ??
-    config.getOptionalConfigArray('catalog.processors.githubApi.providers') ??
-    [];
-  const legacyToken =
-    config.getOptionalString('catalog.processors.github.privateToken') ??
-    config.getOptionalString('catalog.processors.githubApi.privateToken');
+    config.getOptionalConfigArray('integrations.github') ?? [];
 
   // First read all the explicit providers
   for (const providerConfig of providerConfigs) {
-    const target = providerConfig.getString('target').replace(/\/+$/, '');
+    const host = providerConfig.getOptionalString('host') ?? 'github.com';
     let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
     let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl');
     const token = providerConfig.getOptionalString('token');
 
     if (apiBaseUrl) {
       apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
-    } else if (target === 'https://github.com') {
+    } else if (host === 'github.com') {
       apiBaseUrl = 'https://api.github.com';
     }
 
     if (rawBaseUrl) {
       rawBaseUrl = rawBaseUrl.replace(/\/+$/, '');
-    } else if (target === 'https://github.com') {
+    } else if (host === 'github.com') {
       rawBaseUrl = 'https://raw.githubusercontent.com';
     }
 
     if (!apiBaseUrl && !rawBaseUrl) {
       throw new Error(
-        `Provider at ${target} must configure an explicit apiBaseUrl or rawBaseUrl`,
+        `GitHub integration for ${host} must configure an explicit apiBaseUrl and rawBaseUrl`,
       );
     }
 
-    providers.push({ target, apiBaseUrl, rawBaseUrl, token });
+    providers.push({ host, apiBaseUrl, rawBaseUrl, token });
   }
 
   // If no explicit github.com provider was added, put one in the list as
   // a convenience
-  if (!providers.some(p => p.target === 'https://github.com')) {
+  if (!providers.some(p => p.host === 'github.com')) {
     providers.push({
-      target: 'https://github.com',
+      host: 'github.com',
       apiBaseUrl: 'https://api.github.com',
       rawBaseUrl: 'https://raw.githubusercontent.com',
-      token: legacyToken,
     });
   }
 
@@ -207,6 +190,14 @@ export function readConfig(config: Config, logger: Logger): ProviderConfig[] {
 export class GithubUrlReader implements UrlReader {
   private config: ProviderConfig;
 
+  static factory: ReaderFactory = ({ config }) => {
+    return readConfig(config).map(provider => {
+      const reader = new GithubUrlReader(provider);
+      const predicate = (url: URL) => url.host === provider.host;
+      return { reader, predicate };
+    });
+  };
+
   constructor(config: ProviderConfig) {
     this.config = config;
   }

packages/backend-common/src/reading/UrlReaders.ts

@@ -14,16 +14,27 @@
  * limitations under the License.
  */
 
+import { Logger } from 'winston';
 import { Config } from '@backstage/config';
 import { GithubUrlReader } from './GithubUrlReader';
 import { UrlReader } from './types';
-import { UrlReaderHostMux } from './UrlReaderHostMux';
+import {
+  UrlReaderPredicateMux,
+  UrlReaderPredicateTuple,
+} from './UrlReaderPredicateMux';
 
 export type ReaderFactoryOptions = {
   config: Config;
   logger: Logger;
 };
-export type ReaderFactory = (options: ReaderFactoryOptions) => UrlReader;
+
+/**
+ * A factory function that can read config to construct zero or more
+ * UrlReaders along with a predicate for when it should be used.
+ */
+export type ReaderFactory = (
+  options: ReaderFactoryOptions,
+) => UrlReaderPredicateTuple[];
 
 /**
  * UrlReaders provide various utilities related to the UrlReader interface.
@@ -33,22 +44,18 @@ export class UrlReaders {
    * Creates a new UrlReaders instance without any known types.
    */
   static empty({ logger }: { logger: Logger }) {
-    return new UrlReaders(new Map(), logger);
+    return new UrlReaders([], logger);
   }
 
   /**
-   * Creates a new UrlReaders instance that includes factories for all types
-   * implemented in this package.
+   * Creates a new UrlReaders instance that includes all the default factories from this package
    */
   static default({ logger }: { logger: Logger }) {
-    return new UrlReaders(
-      new Map([['github', GithubUrlReader.factory]]),
-      logger,
-    );
+    return new UrlReaders([GithubUrlReader.factory], logger);
   }
 
   private constructor(
-    private readonly factories: Map<string, ReaderFactory>,
+    private readonly factories: ReaderFactory[],
     private readonly logger: Logger,
   ) {}
 
@@ -57,33 +64,29 @@ export class UrlReaders {
    * reader type needs to have a registered factory, or an error will be thrown.
    */
   createWithConfig(config: Config): UrlReader {
-    const readerItems = config.getOptionalConfigArray('readers.byHost') ?? [];
-    const mux = new UrlReaderHostMux();
+    const mux = new UrlReaderPredicateMux();
+    const readers = [];
 
-    for (const readerItem of readerItems) {
-      const type = readerItem.getString('type');
-      const host = readerItem.getString('host');
-      const readerConfig = readerItem.getConfig('config');
+    for (const factory of this.factories) {
+      const tuples = factory({ config, logger: this.logger });
 
-      const factory = this.factories.get(type);
-      if (!factory) {
-        throw new Error(`Failed to create reader for unknown type '${type}'`);
+      for (const tuple of tuples) {
+        mux.register(tuple);
+        readers.push(tuple.reader);
       }
-      const reader = factory({
-        config: readerConfig,
-        logger: this.logger,
-      });
-
-      mux.register(host, reader);
     }
 
+    this.logger.info(
+      `Registered the following UrlReaders: ${readers.join(', ')}`,
+    );
+
     return mux;
   }
 
   /**
-   * Register a UrlReader factory for a given type
+   * Register a UrlReader factory
    */
-  register(type: string, factory: ReaderFactory) {
-    this.factories.set(type, factory);
+  addFactory(factory: ReaderFactory) {
+    this.factories.push(factory);
   }
 }