Merge pull request #2665 from spotify/rugvip/reading

backend-common: add new common UrlReader to use for reading remote data

packages/backend-common/package.json

@@ -39,11 +39,13 @@
     "express": "^4.17.1",
     "express-prom-bundle": "^6.1.0",
     "express-promise-router": "^3.0.3",
+    "git-url-parse": "^11.2.0",
     "helmet": "^4.0.0",
     "knex": "^0.21.1",
     "lodash": "^4.17.15",
     "logform": "^2.1.1",
     "morgan": "^1.10.0",
+    "node-fetch": "^2.6.0",
     "prom-client": "^12.0.0",
     "selfsigned": "^1.10.7",
     "stoppable": "^1.1.0",
@@ -62,6 +64,7 @@
     "@types/compression": "^1.7.0",
     "@types/http-errors": "^1.6.3",
     "@types/morgan": "^1.9.0",
+    "@types/node-fetch": "^2.5.7",
     "@types/stoppable": "^1.1.0",
     "@types/supertest": "^2.0.8",
     "@types/webpack-env": "^1.15.2",
@@ -70,6 +73,7 @@
     "http-errors": "^1.7.3",
     "jest": "^26.0.1",
     "jest-fetch-mock": "^3.0.3",
+    "msw": "^0.20.5",
     "supertest": "^4.0.2"
   },
   "files": [

packages/backend-common/src/index.ts

@@ -20,6 +20,7 @@ export * from './discovery';
 export * from './errors';
 export * from './logging';
 export * from './middleware';
+export * from './reading';
 export * from './service';
 export * from './paths';
 export * from './hot';

packages/backend-common/src/reading/AzureUrlReader.test.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { AzureUrlReader } from './AzureUrlReader';
+
+const logger = getVoidLogger();
+
+describe('AzureUrlReader', () => {
+  const worker = setupServer();
+
+  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
+  afterAll(() => worker.close());
+
+  beforeEach(() => {
+    worker.use(
+      rest.get('*', (req, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            url: req.url.toString(),
+            headers: req.headers.getAllHeaders(),
+          }),
+        ),
+      ),
+    );
+  });
+  afterEach(() => worker.resetHandlers());
+
+  const createConfig = (token?: string) =>
+    new ConfigReader(
+      {
+        integrations: { azure: [{ host: 'dev.azure.com', token }] },
+      },
+      'test-config',
+    );
+
+  it.each([
+    {
+      url:
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+      config: createConfig(),
+      response: expect.objectContaining({
+        url:
+          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master',
+      }),
+    },
+    {
+      url:
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml',
+      config: createConfig(),
+      response: expect.objectContaining({
+        url:
+          'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml',
+      }),
+    },
+    {
+      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+      config: createConfig('0123456789'),
+      response: expect.objectContaining({
+        headers: expect.objectContaining({
+          authorization: 'Basic OjAxMjM0NTY3ODk=',
+        }),
+      }),
+    },
+    {
+      url: 'https://dev.azure.com/a/b/_git/repo-name?path=my-template.yaml',
+      config: createConfig(undefined),
+      response: expect.objectContaining({
+        headers: expect.not.objectContaining({
+          authorization: expect.anything(),
+        }),
+      }),
+    },
+  ])('should handle happy path %#', async ({ url, config, response }) => {
+    const [{ reader }] = AzureUrlReader.factory({ config, logger });
+
+    const data = await reader.read(url);
+    const res = await JSON.parse(data.toString('utf-8'));
+    expect(res).toEqual(response);
+  });
+
+  it.each([
+    {
+      url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+      config: createConfig(),
+      error:
+        'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path',
+    },
+    {
+      url: 'com/a/b/blob/master/path/to/c.yaml',
+      config: createConfig(),
+      error:
+        'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+    },
+    {
+      url: '',
+      config: createConfig(''),
+      error:
+        "Invalid type in config for key 'integrations.azure[0].token' in 'test-config', got empty-string, wanted string",
+    },
+  ])('should handle error path %#', async ({ url, config, error }) => {
+    await expect(async () => {
+      const [{ reader }] = AzureUrlReader.factory({ config, logger });
+      await reader.read(url);
+    }).rejects.toThrow(error);
+  });
+});

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -0,0 +1,165 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch';
+import { Config } from '@backstage/config';
+import { NotFoundError } from '../errors';
+import { ReaderFactory, UrlReader } from './types';
+
+type Options = {
+  // TODO: added here for future support, but we only allow dev.azure.com for now
+  host: string;
+  token?: string;
+};
+
+function readConfig(config: Config): Options[] {
+  const optionsArr = Array<Options>();
+
+  const providerConfigs =
+    config.getOptionalConfigArray('integrations.azure') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const host = providerConfig.getOptionalString('host') ?? 'dev.azure.com';
+    const token = providerConfig.getOptionalString('token');
+
+    optionsArr.push({ host, token });
+  }
+
+  // As a convenience we always make sure there's at least an unauthenticated
+  // reader for public azure repos.
+  if (!optionsArr.some(p => p.host === 'dev.azure.com')) {
+    optionsArr.push({ host: 'dev.azure.com' });
+  }
+
+  return optionsArr;
+}
+
+export class AzureUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config }) => {
+    return readConfig(config).map(options => {
+      const reader = new AzureUrlReader(options);
+      const predicate = (url: URL) => url.host === options.host;
+      return { reader, predicate };
+    });
+  };
+
+  constructor(private readonly options: Options) {
+    if (options.host !== 'dev.azure.com') {
+      throw Error(
+        `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`,
+      );
+    }
+  }
+
+  async read(url: string): Promise<Buffer> {
+    const builtUrl = this.buildRawUrl(url);
+
+    let response: Response;
+    try {
+      response = await fetch(builtUrl.toString(), this.getRequestOptions());
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
+    if (response.ok && response.status !== 203) {
+      return response.buffer();
+    }
+
+    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+
+  // Converts
+  // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
+  // to:   https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch}
+  private buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        project,
+        srcKeyword,
+        repoName,
+      ] = url.pathname.split('/');
+
+      const path = url.searchParams.get('path') || '';
+      const ref = url.searchParams.get('version')?.substr(2);
+
+      if (
+        url.hostname !== 'dev.azure.com' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        project === '' ||
+        srcKeyword !== '_git' ||
+        repoName === '' ||
+        path === '' ||
+        ref === ''
+      ) {
+        throw new Error('Wrong Azure Devops URL or Invalid file path');
+      }
+
+      // transform to api
+      url.pathname = [
+        empty,
+        userOrOrg,
+        project,
+        '_apis',
+        'git',
+        'repositories',
+        repoName,
+        'items',
+      ].join('/');
+
+      const queryParams = [`path=${path}`];
+
+      if (ref) {
+        queryParams.push(`version=${ref}`);
+      }
+
+      url.search = queryParams.join('&');
+
+      url.protocol = 'https';
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+
+  private getRequestOptions(): RequestInit {
+    const headers: HeadersInit = {};
+
+    if (this.options.token) {
+      headers.Authorization = `Basic ${Buffer.from(
+        `:${this.options.token}`,
+        'utf8',
+      ).toString('base64')}`;
+    }
+
+    return { headers };
+  }
+
+  toString() {
+    const { host, token } = this.options;
+    return `azure{host=${host},authed=${Boolean(token)}}`;
+  }
+}

packages/backend-common/src/reading/BitbucketUrlReader.test.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { BitbucketUrlReader } from './BitbucketUrlReader';
+
+const logger = getVoidLogger();
+
+describe('BitbucketUrlReader', () => {
+  const worker = setupServer();
+
+  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
+  afterAll(() => worker.close());
+
+  beforeEach(() => {
+    worker.use(
+      rest.get('*', (req, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            url: req.url.toString(),
+            headers: req.headers.getAllHeaders(),
+          }),
+        ),
+      ),
+    );
+  });
+  afterEach(() => worker.resetHandlers());
+
+  const createConfig = (username?: string, appPassword?: string) =>
+    new ConfigReader(
+      {
+        integrations: {
+          bitbucket: [
+            {
+              host: 'bitbucket.org',
+              username: username,
+              appPassword: appPassword,
+            },
+          ],
+        },
+      },
+      'test-config',
+    );
+
+  it.each([
+    {
+      url:
+        'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
+      config: createConfig(),
+      response: expect.objectContaining({
+        url:
+          'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml',
+      }),
+    },
+    {
+      url:
+        'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
+      config: createConfig('some-user', 'my-secret'),
+      response: expect.objectContaining({
+        headers: expect.objectContaining({
+          authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==',
+        }),
+      }),
+    },
+    {
+      url:
+        'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
+      config: createConfig(),
+      response: expect.objectContaining({
+        headers: expect.not.objectContaining({
+          authorization: expect.anything(),
+        }),
+      }),
+    },
+    {
+      url:
+        'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml',
+      config: createConfig(undefined, 'only-password-provided'),
+      response: expect.objectContaining({
+        headers: expect.not.objectContaining({
+          authorization: expect.anything(),
+        }),
+      }),
+    },
+  ])('should handle happy path %#', async ({ url, config, response }) => {
+    const [{ reader }] = BitbucketUrlReader.factory({ config, logger });
+
+    const data = await reader.read(url);
+    const res = await JSON.parse(data.toString('utf-8'));
+    expect(res).toEqual(response);
+  });
+
+  it.each([
+    {
+      url: 'https://api.com/a/b/blob/master/path/to/c.yaml',
+      config: createConfig(),
+      error:
+        'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path',
+    },
+    {
+      url: 'com/a/b/blob/master/path/to/c.yaml',
+      config: createConfig(),
+      error:
+        'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml',
+    },
+    {
+      url: '',
+      config: createConfig('', ''),
+      error:
+        "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string",
+    },
+    {
+      url: '',
+      config: createConfig('only-user-provided', ''),
+      error:
+        "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in 'test-config', got empty-string, wanted string",
+    },
+    {
+      url: '',
+      config: createConfig('', 'only-password-provided'),
+      error:
+        "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string",
+    },
+    {
+      url: '',
+      config: createConfig('only-user-provided', undefined),
+      error:
+        "Missing required config value at 'integrations.bitbucket[0].appPassword'",
+    },
+  ])('should handle error path %#', async ({ url, config, error }) => {
+    await expect(async () => {
+      const [{ reader }] = BitbucketUrlReader.factory({ config, logger });
+      await reader.read(url);
+    }).rejects.toThrow(error);
+  });
+});

packages/backend-common/src/reading/BitbucketUrlReader.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch, { RequestInit, HeadersInit, Response } from 'node-fetch';
+import { Config } from '@backstage/config';
+import { ReaderFactory, UrlReader } from './types';
+import { NotFoundError } from '../errors';
+
+type Options = {
+  // TODO: added here for future support, but we only allow bitbucket.org for now
+  host: string;
+  auth?: {
+    username: string;
+    appPassword: string;
+  };
+};
+
+function readConfig(config: Config): Options[] {
+  const optionsArr = Array<Options>();
+
+  const providerConfigs =
+    config.getOptionalConfigArray('integrations.bitbucket') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org';
+
+    let auth;
+    if (providerConfig.has('username')) {
+      const username = providerConfig.getString('username');
+      const appPassword = providerConfig.getString('appPassword');
+      auth = { username, appPassword };
+    }
+
+    optionsArr.push({ host, auth });
+  }
+
+  // As a convenience we always make sure there's at least an unauthenticated
+  // reader for public bitbucket repos.
+  if (!optionsArr.some(p => p.host === 'bitbucket.org')) {
+    optionsArr.push({ host: 'bitbucket.org' });
+  }
+
+  return optionsArr;
+}
+
+export class BitbucketUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config }) => {
+    return readConfig(config).map(options => {
+      const reader = new BitbucketUrlReader(options);
+      const predicate = (url: URL) => url.host === options.host;
+      return { reader, predicate };
+    });
+  };
+
+  constructor(private readonly options: Options) {
+    if (options.host !== 'bitbucket.org') {
+      throw Error(
+        `Bitbucket integration currently only supports 'bitbucket.org', tried to use host '${options.host}'`,
+      );
+    }
+  }
+
+  async read(url: string): Promise<Buffer> {
+    const builtUrl = this.buildRawUrl(url);
+
+    let response: Response;
+    try {
+      response = await fetch(builtUrl.toString(), this.getRequestOptions());
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    if (response.ok) {
+      return response.buffer();
+    }
+
+    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+
+  // Converts
+  // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml
+  // to:   https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml
+  private buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        srcKeyword,
+        ref,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        url.hostname !== 'bitbucket.org' ||
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        srcKeyword !== 'src'
+      ) {
+        throw new Error('Wrong Bitbucket URL or Invalid file path');
+      }
+
+      // transform to api
+      url.pathname = [
+        empty,
+        '2.0',
+        'repositories',
+        userOrOrg,
+        repoName,
+        'src',
+        ref,
+        ...restOfPath,
+      ].join('/');
+      url.hostname = 'api.bitbucket.org';
+      url.protocol = 'https';
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+
+  private getRequestOptions(): RequestInit {
+    const headers: HeadersInit = {};
+
+    if (this.options.auth) {
+      headers.Authorization = `Basic ${Buffer.from(
+        `${this.options.auth.username}:${this.options.auth.appPassword}`,
+        'utf8',
+      ).toString('base64')}`;
+    }
+
+    return {
+      headers,
+    };
+  }
+
+  toString() {
+    const { host, auth } = this.options;
+    return `bitbucket{host=${host},authed=${Boolean(auth)}}`;
+  }
+}

packages/backend-common/src/reading/FetchUrlReader.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch, { Response } from 'node-fetch';
+import { NotFoundError } from '../errors';
+import { UrlReader } from './types';
+
+/**
+ * A UrlReader that does a plain fetch of the URL.
+ */
+export class FetchUrlReader implements UrlReader {
+  async read(url: string): Promise<Buffer> {
+    let response: Response;
+    try {
+      response = await fetch(url);
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    if (response.ok) {
+      return response.buffer();
+    }
+
+    const message = `could not read ${url}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+
+  toString() {
+    return 'fetch{}';
+  }
+}

packages/backend-common/src/reading/GithubUrlReader.test.ts

@@ -0,0 +1,233 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import {
+  getApiRequestOptions,
+  getApiUrl,
+  getRawRequestOptions,
+  getRawUrl,
+  GithubUrlReader,
+  ProviderConfig,
+  readConfig,
+} from './GithubUrlReader';
+
+describe('GithubUrlReader', () => {
+  describe('getApiRequestOptions', () => {
+    it('sets the correct API version', () => {
+      const config: ProviderConfig = { host: '', apiBaseUrl: '' };
+      expect((getApiRequestOptions(config).headers as any).Accept).toEqual(
+        'application/vnd.github.v3.raw',
+      );
+    });
+
+    it('inserts a token when needed', () => {
+      const withToken: ProviderConfig = {
+        host: '',
+        apiBaseUrl: '',
+        token: 'A',
+      };
+      const withoutToken: ProviderConfig = {
+        host: '',
+        apiBaseUrl: '',
+      };
+      expect(
+        (getApiRequestOptions(withToken).headers as any).Authorization,
+      ).toEqual('token A');
+      expect(
+        (getApiRequestOptions(withoutToken).headers as any).Authorization,
+      ).toBeUndefined();
+    });
+  });
+
+  describe('getRawRequestOptions', () => {
+    it('inserts a token when needed', () => {
+      const withToken: ProviderConfig = {
+        host: '',
+        rawBaseUrl: '',
+        token: 'A',
+      };
+      const withoutToken: ProviderConfig = {
+        host: '',
+        rawBaseUrl: '',
+      };
+      expect(
+        (getRawRequestOptions(withToken).headers as any).Authorization,
+      ).toEqual('token A');
+      expect(
+        (getRawRequestOptions(withoutToken).headers as any).Authorization,
+      ).toBeUndefined();
+    });
+  });
+
+  describe('getApiUrl', () => {
+    it('rejects targets that do not look like URLs', () => {
+      const config: ProviderConfig = { host: '', apiBaseUrl: '' };
+      expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
+    });
+
+    it('happy path for github', () => {
+      const config: ProviderConfig = {
+        host: 'github.com',
+        apiBaseUrl: 'https://api.github.com',
+      };
+      expect(
+        getApiUrl(
+          'https://github.com/a/b/blob/branchname/path/to/c.yaml',
+          config,
+        ),
+      ).toEqual(
+        new URL(
+          'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
+        ),
+      );
+      expect(
+        getApiUrl(
+          'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
+          config,
+        ),
+      ).toEqual(
+        new URL(
+          'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname',
+        ),
+      );
+    });
+
+    it('happy path for ghe', () => {
+      const config: ProviderConfig = {
+        host: 'ghe.mycompany.net',
+        apiBaseUrl: 'https://ghe.mycompany.net/api/v3',
+      };
+      expect(
+        getApiUrl(
+          'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
+          config,
+        ),
+      ).toEqual(
+        new URL(
+          'https://ghe.mycompany.net/api/v3/repos/a/b/contents/path/to/c.yaml?ref=branchname',
+        ),
+      );
+    });
+  });
+
+  describe('getRawUrl', () => {
+    it('rejects targets that do not look like URLs', () => {
+      const config: ProviderConfig = { host: '', apiBaseUrl: '' };
+      expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/);
+    });
+
+    it('happy path for github', () => {
+      const config: ProviderConfig = {
+        host: 'github.com',
+        rawBaseUrl: 'https://raw.githubusercontent.com',
+      };
+      expect(
+        getRawUrl(
+          'https://github.com/a/b/blob/branchname/path/to/c.yaml',
+          config,
+        ),
+      ).toEqual(
+        new URL(
+          'https://raw.githubusercontent.com/a/b/branchname/path/to/c.yaml',
+        ),
+      );
+    });
+
+    it('happy path for ghe', () => {
+      const config: ProviderConfig = {
+        host: 'ghe.mycompany.net',
+        rawBaseUrl: 'https://ghe.mycompany.net/raw',
+      };
+      expect(
+        getRawUrl(
+          'https://ghe.mycompany.net/a/b/blob/branchname/path/to/c.yaml',
+          config,
+        ),
+      ).toEqual(
+        new URL('https://ghe.mycompany.net/raw/a/b/branchname/path/to/c.yaml'),
+      );
+    });
+  });
+
+  describe('readConfig', () => {
+    function config(
+      providers: { host: string; apiBaseUrl?: string; token?: string }[],
+    ) {
+      return ConfigReader.fromConfigs([
+        {
+          context: '',
+          data: {
+            integrations: { github: providers },
+          },
+        },
+      ]);
+    }
+
+    it('adds a default GitHub entry when missing', () => {
+      const output = readConfig(config([]));
+      expect(output).toEqual([
+        {
+          host: 'github.com',
+          apiBaseUrl: 'https://api.github.com',
+          rawBaseUrl: 'https://raw.githubusercontent.com',
+        },
+      ]);
+    });
+
+    it('injects the correct GitHub API base URL when missing', () => {
+      const output = readConfig(config([{ host: 'github.com' }]));
+      expect(output).toEqual([
+        {
+          host: 'github.com',
+          apiBaseUrl: 'https://api.github.com',
+          rawBaseUrl: 'https://raw.githubusercontent.com',
+        },
+      ]);
+    });
+
+    it('rejects custom targets with no base URLs', () => {
+      expect(() => readConfig(config([{ host: 'ghe.company.com' }]))).toThrow(
+        "GitHub integration for 'ghe.company.com' must configure an explicit apiBaseUrl and rawBaseUrl",
+      );
+    });
+
+    it('rejects funky configs', () => {
+      expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/);
+      expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/);
+      expect(() =>
+        readConfig(config([{ host: 'github.com', apiBaseUrl: 7 } as any])),
+      ).toThrow(/apiBaseUrl/);
+      expect(() =>
+        readConfig(config([{ host: 'github.com', token: 7 } as any])),
+      ).toThrow(/token/);
+    });
+  });
+
+  describe('implementation', () => {
+    it('rejects unknown targets', async () => {
+      const processor = new GithubUrlReader({
+        host: 'github.com',
+        apiBaseUrl: 'https://api.github.com',
+      });
+      await expect(
+        processor.read('https://not.github.com/apa'),
+      ).rejects.toThrow(
+        'Incorrect URL: https://not.github.com/apa, Error: Invalid GitHub URL or file path',
+      );
+    });
+  });
+});

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -0,0 +1,236 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Config } from '@backstage/config';
+import parseGitUri from 'git-url-parse';
+import fetch, { HeadersInit, RequestInit, Response } from 'node-fetch';
+import { NotFoundError } from '../errors';
+import { ReaderFactory, UrlReader } from './types';
+
+/**
+ * The configuration parameters for a single GitHub API provider.
+ */
+export type ProviderConfig = {
+  /**
+   * The host of the target that this matches on, e.g. "github.com"
+   */
+  host: string;
+
+  /**
+   * The base URL of the API of this provider, e.g. "https://api.github.com",
+   * with no trailing slash.
+   *
+   * May be omitted specifically for GitHub; then it will be deduced.
+   *
+   * The API will always be preferred if both its base URL and a token are
+   * present.
+   */
+  apiBaseUrl?: string;
+
+  /**
+   * The base URL of the raw fetch endpoint of this provider, e.g.
+   * "https://raw.githubusercontent.com", with no trailing slash.
+   *
+   * May be omitted specifically for GitHub; then it will be deduced.
+   *
+   * The API will always be preferred if both its base URL and a token are
+   * present.
+   */
+  rawBaseUrl?: string;
+
+  /**
+   * The authorization token to use for requests to this provider.
+   *
+   * If no token is specified, anonymous access is used.
+   */
+  token?: string;
+};
+
+export function getApiRequestOptions(provider: ProviderConfig): RequestInit {
+  const headers: HeadersInit = {
+    Accept: 'application/vnd.github.v3.raw',
+  };
+
+  if (provider.token) {
+    headers.Authorization = `token ${provider.token}`;
+  }
+
+  return {
+    headers,
+  };
+}
+
+export function getRawRequestOptions(provider: ProviderConfig): RequestInit {
+  const headers: HeadersInit = {};
+
+  if (provider.token) {
+    headers.Authorization = `token ${provider.token}`;
+  }
+
+  return {
+    headers,
+  };
+}
+
+// Converts for example
+// from: https://github.com/a/b/blob/branchname/path/to/c.yaml
+// to:   https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=branchname
+export function getApiUrl(target: string, provider: ProviderConfig): URL {
+  try {
+    const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
+
+    if (
+      !owner ||
+      !name ||
+      !ref ||
+      (filepathtype !== 'blob' && filepathtype !== 'raw')
+    ) {
+      throw new Error('Invalid GitHub URL or file path');
+    }
+
+    const pathWithoutSlash = filepath.replace(/^\//, '');
+    return new URL(
+      `${provider.apiBaseUrl}/repos/${owner}/${name}/contents/${pathWithoutSlash}?ref=${ref}`,
+    );
+  } catch (e) {
+    throw new Error(`Incorrect URL: ${target}, ${e}`);
+  }
+}
+
+// Converts for example
+// from: https://github.com/a/b/blob/branchname/c.yaml
+// to:   https://raw.githubusercontent.com/a/b/branchname/c.yaml
+export function getRawUrl(target: string, provider: ProviderConfig): URL {
+  try {
+    const { owner, name, ref, filepathtype, filepath } = parseGitUri(target);
+
+    if (
+      !owner ||
+      !name ||
+      !ref ||
+      (filepathtype !== 'blob' && filepathtype !== 'raw')
+    ) {
+      throw new Error('Invalid GitHub URL or file path');
+    }
+
+    const pathWithoutSlash = filepath.replace(/^\//, '');
+    return new URL(
+      `${provider.rawBaseUrl}/${owner}/${name}/${ref}/${pathWithoutSlash}`,
+    );
+  } catch (e) {
+    throw new Error(`Incorrect URL: ${target}, ${e}`);
+  }
+}
+
+export function readConfig(config: Config): ProviderConfig[] {
+  const providers: ProviderConfig[] = [];
+
+  const providerConfigs =
+    config.getOptionalConfigArray('integrations.github') ?? [];
+
+  // First read all the explicit providers
+  for (const providerConfig of providerConfigs) {
+    const host = providerConfig.getOptionalString('host') ?? 'github.com';
+    let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl');
+    let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl');
+    const token = providerConfig.getOptionalString('token');
+
+    if (apiBaseUrl) {
+      apiBaseUrl = apiBaseUrl.replace(/\/+$/, '');
+    } else if (host === 'github.com') {
+      apiBaseUrl = 'https://api.github.com';
+    }
+
+    if (rawBaseUrl) {
+      rawBaseUrl = rawBaseUrl.replace(/\/+$/, '');
+    } else if (host === 'github.com') {
+      rawBaseUrl = 'https://raw.githubusercontent.com';
+    }
+
+    if (!apiBaseUrl && !rawBaseUrl) {
+      throw new Error(
+        `GitHub integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`,
+      );
+    }
+
+    providers.push({ host, apiBaseUrl, rawBaseUrl, token });
+  }
+
+  // If no explicit github.com provider was added, put one in the list as
+  // a convenience
+  if (!providers.some(p => p.host === 'github.com')) {
+    providers.push({
+      host: 'github.com',
+      apiBaseUrl: 'https://api.github.com',
+      rawBaseUrl: 'https://raw.githubusercontent.com',
+    });
+  }
+
+  return providers;
+}
+
+/**
+ * A processor that adds the ability to read files from GitHub v3 APIs, such as
+ * the one exposed by GitHub itself.
+ */
+export class GithubUrlReader implements UrlReader {
+  private config: ProviderConfig;
+
+  static factory: ReaderFactory = ({ config }) => {
+    return readConfig(config).map(provider => {
+      const reader = new GithubUrlReader(provider);
+      const predicate = (url: URL) => url.host === provider.host;
+      return { reader, predicate };
+    });
+  };
+
+  constructor(config: ProviderConfig) {
+    this.config = config;
+  }
+
+  async read(url: string): Promise<Buffer> {
+    const useApi =
+      this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl);
+    const ghUrl = useApi
+      ? getApiUrl(url, this.config)
+      : getRawUrl(url, this.config);
+    const options = useApi
+      ? getApiRequestOptions(this.config)
+      : getRawRequestOptions(this.config);
+
+    let response: Response;
+    try {
+      response = await fetch(ghUrl.toString(), options);
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    if (response.ok) {
+      return response.buffer();
+    }
+
+    const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+
+  toString() {
+    const { host, token } = this.config;
+    return `github{host=${host},authed=${Boolean(token)}}`;
+  }
+}

packages/backend-common/src/reading/GitlabUrlReader.test.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { ConfigReader } from '@backstage/config';
+import { getVoidLogger } from '../logging';
+import { GitlabUrlReader } from './GitlabUrlReader';
+
+const logger = getVoidLogger();
+
+describe('GitlabUrlReader', () => {
+  const worker = setupServer();
+
+  beforeAll(() => worker.listen({ onUnhandledRequest: 'error' }));
+  afterAll(() => worker.close());
+
+  beforeEach(() => {
+    worker.use(
+      rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+        res(ctx.status(200), ctx.json({ id: 12345 })),
+      ),
+      rest.get('*', (req, res, ctx) =>
+        res(
+          ctx.status(200),
+          ctx.json({
+            url: req.url.toString(),
+            headers: req.headers.getAllHeaders(),
+          }),
+        ),
+      ),
+    );
+  });
+  afterEach(() => worker.resetHandlers());
+
+  const createConfig = (token?: string) =>
+    new ConfigReader(
+      {
+        integrations: { gitlab: [{ host: 'gitlab.com', token }] },
+      },
+      'test-config',
+    );
+
+  it.each([
+    // Project URLs
+    {
+      url:
+        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+      config: createConfig(),
+      response: expect.objectContaining({
+        url:
+          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+        headers: expect.objectContaining({
+          'private-token': '',
+        }),
+      }),
+    },
+    {
+      url:
+        'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+      config: createConfig('0123456789'),
+      response: expect.objectContaining({
+        url:
+          'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+        headers: expect.objectContaining({
+          'private-token': '0123456789',
+        }),
+      }),
+    },
+    {
+      url:
+        'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
+      config: createConfig(),
+      response: expect.objectContaining({
+        url:
+          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+      }),
+    },
+
+    // Raw URLs
+    {
+      url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
+      config: createConfig(),
+      response: expect.objectContaining({
+        url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
+      }),
+    },
+  ])('should handle happy path %#', async ({ url, config, response }) => {
+    const [{ reader }] = GitlabUrlReader.factory({ config, logger });
+
+    const data = await reader.read(url);
+    const res = await JSON.parse(data.toString('utf-8'));
+    expect(res).toEqual(response);
+  });
+
+  it.each([
+    {
+      url: '',
+      config: createConfig(''),
+      error:
+        "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
+    },
+  ])('should handle error path %#', async ({ url, config, error }) => {
+    await expect(async () => {
+      const [{ reader }] = GitlabUrlReader.factory({ config, logger });
+      await reader.read(url);
+    }).rejects.toThrow(error);
+  });
+});

packages/backend-common/src/reading/GitlabUrlReader.ts

@@ -0,0 +1,197 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch, { RequestInit, Response } from 'node-fetch';
+import { Config } from '@backstage/config';
+import { NotFoundError } from '../errors';
+import { ReaderFactory, UrlReader } from './types';
+
+type Options = {
+  host: string;
+  token?: string;
+};
+
+function readConfig(config: Config): Options[] {
+  const optionsArr = Array<Options>();
+
+  const providerConfigs =
+    config.getOptionalConfigArray('integrations.gitlab') ?? [];
+
+  for (const providerConfig of providerConfigs) {
+    const host = providerConfig.getOptionalString('host') ?? 'gitlab.com';
+    const token = providerConfig.getOptionalString('token');
+
+    optionsArr.push({ host, token });
+  }
+
+  // As a convenience we always make sure there's at least an unauthenticated
+  // reader for public gitlab repos.
+  if (!optionsArr.some(p => p.host === 'gitlab.com')) {
+    optionsArr.push({ host: 'gitlab.com' });
+  }
+
+  return optionsArr;
+}
+
+export class GitlabUrlReader implements UrlReader {
+  static factory: ReaderFactory = ({ config }) => {
+    return readConfig(config).map(options => {
+      const reader = new GitlabUrlReader(options);
+      const predicate = (url: URL) => url.host === options.host;
+      return { reader, predicate };
+    });
+  };
+
+  constructor(private readonly options: Options) {}
+
+  async read(url: string): Promise<Buffer> {
+    // TODO(Rugvip): merged the old GitlabReaderProcessor in here and used
+    // the existence of /~/blob/ to switch the logic. Don't know if this
+    // makes sense and it might require some more work.
+    let builtUrl: URL;
+    if (url.includes('/-/blob/')) {
+      const projectID = await this.getProjectID(url);
+      builtUrl = this.buildProjectUrl(url, projectID);
+    } else {
+      builtUrl = this.buildRawUrl(url);
+    }
+
+    let response: Response;
+    try {
+      response = await fetch(builtUrl.toString(), this.getRequestOptions());
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    if (response.ok) {
+      return response.buffer();
+    }
+
+    const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+
+  // Converts
+  // from: https://gitlab.example.com/a/b/blob/master/c.yaml
+  // to:   https://gitlab.example.com/a/b/raw/master/c.yaml
+  private buildRawUrl(target: string): URL {
+    try {
+      const url = new URL(target);
+
+      const [
+        empty,
+        userOrOrg,
+        repoName,
+        blobKeyword,
+        ...restOfPath
+      ] = url.pathname.split('/');
+
+      if (
+        empty !== '' ||
+        userOrOrg === '' ||
+        repoName === '' ||
+        blobKeyword !== 'blob' ||
+        !restOfPath.join('/').match(/\.yaml$/)
+      ) {
+        throw new Error('Wrong GitLab URL');
+      }
+
+      // Replace 'blob' with 'raw'
+      url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join(
+        '/',
+      );
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+
+  // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
+  // to https://gitlab.com/api/v4/projects/<PROJECTID>/repository/files/filepath?ref=branch
+  private buildProjectUrl(target: string, projectID: Number): URL {
+    try {
+      const url = new URL(target);
+
+      const branchAndfilePath = url.pathname.split('/-/blob/')[1];
+
+      const [branch, ...filePath] = branchAndfilePath.split('/');
+
+      url.pathname = [
+        '/api/v4/projects',
+        projectID,
+        'repository/files',
+        encodeURIComponent(filePath.join('/')),
+        'raw',
+      ].join('/');
+      url.search = `?ref=${branch}`;
+
+      return url;
+    } catch (e) {
+      throw new Error(`Incorrect url: ${target}, ${e}`);
+    }
+  }
+
+  private async getProjectID(target: string): Promise<Number> {
+    const url = new URL(target);
+
+    if (
+      // absPaths to gitlab files should contain /-/blob
+      // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath
+      !url.pathname.match(/\/\-\/blob\//)
+    ) {
+      throw new Error('Please provide full path to yaml file from Gitlab');
+    }
+    try {
+      const repo = url.pathname.split('/-/blob/')[0];
+
+      // Find ProjectID from url
+      // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath'
+      // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo'
+      const repoIDLookup = new URL(
+        `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent(
+          repo.replace(/^\//, ''),
+        )}`,
+      );
+      const response = await fetch(
+        repoIDLookup.toString(),
+        this.getRequestOptions(),
+      );
+      const projectIDJson = await response.json();
+      const projectID: Number = projectIDJson.id;
+
+      return projectID;
+    } catch (e) {
+      throw new Error(`Could not get GitLab ProjectID for: ${target}, ${e}`);
+    }
+  }
+
+  private getRequestOptions(): RequestInit {
+    return {
+      headers: {
+        ['PRIVATE-TOKEN']: this.options.token ?? '',
+      },
+    };
+  }
+
+  toString() {
+    const { host, token } = this.options;
+    return `gitlab{host=${host},authed=${Boolean(token)}}`;
+  }
+}

packages/backend-common/src/reading/UrlReaderPredicateMux.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader, UrlReaderPredicateTuple } from './types';
+
+type Options = {
+  // UrlReader to fall back to if no other reader is matched
+  fallback?: UrlReader;
+};
+
+/**
+ * A UrlReader implementation that selects from a set of UrlReaders
+ * based on a predicate tied to each reader.
+ */
+export class UrlReaderPredicateMux implements UrlReader {
+  private readonly readers: UrlReaderPredicateTuple[] = [];
+  private readonly fallback?: UrlReader;
+
+  constructor({ fallback }: Options) {
+    this.fallback = fallback;
+  }
+
+  register(tuple: UrlReaderPredicateTuple): void {
+    this.readers.push(tuple);
+  }
+
+  read(url: string): Promise<Buffer> {
+    const parsed = new URL(url);
+
+    for (const { predicate, reader } of this.readers) {
+      if (predicate(parsed)) {
+        return reader.read(url);
+      }
+    }
+
+    if (this.fallback) {
+      return this.fallback.read(url);
+    }
+
+    throw new Error(`No reader found that could handle '${url}'`);
+  }
+
+  toString() {
+    return `predicateMux{readers=${this.readers
+      .map(t => t.reader)
+      .join(',')},fallback=${this.fallback}}`;
+  }
+}

packages/backend-common/src/reading/UrlReaders.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+import { ReaderFactory, UrlReader } from './types';
+import { UrlReaderPredicateMux } from './UrlReaderPredicateMux';
+import { AzureUrlReader } from './AzureUrlReader';
+import { BitbucketUrlReader } from './BitbucketUrlReader';
+import { GithubUrlReader } from './GithubUrlReader';
+import { GitlabUrlReader } from './GitlabUrlReader';
+import { FetchUrlReader } from './FetchUrlReader';
+
+type CreateOptions = {
+  /** Root config object */
+  config: Config;
+  /** Logger used by all the readers */
+  logger: Logger;
+  /** A list of factories used to construct individual readers that match on URLs */
+  factories?: ReaderFactory[];
+  /** Fallback reader to use if none of the readers created by the factories match */
+  fallback?: UrlReader;
+};
+
+/**
+ * UrlReaders provide various utilities related to the UrlReader interface.
+ */
+export class UrlReaders {
+  /**
+   * Creates a UrlReader without any known types.
+   */
+  static create({
+    logger,
+    config,
+    factories,
+    fallback,
+  }: CreateOptions): UrlReader {
+    const mux = new UrlReaderPredicateMux({ fallback: fallback });
+
+    for (const factory of factories ?? []) {
+      const tuples = factory({ config, logger: logger });
+
+      for (const tuple of tuples) {
+        mux.register(tuple);
+      }
+    }
+
+    return mux;
+  }
+
+  /**
+   * Creates a UrlReader that includes all the default factories from this package.
+   *
+   * Any additional factories passed will be loaded before the default ones.
+   *
+   * If no fallback reader is passed, a plain fetch reader will be used.
+   */
+  static default({ logger, config, factories = [], fallback }: CreateOptions) {
+    return UrlReaders.create({
+      logger,
+      config,
+      factories: factories.concat([
+        AzureUrlReader.factory,
+        BitbucketUrlReader.factory,
+        GithubUrlReader.factory,
+        GitlabUrlReader.factory,
+      ]),
+      fallback: fallback ?? new FetchUrlReader(),
+    });
+  }
+}

packages/backend-common/src/reading/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type { UrlReader } from './types';
+export { UrlReaders } from './UrlReaders';
+export { AzureUrlReader } from './AzureUrlReader';
+export { BitbucketUrlReader } from './BitbucketUrlReader';
+export { GithubUrlReader } from './GithubUrlReader';
+export { GitlabUrlReader } from './GitlabUrlReader';

packages/backend-common/src/reading/types.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+
+/**
+ * A generic interface for fetching plain data from URLs.
+ */
+export type UrlReader = {
+  read(url: string): Promise<Buffer>;
+};
+
+export type UrlReaderPredicateTuple = {
+  predicate: (url: URL) => boolean;
+  reader: UrlReader;
+};
+
+/**
+ * A factory function that can read config to construct zero or more
+ * UrlReaders along with a predicate for when it should be used.
+ */
+export type ReaderFactory = (options: {
+  config: Config;
+  logger: Logger;
+}) => UrlReaderPredicateTuple[];

packages/backend-common/src/setupTests.ts

@@ -14,6 +14,4 @@
  * limitations under the License.
  */
 
-require('jest-fetch-mock').enableMocks();
-
 export {};

packages/backend/src/index.ts

@@ -32,6 +32,7 @@ import {
   useHotMemoize,
   notFoundHandler,
   SingleHostDiscovery,
+  UrlReaders,
 } from '@backstage/backend-common';
 import { ConfigReader, AppConfig } from '@backstage/config';
 import healthcheck from './plugins/healthcheck';
@@ -49,9 +50,14 @@ import { PluginEnvironment } from './types';
 
 function makeCreateEnv(loadedConfigs: AppConfig[]) {
   const config = ConfigReader.fromConfigs(loadedConfigs);
+  const root = getRootLogger();
+  const reader = UrlReaders.default({ logger: root, config });
+  const discovery = SingleHostDiscovery.fromConfig(config);
+
+  root.info(`Created UrlReader ${reader}`);
 
   return (plugin: string): PluginEnvironment => {
-    const logger = getRootLogger().child({ type: 'plugin', plugin });
+    const logger = root.child({ type: 'plugin', plugin });
     const database = createDatabaseClient(
       config.getConfig('backend.database'),
       {
@@ -60,8 +66,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
         },
       },
     );
-    const discovery = SingleHostDiscovery.fromConfig(config);
-    return { logger, database, config, discovery };
+    return { logger, database, config, reader, discovery };
   };
 }
 

packages/backend/src/plugins/catalog.ts

@@ -28,10 +28,11 @@ import { useHotCleanup } from '@backstage/backend-common';
 
 export default async function createPlugin({
   logger,
-  database,
   config,
+  reader,
+  database,
 }: PluginEnvironment) {
-  const locationReader = new LocationReaders({ logger, config });
+  const locationReader = new LocationReaders({ logger, reader, config });
 
   const db = await DatabaseManager.createDatabase(database, { logger });
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);

packages/backend/src/types.ts

@@ -17,11 +17,12 @@
 import Knex from 'knex';
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
 
 export type PluginEnvironment = {
   logger: Logger;
   database: Knex;
   config: Config;
+  reader: UrlReader;
   discovery: PluginEndpointDiscovery;
 };

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -82,31 +82,31 @@ catalog:
         #       env: GHE_PRIVATE_TOKEN
   locations:
     # Backstage example components
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-components.yaml
 
     # Backstage example APIs
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml
 
     # Backstage example templates
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml
       rules:
         - allow: [Template]
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml
       rules:
         - allow: [Template]
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml
       rules:
         - allow: [Template]
-    - type: github
+    - type: url
       target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml
       rules:
         - allow: [Template]
-    - type: github
+    - type: url
       target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml
       rules:
         - allow: [Template]

packages/create-app/templates/default-app/packages/backend/src/index.ts

@@ -16,6 +16,7 @@ import {
   useHotMemoize,
   notFoundHandler,
   SingleHostDiscovery,
+  UrlReaders,
 } from '@backstage/backend-common';
 import { ConfigReader, AppConfig } from '@backstage/config';
 import auth from './plugins/auth';
@@ -27,9 +28,14 @@ import { PluginEnvironment } from './types';
 
 function makeCreateEnv(loadedConfigs: AppConfig[]) {
   const config = ConfigReader.fromConfigs(loadedConfigs);
+  const root = getRootLogger();
+  const reader = UrlReaders.default({ logger: root, config });
+  const discovery = SingleHostDiscovery.fromConfig(config);
+
+  root.info(`Created UrlReader ${reader}`);
 
   return (plugin: string): PluginEnvironment => {
-    const logger = getRootLogger().child({ type: 'plugin', plugin });
+    const logger = root.child({ type: 'plugin', plugin });
     const database = createDatabaseClient(
       config.getConfig('backend.database'),
       {
@@ -38,8 +44,7 @@ function makeCreateEnv(loadedConfigs: AppConfig[]) {
         },
       },
     );
-    const discovery = SingleHostDiscovery.fromConfig(config);
-    return { logger, database, config, discovery };
+    return { logger, database, config, reader, discovery };
   };
 }
 

packages/create-app/templates/default-app/packages/backend/src/plugins/catalog.ts

@@ -13,9 +13,10 @@ import { useHotCleanup } from '@backstage/backend-common';
 export default async function createPlugin({
   logger,
   config,
+  reader,
   database,
 }: PluginEnvironment) {
-  const locationReader = new LocationReaders({ logger, config });
+  const locationReader = new LocationReaders({ logger, reader, config });
 
   const db = await DatabaseManager.createDatabase(database, { logger });
   const entitiesCatalog = new DatabaseEntitiesCatalog(db);

packages/create-app/templates/default-app/packages/backend/src/types.ts

@@ -1,11 +1,12 @@
 import Knex from 'knex';
 import { Logger } from 'winston';
 import { Config } from '@backstage/config';
-import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
 
 export type PluginEnvironment = {
   logger: Logger;
   database: Knex;
   config: Config;
+  reader: UrlReader
   discovery: PluginEndpointDiscovery;
 };