backend-common: added FetchUrlReader and use as fallback by default

packages/backend-common/src/reading/FetchUrlReader.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch, { Response } from 'node-fetch';
+import { NotFoundError } from '../errors';
+import { UrlReader } from './types';
+
+/**
+ * A UrlReader that does a plain fetch of the URL.
+ */
+export class FetchUrlReader implements UrlReader {
+  async read(url: string): Promise<Buffer> {
+    let response: Response;
+    try {
+      response = await fetch(url);
+    } catch (e) {
+      throw new Error(`Unable to read ${url}, ${e}`);
+    }
+
+    if (response.ok) {
+      return response.buffer();
+    }
+
+    const message = `could not read ${url}, ${response.status} ${response.statusText}`;
+    if (response.status === 404) {
+      throw new NotFoundError(message);
+    }
+    throw new Error(message);
+  }
+}

packages/backend-common/src/reading/UrlReaderPredicateMux.ts

@@ -23,12 +23,22 @@ export type UrlReaderPredicateTuple = {
   reader: UrlReader;
 };
 
+type Options = {
+  // UrlReader to fall back to if no other reader is matched
+  fallback?: UrlReader;
+};
+
 /**
  * A UrlReader implementation that selects from a set of UrlReaders
  * based on a predicate tied to each reader.
  */
 export class UrlReaderPredicateMux implements UrlReader {
   private readonly readers: UrlReaderPredicateTuple[] = [];
+  private readonly fallback?: UrlReader;
+
+  constructor({ fallback }: Options) {
+    this.fallback = fallback;
+  }
 
   register(tuple: UrlReaderPredicateTuple): void {
     this.readers.push(tuple);
@@ -43,6 +53,10 @@ export class UrlReaderPredicateMux implements UrlReader {
       }
     }
 
+    if (this.fallback) {
+      return this.fallback.read(url);
+    }
+
     throw new Error(`No reader found that could handle '${url}'`);
   }
 }

packages/backend-common/src/reading/UrlReaders.ts

@@ -25,6 +25,7 @@ import { AzureUrlReader } from './AzureUrlReader';
 import { BitbucketUrlReader } from './BitbucketUrlReader';
 import { GithubUrlReader } from './GithubUrlReader';
 import { GitlabUrlReader } from './GitlabUrlReader';
+import { FetchUrlReader } from './FetchUrlReader';
 
 export type ReaderFactoryOptions = {
   config: Config;
@@ -62,12 +63,14 @@ export class UrlReaders {
         GitlabUrlReader.factory,
       ],
       logger,
+      new FetchUrlReader(),
     );
   }
 
   private constructor(
     private readonly factories: ReaderFactory[],
     private readonly logger: Logger,
+    private fallback?: UrlReader,
   ) {}
 
   /**
@@ -75,7 +78,7 @@ export class UrlReaders {
    * reader type needs to have a registered factory, or an error will be thrown.
    */
   createWithConfig(config: Config): UrlReader {
-    const mux = new UrlReaderPredicateMux();
+    const mux = new UrlReaderPredicateMux({ fallback: this.fallback });
     const readers = [];
 
     for (const factory of this.factories) {
@@ -100,4 +103,8 @@ export class UrlReaders {
   addFactory(factory: ReaderFactory) {
     this.factories.push(factory);
   }
+
+  setFallback(reader?: UrlReader) {
+    this.fallback = reader;
+  }
 }