catalog-backend: switch UrlReaderProcessor to use UrlReader

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { getVoidLogger } from '@backstage/backend-common';
+import { getVoidLogger, UrlReader } from '@backstage/backend-common';
 import {
   Entity,
   EntityPolicies,
@@ -56,6 +56,7 @@ import { LocationReader, ReadLocationResult } from './types';
 const MAX_DEPTH = 10;
 
 type Options = {
+  reader?: UrlReader;
   logger?: Logger;
   config?: Config;
   processors?: LocationProcessor[];
@@ -71,6 +72,7 @@ export class LocationReaders implements LocationReader {
 
   static defaultProcessors(options: {
     logger: Logger;
+    reader?: UrlReader;
     config?: Config;
     entityPolicy?: EntityPolicy;
   }): LocationProcessor[] {
@@ -86,7 +88,7 @@ export class LocationReaders implements LocationReader {
       new BitbucketApiReaderProcessor(config),
       new AzureApiReaderProcessor(config),
       GithubOrgReaderProcessor.fromConfig(config),
-      new UrlReaderProcessor(),
+      options.reader ? new UrlReaderProcessor(options.reader) : [],
       new YamlProcessor(),
       PlaceholderProcessor.default(),
       new CodeOwnersProcessor(),
@@ -94,13 +96,14 @@ export class LocationReaders implements LocationReader {
       new EntityPolicyProcessor(entityPolicy),
       new LocationRefProcessor(),
       new AnnotateLocationEntityProcessor(),
-    ];
+    ].flat();
   }
 
   constructor({
     logger = getVoidLogger(),
     config,
-    processors = LocationReaders.defaultProcessors({ logger, config }),
+    reader,
+    processors = LocationReaders.defaultProcessors({ logger, reader, config }),
   }: Options) {
     this.logger = logger;
     this.processors = processors;

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+import { UrlReader } from '@backstage/backend-common';
 import { LocationSpec } from '@backstage/catalog-model';
-import fetch from 'node-fetch';
 import * as result from './results';
 import { LocationProcessor, LocationProcessorEmit } from './types';
 
 export class UrlReaderProcessor implements LocationProcessor {
+  constructor(private readonly reader: UrlReader) {}
+
   async readLocation(
     location: LocationSpec,
     optional: boolean,
@@ -30,24 +32,18 @@ export class UrlReaderProcessor implements LocationProcessor {
     }
 
     try {
-      const response = await fetch(location.target);
+      const data = await this.reader.read(location.target);
+      emit(result.data(location, data));
+    } catch (error) {
+      const message = `Unable to read ${location.type}, ${error}`;
 
-      if (response.ok) {
-        const data = await response.buffer();
-        emit(result.data(location, data));
-      } else {
-        const message = `${location.target} could not be read, ${response.status} ${response.statusText}`;
-        if (response.status === 404) {
-          if (!optional) {
-            emit(result.notFoundError(location, message));
-          }
-        } else {
-          emit(result.generalError(location, message));
+      if (error.name === 'NotFoundError') {
+        if (!optional) {
+          emit(result.notFoundError(location, message));
         }
+      } else {
+        emit(result.generalError(location, message));
       }
-    } catch (e) {
-      const message = `Unable to read ${location.type} ${location.target}, ${e}`;
-      emit(result.generalError(location, message));
     }
 
     return true;