Merge pull request #6305 from backstage/mob/read-url

backend-common: add new optional readUrl method to UrlReader

plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts

@@ -94,12 +94,21 @@ export class PlaceholderProcessor implements CatalogProcessor {
         return [data, false];
       }
 
+      const read = async (url: string): Promise<Buffer> => {
+        if (this.options.reader.readUrl) {
+          const response = await this.options.reader.readUrl(url);
+          const buffer = await response.buffer();
+          return buffer;
+        }
+        return this.options.reader.read(url);
+      };
+
       return [
         await resolver({
           key: resolverKey,
           value: resolverValue,
           baseUrl: location.target,
-          read: this.options.reader.read.bind(this.options.reader),
+          read,
         }),
         true,
       ];

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -101,7 +101,12 @@ export class UrlReaderProcessor implements CatalogProcessor {
       return Promise.all(output);
     }
 
-    // Otherwise do a plain read
+    // Otherwise do a plain read, prioritizing readUrl if available
+    if (this.options.reader.readUrl) {
+      const data = await this.options.reader.readUrl(location);
+      return [{ url: location, data: await data.buffer() }];
+    }
+
     const data = await this.options.reader.read(location);
     return [{ url: location, data }];
   }

plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts

@@ -28,6 +28,12 @@ export async function readCodeOwners(
 ): Promise<string | undefined> {
   const readOwnerLocation = async (path: string): Promise<string> => {
     const url = `${sourceUrl}${path}`;
+
+    if (reader.readUrl) {
+      const data = await reader.readUrl(url);
+      const buffer = await data.buffer();
+      return buffer.toString();
+    }
     const data = await reader.read(url);
     return data.toString();
   };