Merge pull request #6305 from backstage/mob/read-url

backend-common: add new optional readUrl method to UrlReader

.changeset/honest-cars-glow.md

@@ -0,0 +1,54 @@
+---
+'@backstage/backend-common': patch
+---
+
+Added a `readUrl` method to the `UrlReader` interface that allows for complex response objects and is intended to replace the `read` method. This new method is currently optional to implement which allows for a soft migration to `readUrl` instead of `read` in the future.
+
+The main use case for `readUrl` returning an object instead of solely a read buffer is to allow for additional metadata such as ETag, which is a requirement for more efficient catalog processing.
+
+The `GithubUrlReader` and `GitlabUrlReader` readers fully implement `readUrl`. The other existing readers implement the new method but do not propagate or return ETags.
+
+While the `readUrl` method is not yet required, it will be in the future, and we already log deprecation warnings when custom `UrlReader` implementations that do not implement `readUrl` are used. We therefore recommend that any existing custom implementations are migrated to implement `readUrl`.
+
+The old `read` and the new `readUrl` methods can easily be implemented using one another, but we recommend moving the chunk of the implementation to the new `readUrl` method as `read` is being removed, for example this:
+
+```ts
+class CustomUrlReader implements UrlReader {
+  read(url: string): Promise<Buffer> {
+    const res = await fetch(url);
+
+    if (!res.ok) {
+      // error handling ...
+    }
+
+    return Buffer.from(await res.text());
+  }
+}
+```
+
+Can be migrated to something like this:
+
+```ts
+class CustomUrlReader implements UrlReader {
+  read(url: string): Promise<Buffer> {
+    const res = await this.readUrl(url);
+    return res.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    const res = await fetch(url);
+
+    if (!res.ok) {
+      // error handling ...
+    }
+
+    const buffer = Buffer.from(await res.text());
+    return { buffer: async () => buffer };
+  }
+}
+```
+
+While there is no usage of the ETag capability yet in the main Backstage packages, you can already add it to your custom implementations. To do so, refer to the documentation of the `readUrl` method and surrounding types, and the existing implementation in `packages/backend-common/src/reading/GithubUrlReader.ts`.

.changeset/rotten-walls-cross.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Make use of the new `readUrl` method on `UrlReader` from `@backstage/backend-common`.

packages/backend-common/api-report.md

@@ -42,6 +42,8 @@ export class AzureUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -59,6 +61,8 @@ export class BitbucketUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, _options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -232,6 +236,8 @@ export class GithubUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -249,6 +255,8 @@ export class GitlabUrlReader implements UrlReader {
     // (undocumented)
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     // (undocumented)
+    readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+    // (undocumented)
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
     // (undocumented)
     toString(): string;
@@ -278,7 +286,7 @@ export type PluginEndpointDiscovery = {
     getExternalBaseUrl(pluginId: string): Promise<string>;
 };
 
-// @public (undocumented)
+// @public
 export type ReadTreeResponse = {
     files(): Promise<ReadTreeResponseFile[]>;
     archive(): Promise<NodeJS.ReadableStream>;
@@ -367,6 +375,7 @@ export interface StatusCheckHandlerOptions {
 // @public
 export type UrlReader = {
     read(url: string): Promise<Buffer>;
+    readUrl?(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
     readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
     search(url: string, options?: SearchOptions): Promise<SearchResponse>;
 };

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -36,6 +36,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlOptions,
+  ReadUrlResponse,
 } from './types';
 
 export class AzureUrlReader implements UrlReader {
@@ -78,6 +80,15 @@ export class AzureUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/BitbucketUrlReader.test.ts

@@ -74,7 +74,24 @@ describe('BitbucketUrlReader', () => {
   const worker = setupServer();
   msw.setupDefaultHandlers(worker);
 
-  describe('implementation', () => {
+  describe('readUrl', () => {
+    worker.use(
+      rest.get(
+        'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
+        (_, res, ctx) => res(ctx.status(200), ctx.body('foo')),
+      ),
+    );
+
+    it('should be able to readUrl', async () => {
+      const result = await bitbucketProcessor.readUrl(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      const buffer = await result.buffer();
+      expect(buffer.toString()).toBe('foo');
+    });
+  });
+
+  describe('read', () => {
     it('rejects unknown targets', async () => {
       await expect(
         bitbucketProcessor.read('https://not.bitbucket.com/apa'),

packages/backend-common/src/reading/BitbucketUrlReader.ts

@@ -36,6 +36,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlResponse,
+  ReadUrlOptions,
 } from './types';
 
 /**
@@ -99,6 +101,15 @@ export class BitbucketUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/FetchUrlReader.ts

@@ -19,6 +19,8 @@ import { NotFoundError } from '@backstage/errors';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchResponse,
   UrlReader,
 } from './types';
@@ -73,6 +75,15 @@ export class FetchUrlReader implements UrlReader {
     throw new Error(message);
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(): Promise<ReadTreeResponse> {
     throw new Error('FetchUrlReader does not implement readTree');
   }

packages/backend-common/src/reading/GithubUrlReader.test.ts

@@ -141,6 +141,113 @@ describe('GithubUrlReader', () => {
     });
   });
 
+  /*
+   * readUrl
+   */
+  describe('readUrl', () => {
+    it('should use the headers from the credentials provider to the fetch request when doing readUrl', async () => {
+      expect.assertions(2);
+
+      const mockHeaders = {
+        Authorization: 'bearer blah',
+        otherheader: 'something',
+      };
+
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: mockHeaders,
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              mockHeaders.Authorization,
+            );
+            expect(req.headers.get('otherheader')).toBe(
+              mockHeaders.otherheader,
+            );
+            return res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body('foo'),
+            );
+          },
+        ),
+      );
+
+      await gheProcessor.readUrl(
+        'https://github.com/backstage/mock/tree/blob/main',
+      );
+    });
+
+    it('should throw NotModified if GitHub responds with 304', async () => {
+      expect.assertions(4);
+
+      const mockHeaders = {
+        Authorization: 'bearer blah',
+        otherheader: 'something',
+      };
+
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: mockHeaders,
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (req, res, ctx) => {
+            expect(req.headers.get('authorization')).toBe(
+              mockHeaders.Authorization,
+            );
+            expect(req.headers.get('otherheader')).toBe(
+              mockHeaders.otherheader,
+            );
+            expect(req.headers.get('if-none-match')).toBe('foo');
+            return res(
+              ctx.status(304),
+              ctx.set('Content-Type', 'application/x-gzip'),
+              ctx.body('foo'),
+            );
+          },
+        ),
+      );
+
+      await expect(
+        gheProcessor.readUrl(
+          'https://github.com/backstage/mock/tree/blob/main',
+          { etag: 'foo' },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return etag from the response', async () => {
+      (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({
+        headers: {
+          Authorization: 'bearer blah',
+        },
+      });
+
+      worker.use(
+        rest.get(
+          'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/',
+          (_req, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.set('Etag', 'foo'),
+              ctx.body('bar'),
+            );
+          },
+        ),
+      );
+
+      const response = await gheProcessor.readUrl(
+        'https://github.com/backstage/mock/tree/blob/main',
+      );
+      expect(response.etag).toBe('foo');
+    });
+  });
+
   /*
    * readTree
    */

packages/backend-common/src/reading/GithubUrlReader.ts

@@ -35,6 +35,8 @@ import {
   SearchResponse,
   SearchResponseFile,
   UrlReader,
+  ReadUrlOptions,
+  ReadUrlResponse,
 } from './types';
 
 export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data'];
@@ -77,6 +79,14 @@ export class GithubUrlReader implements UrlReader {
   }
 
   async read(url: string): Promise<Buffer> {
+    const response = await this.readUrl(url);
+    return response.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
     const ghUrl = getGitHubFileFetchUrl(url, this.integration.config);
     const { headers } = await this.deps.credentialsProvider.getCredentials({
       url,
@@ -86,6 +96,7 @@ export class GithubUrlReader implements UrlReader {
       response = await fetch(ghUrl.toString(), {
         headers: {
           ...headers,
+          ...(options?.etag && { 'If-None-Match': options.etag }),
           Accept: 'application/vnd.github.v3.raw',
         },
       });
@@ -93,8 +104,15 @@ export class GithubUrlReader implements UrlReader {
       throw new Error(`Unable to read ${url}, ${e}`);
     }
 
+    if (response.status === 304) {
+      throw new NotModifiedError();
+    }
+
     if (response.ok) {
-      return Buffer.from(await response.text());
+      return {
+        buffer: async () => Buffer.from(await response.text()),
+        etag: response.headers.get('ETag') ?? undefined,
+      };
     }
 
     const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`;

packages/backend-common/src/reading/GitlabUrlReader.test.ts

@@ -79,7 +79,7 @@ describe('GitlabUrlReader', () => {
   const worker = setupServer();
   msw.setupDefaultHandlers(worker);
 
-  describe('implementation', () => {
+  describe('read', () => {
     beforeEach(() => {
       worker.use(
         rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
@@ -180,6 +180,53 @@ describe('GitlabUrlReader', () => {
     });
   });
 
+  describe('readUrl', () => {
+    const [{ reader }] = GitlabUrlReader.factory({
+      config: new ConfigReader({}),
+      logger,
+      treeResponseFactory,
+    });
+
+    it('should throw NotModified on HTTP 304', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
+        ),
+        rest.get('*', (req, res, ctx) => {
+          expect(req.headers.get('If-None-Match')).toBe('999');
+          return res(ctx.status(304));
+        }),
+      );
+
+      await expect(
+        reader.readUrl!(
+          'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+          {
+            etag: '999',
+          },
+        ),
+      ).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should return etag in response', async () => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
+        ),
+        rest.get('*', (_req, res, ctx) => {
+          return res(ctx.status(200), ctx.set('ETag', '999'), ctx.body('foo'));
+        }),
+      );
+
+      const result = await reader.readUrl!(
+        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+      );
+      expect(result.etag).toBe('999');
+      const content = await result.buffer();
+      expect(content.toString()).toBe('foo');
+    });
+  });
+
   describe('readTree', () => {
     const archiveBuffer = fs.readFileSync(
       path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.tar.gz'),

packages/backend-common/src/reading/GitlabUrlReader.ts

@@ -34,6 +34,8 @@ import {
   SearchOptions,
   SearchResponse,
   UrlReader,
+  ReadUrlResponse,
+  ReadUrlOptions,
 } from './types';
 
 export class GitlabUrlReader implements UrlReader {
@@ -54,20 +56,37 @@ export class GitlabUrlReader implements UrlReader {
   ) {}
 
   async read(url: string): Promise<Buffer> {
+    const response = await this.readUrl(url);
+    return response.buffer();
+  }
+
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
     const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config);
 
     let response: Response;
     try {
-      response = await fetch(
-        builtUrl,
-        getGitLabRequestOptions(this.integration.config),
-      );
+      response = await fetch(builtUrl, {
+        headers: {
+          ...getGitLabRequestOptions(this.integration.config).headers,
+          ...(options?.etag && { 'If-None-Match': options.etag }),
+        },
+      });
     } catch (e) {
       throw new Error(`Unable to read ${url}, ${e}`);
     }
 
+    if (response.status === 304) {
+      throw new NotModifiedError();
+    }
+
     if (response.ok) {
-      return Buffer.from(await response.text());
+      return {
+        buffer: async () => Buffer.from(await response.text()),
+        etag: response.headers.get('ETag') ?? undefined,
+      };
     }
 
     const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`;

packages/backend-common/src/reading/GoogleGcsUrlReader.ts

@@ -18,6 +18,8 @@ import { Storage } from '@google-cloud/storage';
 import {
   ReaderFactory,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchResponse,
   UrlReader,
 } from './types';
@@ -90,6 +92,15 @@ export class GoogleGcsUrlReader implements UrlReader {
     }
   }
 
+  async readUrl(
+    url: string,
+    _options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    // TODO etag is not implemented yet.
+    const buffer = await this.read(url);
+    return { buffer: async () => buffer };
+  }
+
   async readTree(): Promise<ReadTreeResponse> {
     throw new Error('GcsUrlReader does not implement readTree');
   }

packages/backend-common/src/reading/UrlReaderPredicateMux.ts

@@ -15,27 +15,35 @@
  */
 
 import { NotAllowedError } from '@backstage/errors';
+import { Logger } from 'winston';
 import {
   ReadTreeOptions,
   ReadTreeResponse,
+  ReadUrlOptions,
+  ReadUrlResponse,
   SearchOptions,
   SearchResponse,
   UrlReader,
   UrlReaderPredicateTuple,
 } from './types';
 
+const MIN_WARNING_INTERVAL_MS = 1000 * 60 * 15;
+
 /**
  * A UrlReader implementation that selects from a set of UrlReaders
  * based on a predicate tied to each reader.
  */
 export class UrlReaderPredicateMux implements UrlReader {
   private readonly readers: UrlReaderPredicateTuple[] = [];
+  private readonly readerWarnings: Map<UrlReader, number> = new Map();
+
+  constructor(private readonly logger: Logger) {}
 
   register(tuple: UrlReaderPredicateTuple): void {
     this.readers.push(tuple);
   }
 
-  read(url: string): Promise<Buffer> {
+  async read(url: string): Promise<Buffer> {
     const parsed = new URL(url);
 
     for (const { predicate, reader } of this.readers) {
@@ -47,6 +55,37 @@ export class UrlReaderPredicateMux implements UrlReader {
     throw new NotAllowedError(`Reading from '${url}' is not allowed`);
   }
 
+  async readUrl(
+    url: string,
+    options?: ReadUrlOptions,
+  ): Promise<ReadUrlResponse> {
+    const parsed = new URL(url);
+
+    for (const { predicate, reader } of this.readers) {
+      if (predicate(parsed)) {
+        if (reader.readUrl) {
+          return reader.readUrl(url, options);
+        }
+        const now = Date.now();
+        const lastWarned = this.readerWarnings.get(reader) ?? 0;
+        if (now > lastWarned + MIN_WARNING_INTERVAL_MS) {
+          this.readerWarnings.set(reader, now);
+          this.logger.warn(
+            `No implementation of readUrl found for ${reader}, this method will be required in the ` +
+              `future and will replace the 'read' method. See the changelog for more details here: ` +
+              'https://github.com/backstage/backstage/blob/master/packages/backend-common/CHANGELOG.md#085',
+          );
+        }
+        const buffer = await reader.read(url);
+        return {
+          buffer: async () => buffer,
+        };
+      }
+    }
+
+    throw new NotAllowedError(`Reading from '${url}' is not allowed`);
+  }
+
   async readTree(
     url: string,
     options?: ReadTreeOptions,

packages/backend-common/src/reading/UrlReaders.ts

@@ -43,7 +43,7 @@ export class UrlReaders {
    * Creates a UrlReader without any known types.
    */
   static create({ logger, config, factories }: CreateOptions): UrlReader {
-    const mux = new UrlReaderPredicateMux();
+    const mux = new UrlReaderPredicateMux(logger);
     const treeResponseFactory = DefaultReadTreeResponseFactory.create({
       config,
     });

packages/backend-common/src/reading/types.ts

@@ -23,7 +23,17 @@ import { Config } from '@backstage/config';
  */
 export type UrlReader = {
   read(url: string): Promise<Buffer>;
+
+  /**
+   * A replacement for the read method that supports options and complex responses.
+   *
+   * Use this whenever it is available, as the read method will be deprecated and
+   * eventually removed in the future.
+   */
+  readUrl?(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
+
   readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
+
   search(url: string, options?: SearchOptions): Promise<SearchResponse>;
 };
 
@@ -42,6 +52,40 @@ export type ReaderFactory = (options: {
   treeResponseFactory: ReadTreeResponseFactory;
 }) => UrlReaderPredicateTuple[];
 
+/**
+ * An options object for readUrl operations.
+ */
+export type ReadUrlOptions = {
+  /**
+   * An etag can be provided to check whether readUrl's response has changed from a previous execution.
+   *
+   * In the readUrl() response, an etag is returned along with the data. The etag is a unique identifer
+   * of the data, usually the commit SHA or etag from the target.
+   *
+   * When an etag is given in ReadUrlOptions, readUrl will first compare the etag against the etag
+   * on the target. If they match, readUrl will throw a NotModifiedError indicating that the readUrl
+   * response will not differ from the previous response which included this particular etag. If they
+   * do not match, readUrl will return the rest of ReadUrlResponse along with a new etag.
+   */
+  etag?: string;
+};
+
+/**
+ * A response object for readUrl operations.
+ */
+export type ReadUrlResponse = {
+  /**
+   * Returns the data that was read from the remote URL.
+   */
+  buffer(): Promise<Buffer>;
+
+  /**
+   * Etag returned by content provider.
+   * Can be used to compare and cache responses when doing subsequent calls.
+   */
+  etag?: string;
+};
+
 /**
  * An options object for readTree operations.
  */
@@ -66,14 +110,17 @@ export type ReadTreeOptions = {
    * In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer
    * of the tree blob, usually the commit SHA or etag from the target.
    *
-   * When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
+   * When an etag is given in ReadTreeOptions, readTree will first compare the etag against the etag
    * on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree
-   * response will not differ from the previous response which included this particular etag. If they mismatch,
-   * readTree will return the rest of ReadTreeResponse along with a new etag.
+   * response will not differ from the previous response which included this particular etag. If they
+   * do not match, readTree will return the rest of ReadTreeResponse along with a new etag.
    */
   etag?: string;
 };
 
+/**
+ * A response object for readTree operations.
+ */
 export type ReadTreeResponse = {
   /**
    * files() returns an array of all the files inside the tree and corresponding functions to read their content.
@@ -87,7 +134,8 @@ export type ReadTreeResponse = {
   dir(options?: ReadTreeResponseDirOptions): Promise<string>;
 
   /**
-   * A unique identifer of the tree blob, usually the commit SHA or etag from the target.
+   * Etag returned by content provider.
+   * Can be used to compare and cache responses when doing subsequent calls.
    */
   etag: string;
 };

plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts

@@ -94,12 +94,21 @@ export class PlaceholderProcessor implements CatalogProcessor {
         return [data, false];
       }
 
+      const read = async (url: string): Promise<Buffer> => {
+        if (this.options.reader.readUrl) {
+          const response = await this.options.reader.readUrl(url);
+          const buffer = await response.buffer();
+          return buffer;
+        }
+        return this.options.reader.read(url);
+      };
+
       return [
         await resolver({
           key: resolverKey,
           value: resolverValue,
           baseUrl: location.target,
-          read: this.options.reader.read.bind(this.options.reader),
+          read,
         }),
         true,
       ];

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -101,7 +101,12 @@ export class UrlReaderProcessor implements CatalogProcessor {
       return Promise.all(output);
     }
 
-    // Otherwise do a plain read
+    // Otherwise do a plain read, prioritizing readUrl if available
+    if (this.options.reader.readUrl) {
+      const data = await this.options.reader.readUrl(location);
+      return [{ url: location, data: await data.buffer() }];
+    }
+
     const data = await this.options.reader.read(location);
     return [{ url: location, data }];
   }

plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts

@@ -28,6 +28,12 @@ export async function readCodeOwners(
 ): Promise<string | undefined> {
   const readOwnerLocation = async (path: string): Promise<string> => {
     const url = `${sourceUrl}${path}`;
+
+    if (reader.readUrl) {
+      const data = await reader.readUrl(url);
+      const buffer = await data.buffer();
+      return buffer.toString();
+    }
     const data = await reader.read(url);
     return data.toString();
   };