diff --git a/.changeset/honest-cars-glow.md b/.changeset/honest-cars-glow.md new file mode 100644 index 0000000000..3fdc6e335a --- /dev/null +++ b/.changeset/honest-cars-glow.md @@ -0,0 +1,54 @@ +--- +'@backstage/backend-common': patch +--- + +Added a `readUrl` method to the `UrlReader` interface that allows for complex response objects and is intended to replace the `read` method. This new method is currently optional to implement which allows for a soft migration to `readUrl` instead of `read` in the future. + +The main use case for `readUrl` returning an object instead of solely a read buffer is to allow for additional metadata such as ETag, which is a requirement for more efficient catalog processing. + +The `GithubUrlReader` and `GitlabUrlReader` readers fully implement `readUrl`. The other existing readers implement the new method but do not propagate or return ETags. + +While the `readUrl` method is not yet required, it will be in the future, and we already log deprecation warnings when custom `UrlReader` implementations that do not implement `readUrl` are used. We therefore recommend that any existing custom implementations are migrated to implement `readUrl`. + +The old `read` and the new `readUrl` methods can easily be implemented using one another, but we recommend moving the chunk of the implementation to the new `readUrl` method as `read` is being removed, for example this: + +```ts +class CustomUrlReader implements UrlReader { + read(url: string): Promise { + const res = await fetch(url); + + if (!res.ok) { + // error handling ... + } + + return Buffer.from(await res.text()); + } +} +``` + +Can be migrated to something like this: + +```ts +class CustomUrlReader implements UrlReader { + read(url: string): Promise { + const res = await this.readUrl(url); + return res.buffer(); + } + + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + const res = await fetch(url); + + if (!res.ok) { + // error handling ... + } + + const buffer = Buffer.from(await res.text()); + return { buffer: async () => buffer }; + } +} +``` + +While there is no usage of the ETag capability yet in the main Backstage packages, you can already add it to your custom implementations. To do so, refer to the documentation of the `readUrl` method and surrounding types, and the existing implementation in `packages/backend-common/src/reading/GithubUrlReader.ts`. diff --git a/.changeset/rotten-walls-cross.md b/.changeset/rotten-walls-cross.md new file mode 100644 index 0000000000..498c10b203 --- /dev/null +++ b/.changeset/rotten-walls-cross.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Make use of the new `readUrl` method on `UrlReader` from `@backstage/backend-common`. diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 494a37ed33..a21aaef4a6 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -42,6 +42,8 @@ export class AzureUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, _options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -59,6 +61,8 @@ export class BitbucketUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, _options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -232,6 +236,8 @@ export class GithubUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -249,6 +255,8 @@ export class GitlabUrlReader implements UrlReader { // (undocumented) readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) search(url: string, options?: SearchOptions): Promise; // (undocumented) toString(): string; @@ -278,7 +286,7 @@ export type PluginEndpointDiscovery = { getExternalBaseUrl(pluginId: string): Promise; }; -// @public (undocumented) +// @public export type ReadTreeResponse = { files(): Promise; archive(): Promise; @@ -367,6 +375,7 @@ export interface StatusCheckHandlerOptions { // @public export type UrlReader = { read(url: string): Promise; + readUrl?(url: string, options?: ReadUrlOptions): Promise; readTree(url: string, options?: ReadTreeOptions): Promise; search(url: string, options?: SearchOptions): Promise; }; diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index fe541538ea..bc22854d4e 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -36,6 +36,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlOptions, + ReadUrlResponse, } from './types'; export class AzureUrlReader implements UrlReader { @@ -78,6 +80,15 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 4d8d865084..b032322627 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -74,7 +74,24 @@ describe('BitbucketUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); - describe('implementation', () => { + describe('readUrl', () => { + worker.use( + rest.get( + 'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml', + (_, res, ctx) => res(ctx.status(200), ctx.body('foo')), + ), + ); + + it('should be able to readUrl', async () => { + const result = await bitbucketProcessor.readUrl( + 'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml', + ); + const buffer = await result.buffer(); + expect(buffer.toString()).toBe('foo'); + }); + }); + + describe('read', () => { it('rejects unknown targets', async () => { await expect( bitbucketProcessor.read('https://not.bitbucket.com/apa'), diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index c4dc6e135f..2849bc3cd9 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -36,6 +36,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlResponse, + ReadUrlOptions, } from './types'; /** @@ -99,6 +101,15 @@ export class BitbucketUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/FetchUrlReader.ts b/packages/backend-common/src/reading/FetchUrlReader.ts index 4c03ea904d..30468158aa 100644 --- a/packages/backend-common/src/reading/FetchUrlReader.ts +++ b/packages/backend-common/src/reading/FetchUrlReader.ts @@ -19,6 +19,8 @@ import { NotFoundError } from '@backstage/errors'; import { ReaderFactory, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchResponse, UrlReader, } from './types'; @@ -73,6 +75,15 @@ export class FetchUrlReader implements UrlReader { throw new Error(message); } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree(): Promise { throw new Error('FetchUrlReader does not implement readTree'); } diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 0b3288e59c..c8002cd3ce 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -141,6 +141,113 @@ describe('GithubUrlReader', () => { }); }); + /* + * readUrl + */ + describe('readUrl', () => { + it('should use the headers from the credentials provider to the fetch request when doing readUrl', async () => { + expect.assertions(2); + + const mockHeaders = { + Authorization: 'bearer blah', + otherheader: 'something', + }; + + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: mockHeaders, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (req, res, ctx) => { + expect(req.headers.get('authorization')).toBe( + mockHeaders.Authorization, + ); + expect(req.headers.get('otherheader')).toBe( + mockHeaders.otherheader, + ); + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body('foo'), + ); + }, + ), + ); + + await gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + ); + }); + + it('should throw NotModified if GitHub responds with 304', async () => { + expect.assertions(4); + + const mockHeaders = { + Authorization: 'bearer blah', + otherheader: 'something', + }; + + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: mockHeaders, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (req, res, ctx) => { + expect(req.headers.get('authorization')).toBe( + mockHeaders.Authorization, + ); + expect(req.headers.get('otherheader')).toBe( + mockHeaders.otherheader, + ); + expect(req.headers.get('if-none-match')).toBe('foo'); + return res( + ctx.status(304), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body('foo'), + ); + }, + ), + ); + + await expect( + gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + { etag: 'foo' }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return etag from the response', async () => { + (mockCredentialsProvider.getCredentials as jest.Mock).mockResolvedValue({ + headers: { + Authorization: 'bearer blah', + }, + }); + + worker.use( + rest.get( + 'https://ghe.github.com/api/v3/repos/backstage/mock/tree/contents/', + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Etag', 'foo'), + ctx.body('bar'), + ); + }, + ), + ); + + const response = await gheProcessor.readUrl( + 'https://github.com/backstage/mock/tree/blob/main', + ); + expect(response.etag).toBe('foo'); + }); + }); + /* * readTree */ diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index 70bc9601df..bccddc6838 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -35,6 +35,8 @@ import { SearchResponse, SearchResponseFile, UrlReader, + ReadUrlOptions, + ReadUrlResponse, } from './types'; export type GhRepoResponse = RestEndpointMethodTypes['repos']['get']['response']['data']; @@ -77,6 +79,14 @@ export class GithubUrlReader implements UrlReader { } async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { const ghUrl = getGitHubFileFetchUrl(url, this.integration.config); const { headers } = await this.deps.credentialsProvider.getCredentials({ url, @@ -86,6 +96,7 @@ export class GithubUrlReader implements UrlReader { response = await fetch(ghUrl.toString(), { headers: { ...headers, + ...(options?.etag && { 'If-None-Match': options.etag }), Accept: 'application/vnd.github.v3.raw', }, }); @@ -93,8 +104,15 @@ export class GithubUrlReader implements UrlReader { throw new Error(`Unable to read ${url}, ${e}`); } + if (response.status === 304) { + throw new NotModifiedError(); + } + if (response.ok) { - return Buffer.from(await response.text()); + return { + buffer: async () => Buffer.from(await response.text()), + etag: response.headers.get('ETag') ?? undefined, + }; } const message = `${url} could not be read as ${ghUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 9c2e428239..945444d8ed 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -79,7 +79,7 @@ describe('GitlabUrlReader', () => { const worker = setupServer(); msw.setupDefaultHandlers(worker); - describe('implementation', () => { + describe('read', () => { beforeEach(() => { worker.use( rest.get('*/api/v4/projects/:name', (_, res, ctx) => @@ -180,6 +180,53 @@ describe('GitlabUrlReader', () => { }); }); + describe('readUrl', () => { + const [{ reader }] = GitlabUrlReader.factory({ + config: new ConfigReader({}), + logger, + treeResponseFactory, + }); + + it('should throw NotModified on HTTP 304', async () => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (req, res, ctx) => { + expect(req.headers.get('If-None-Match')).toBe('999'); + return res(ctx.status(304)); + }), + ); + + await expect( + reader.readUrl!( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + { + etag: '999', + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return etag in response', async () => { + worker.use( + rest.get('*/api/v4/projects/:name', (_, res, ctx) => + res(ctx.status(200), ctx.json({ id: 12345 })), + ), + rest.get('*', (_req, res, ctx) => { + return res(ctx.status(200), ctx.set('ETag', '999'), ctx.body('foo')); + }), + ); + + const result = await reader.readUrl!( + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + ); + expect(result.etag).toBe('999'); + const content = await result.buffer(); + expect(content.toString()).toBe('foo'); + }); + }); + describe('readTree', () => { const archiveBuffer = fs.readFileSync( path.resolve('src', 'reading', '__fixtures__', 'gitlab-archive.tar.gz'), diff --git a/packages/backend-common/src/reading/GitlabUrlReader.ts b/packages/backend-common/src/reading/GitlabUrlReader.ts index aa215aee47..6c272471e2 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.ts @@ -34,6 +34,8 @@ import { SearchOptions, SearchResponse, UrlReader, + ReadUrlResponse, + ReadUrlOptions, } from './types'; export class GitlabUrlReader implements UrlReader { @@ -54,20 +56,37 @@ export class GitlabUrlReader implements UrlReader { ) {} async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { const builtUrl = await getGitLabFileFetchUrl(url, this.integration.config); let response: Response; try { - response = await fetch( - builtUrl, - getGitLabRequestOptions(this.integration.config), - ); + response = await fetch(builtUrl, { + headers: { + ...getGitLabRequestOptions(this.integration.config).headers, + ...(options?.etag && { 'If-None-Match': options.etag }), + }, + }); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } + if (response.status === 304) { + throw new NotModifiedError(); + } + if (response.ok) { - return Buffer.from(await response.text()); + return { + buffer: async () => Buffer.from(await response.text()), + etag: response.headers.get('ETag') ?? undefined, + }; } const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; diff --git a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts index e06612cd15..9f3fbad302 100644 --- a/packages/backend-common/src/reading/GoogleGcsUrlReader.ts +++ b/packages/backend-common/src/reading/GoogleGcsUrlReader.ts @@ -18,6 +18,8 @@ import { Storage } from '@google-cloud/storage'; import { ReaderFactory, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchResponse, UrlReader, } from './types'; @@ -90,6 +92,15 @@ export class GoogleGcsUrlReader implements UrlReader { } } + async readUrl( + url: string, + _options?: ReadUrlOptions, + ): Promise { + // TODO etag is not implemented yet. + const buffer = await this.read(url); + return { buffer: async () => buffer }; + } + async readTree(): Promise { throw new Error('GcsUrlReader does not implement readTree'); } diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index c9afecf907..4a166cb517 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -15,27 +15,35 @@ */ import { NotAllowedError } from '@backstage/errors'; +import { Logger } from 'winston'; import { ReadTreeOptions, ReadTreeResponse, + ReadUrlOptions, + ReadUrlResponse, SearchOptions, SearchResponse, UrlReader, UrlReaderPredicateTuple, } from './types'; +const MIN_WARNING_INTERVAL_MS = 1000 * 60 * 15; + /** * A UrlReader implementation that selects from a set of UrlReaders * based on a predicate tied to each reader. */ export class UrlReaderPredicateMux implements UrlReader { private readonly readers: UrlReaderPredicateTuple[] = []; + private readonly readerWarnings: Map = new Map(); + + constructor(private readonly logger: Logger) {} register(tuple: UrlReaderPredicateTuple): void { this.readers.push(tuple); } - read(url: string): Promise { + async read(url: string): Promise { const parsed = new URL(url); for (const { predicate, reader } of this.readers) { @@ -47,6 +55,37 @@ export class UrlReaderPredicateMux implements UrlReader { throw new NotAllowedError(`Reading from '${url}' is not allowed`); } + async readUrl( + url: string, + options?: ReadUrlOptions, + ): Promise { + const parsed = new URL(url); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + if (reader.readUrl) { + return reader.readUrl(url, options); + } + const now = Date.now(); + const lastWarned = this.readerWarnings.get(reader) ?? 0; + if (now > lastWarned + MIN_WARNING_INTERVAL_MS) { + this.readerWarnings.set(reader, now); + this.logger.warn( + `No implementation of readUrl found for ${reader}, this method will be required in the ` + + `future and will replace the 'read' method. See the changelog for more details here: ` + + 'https://github.com/backstage/backstage/blob/master/packages/backend-common/CHANGELOG.md#085', + ); + } + const buffer = await reader.read(url); + return { + buffer: async () => buffer, + }; + } + } + + throw new NotAllowedError(`Reading from '${url}' is not allowed`); + } + async readTree( url: string, options?: ReadTreeOptions, diff --git a/packages/backend-common/src/reading/UrlReaders.ts b/packages/backend-common/src/reading/UrlReaders.ts index bc77877384..2b3a2f166c 100644 --- a/packages/backend-common/src/reading/UrlReaders.ts +++ b/packages/backend-common/src/reading/UrlReaders.ts @@ -43,7 +43,7 @@ export class UrlReaders { * Creates a UrlReader without any known types. */ static create({ logger, config, factories }: CreateOptions): UrlReader { - const mux = new UrlReaderPredicateMux(); + const mux = new UrlReaderPredicateMux(logger); const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config, }); diff --git a/packages/backend-common/src/reading/types.ts b/packages/backend-common/src/reading/types.ts index f7fad4dd7a..ecaa582612 100644 --- a/packages/backend-common/src/reading/types.ts +++ b/packages/backend-common/src/reading/types.ts @@ -23,7 +23,17 @@ import { Config } from '@backstage/config'; */ export type UrlReader = { read(url: string): Promise; + + /** + * A replacement for the read method that supports options and complex responses. + * + * Use this whenever it is available, as the read method will be deprecated and + * eventually removed in the future. + */ + readUrl?(url: string, options?: ReadUrlOptions): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; + search(url: string, options?: SearchOptions): Promise; }; @@ -42,6 +52,40 @@ export type ReaderFactory = (options: { treeResponseFactory: ReadTreeResponseFactory; }) => UrlReaderPredicateTuple[]; +/** + * An options object for readUrl operations. + */ +export type ReadUrlOptions = { + /** + * An etag can be provided to check whether readUrl's response has changed from a previous execution. + * + * In the readUrl() response, an etag is returned along with the data. The etag is a unique identifer + * of the data, usually the commit SHA or etag from the target. + * + * When an etag is given in ReadUrlOptions, readUrl will first compare the etag against the etag + * on the target. If they match, readUrl will throw a NotModifiedError indicating that the readUrl + * response will not differ from the previous response which included this particular etag. If they + * do not match, readUrl will return the rest of ReadUrlResponse along with a new etag. + */ + etag?: string; +}; + +/** + * A response object for readUrl operations. + */ +export type ReadUrlResponse = { + /** + * Returns the data that was read from the remote URL. + */ + buffer(): Promise; + + /** + * Etag returned by content provider. + * Can be used to compare and cache responses when doing subsequent calls. + */ + etag?: string; +}; + /** * An options object for readTree operations. */ @@ -66,14 +110,17 @@ export type ReadTreeOptions = { * In the readTree() response, an etag is returned along with the tree blob. The etag is a unique identifer * of the tree blob, usually the commit SHA or etag from the target. * - * When a etag is given in ReadTreeOptions, readTree will first compare the etag against the etag + * When an etag is given in ReadTreeOptions, readTree will first compare the etag against the etag * on the target branch. If they match, readTree will throw a NotModifiedError indicating that the readTree - * response will not differ from the previous response which included this particular etag. If they mismatch, - * readTree will return the rest of ReadTreeResponse along with a new etag. + * response will not differ from the previous response which included this particular etag. If they + * do not match, readTree will return the rest of ReadTreeResponse along with a new etag. */ etag?: string; }; +/** + * A response object for readTree operations. + */ export type ReadTreeResponse = { /** * files() returns an array of all the files inside the tree and corresponding functions to read their content. @@ -87,7 +134,8 @@ export type ReadTreeResponse = { dir(options?: ReadTreeResponseDirOptions): Promise; /** - * A unique identifer of the tree blob, usually the commit SHA or etag from the target. + * Etag returned by content provider. + * Can be used to compare and cache responses when doing subsequent calls. */ etag: string; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts index 8c06ef419e..1b015d7d93 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -94,12 +94,21 @@ export class PlaceholderProcessor implements CatalogProcessor { return [data, false]; } + const read = async (url: string): Promise => { + if (this.options.reader.readUrl) { + const response = await this.options.reader.readUrl(url); + const buffer = await response.buffer(); + return buffer; + } + return this.options.reader.read(url); + }; + return [ await resolver({ key: resolverKey, value: resolverValue, baseUrl: location.target, - read: this.options.reader.read.bind(this.options.reader), + read, }), true, ]; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index aff126414f..da7bee12d8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -101,7 +101,12 @@ export class UrlReaderProcessor implements CatalogProcessor { return Promise.all(output); } - // Otherwise do a plain read + // Otherwise do a plain read, prioritizing readUrl if available + if (this.options.reader.readUrl) { + const data = await this.options.reader.readUrl(location); + return [{ url: location, data: await data.buffer() }]; + } + const data = await this.options.reader.read(location); return [{ url: location, data }]; } diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts b/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts index ae4255555a..c6b89efcc9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts +++ b/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts @@ -28,6 +28,12 @@ export async function readCodeOwners( ): Promise { const readOwnerLocation = async (path: string): Promise => { const url = `${sourceUrl}${path}`; + + if (reader.readUrl) { + const data = await reader.readUrl(url); + const buffer = await data.buffer(); + return buffer.toString(); + } const data = await reader.read(url); return data.toString(); };