Merge pull request #28379 from leboncoin/feat-urlreader-issearchurl

feat(catalog): always use search for UrlReaderProcessor

.changeset/heavy-bobcats-search.md

@@ -0,0 +1,8 @@
+---
+'@backstage/create-app': patch
+---
+
+Enable `catalog.useUrlReadersSearch` configuration by default in newly created Backstage installations.
+
+This parameter makes `UrlReaderProcessor` always use the search method.
+New adopters are encouraged to enable it as this behavior will be the default in a future release.

.changeset/lucky-seas-live.md

@@ -0,0 +1,11 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+The `UrlReaderProccessor` accepts a new config flag `catalog.useUrlReadersSearch` to always call the `search` method of `UrlReaders`.
+
+This flag currently defaults to `false`, but adopters are encouraged to enable it as this behavior will be the default in a future release.
+
+Previous behavior was to call the `search` method only if the parsed Git URL's filename contained a wildcard and use `readUrl` otherwise. `UrlReaderService` must implement this logic in the `search` method instead.
+
+This allows each `UrlReaderService` implementation to check whether it's a search URL (that contains a wildcard pattern) or not using logic that is specific to each provider.

.changeset/tiny-spoons-repeat.md

@@ -0,0 +1,6 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+In the different `UrlReadersService`, the `search` method have been updated to use the `readUrl` if the given URL doesn't contain a pattern.
+For `UrlReaders` that didn't implement the `search` method, `readUrl` is now called internally and throws if the given URL doesn't contain a pattern.

app-config.yaml

@@ -244,6 +244,7 @@ integrations:
       secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
 
 catalog:
+  useUrlReadersSearch: true
   import:
     entityFilename: catalog-info.yaml
     pullRequestBranchName: backstage-integration

packages/backend-defaults/report-urlReader.api.md

@@ -57,7 +57,10 @@ export class AwsS3UrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -230,7 +233,10 @@ export class FetchUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -265,7 +271,10 @@ export class GerritUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -293,7 +302,10 @@ export class GiteaUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }
@@ -384,7 +396,10 @@ export class HarnessUrlReader implements UrlReaderService {
     options?: UrlReaderServiceReadUrlOptions,
   ): Promise<UrlReaderServiceReadUrlResponse>;
   // (undocumented)
-  search(): Promise<UrlReaderServiceSearchResponse>;
+  search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse>;
   // (undocumented)
   toString(): string;
 }

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.test.ts

@@ -608,4 +608,56 @@ describe('AwsCodeCommitUrlReader', () => {
       expect(bodySubfolderFile.toString().trim()).toBe('site_name: Test2');
     });
   });
+
+  describe('search', () => {
+    const [{ reader }] = createReader({
+      integrations: {
+        awsCodeCommit: [
+          {
+            host: AMAZON_AWS_CODECOMMIT_HOST,
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+            region: 'fakeregion',
+          },
+        ],
+      },
+    });
+
+    beforeEach(() => {
+      codeCommitClient.reset();
+
+      codeCommitClient.on(GetFileCommand).resolves({
+        fileContent: fs.readFileSync(
+          path.resolve(
+            __dirname,
+            '__fixtures__/awsCodeCommit/awsCodeCommit-mock-object.yaml',
+          ),
+        ),
+        commitId: `123abc`,
+      });
+    });
+
+    it('should return a file when given an exact valid url', async () => {
+      const data = await reader.search(
+        'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
+      );
+
+      expect(data.etag).toBe('123abc');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-info.yaml',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'site_name: Test\n',
+      );
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        reader.search(
+          'https://eu-west-1.console.aws.amazon.com/codesuite/codecommit/repositories/my-repo/browse/--/catalog-*.yaml',
+        ),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsCodeCommitUrlReader.ts

@@ -21,6 +21,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import {
@@ -31,7 +32,11 @@ import {
   AwsCodeCommitIntegration,
   ScmIntegrations,
 } from '@backstage/integration';
-import { ForwardedError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  ForwardedError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import {
   CodeCommitClient,
@@ -257,7 +262,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
       }
 
       return ReadUrlResponseFactory.fromReadable(
-        Readable.from([response?.fileContent] || []),
+        Readable.from([response?.fileContent]),
         {
           etag: response.commitId,
         },
@@ -357,7 +362,7 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
           commitSpecifier: commitSpecifier,
         });
         const response = await codeCommitClient.send(getFileCommand);
-        const objectData = await Readable.from([response?.fileContent] || []);
+        const objectData = await Readable.from([response?.fileContent]);
 
         responses.push({
           data: objectData,
@@ -380,8 +385,39 @@ export class AwsCodeCommitUrlReader implements UrlReaderService {
     }
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('AwsCodeCommitReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { path } = parseUrl(url, true);
+
+    if (path.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.test.ts

@@ -527,4 +527,58 @@ describe('AwsS3UrlReader', () => {
       expect(body.toString().trim()).toBe('site_name: Test');
     });
   });
+
+  describe('search', () => {
+    const [{ reader }] = createReader({
+      integrations: {
+        awsS3: [
+          {
+            host: 'amazonaws.com',
+            accessKeyId: 'fake-access-key',
+            secretAccessKey: 'fake-secret-key',
+          },
+        ],
+      },
+    });
+
+    beforeEach(() => {
+      s3Client.reset();
+
+      s3Client.on(GetObjectCommand).resolves({
+        Body: sdkStreamMixin(
+          fs.createReadStream(
+            path.resolve(
+              __dirname,
+              '__fixtures__/awsS3/awsS3-mock-object.yaml',
+            ),
+          ),
+        ),
+        ETag: '123abc',
+        LastModified: new Date('2020-01-01T00:00:00Z'),
+      });
+    });
+
+    it('should return a file when given an exact valid url', async () => {
+      const data = await reader.search(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+
+      expect(data.etag).toBe('123abc');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-object.yaml',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'site_name: Test\n',
+      );
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        reader.search(
+          'https://test-bucket.s3.us-east-2.amazonaws.com/awsS3-mock-*.yaml',
+        ),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/AwsS3UrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
@@ -32,7 +33,11 @@ import {
   ScmIntegrations,
   AwsS3IntegrationConfig,
 } from '@backstage/integration';
-import { ForwardedError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  ForwardedError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { fromTemporaryCredentials } from '@aws-sdk/credential-providers';
 import { AwsCredentialIdentityProvider } from '@aws-sdk/types';
 import {
@@ -357,8 +362,39 @@ export class AwsS3UrlReader implements UrlReaderService {
     }
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('AwsS3Reader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { path } = parseUrl(url, this.integration.config);
+
+    if (path.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.test.ts

@@ -351,6 +351,17 @@ describe('AzureUrlReader', () => {
       );
     });
 
+    it('works for exact urls', async () => {
+      const result = await processor.search(
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+      );
+      expect(result.etag).toBe('');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster',
+      );
+    });
+
     it('works for the naive case', async () => {
       const result = await processor.search(
         'https://dev.azure.com/org-name/project-name/_git/repo-name?path=%2F**%2Findex.*&version=GBmaster',

packages/backend-defaults/src/entrypoints/urlReader/lib/AzureUrlReader.ts

@@ -32,8 +32,13 @@ import {
   ScmIntegrations,
   AzureIntegration,
 } from '@backstage/integration';
+import parseGitUrl from 'git-url-parse';
 import { Minimatch } from 'minimatch';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 
@@ -181,6 +186,35 @@ export class AzureUrlReader implements UrlReaderService {
     url: string,
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
+    const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const treeUrl = new URL(url);
 
     const path = treeUrl.searchParams.get('path');

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.test.ts

@@ -451,5 +451,31 @@ describe('BitbucketCloudUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs', async () => {
+      worker.use(
+        rest.get(
+          'https://api.bitbucket.org/2.0/repositories/backstage-verification/test-template/src/master/template.yaml',
+          (req, res, ctx) => {
+            expect(req.headers.get('If-None-Match')).toBeNull();
+            return res(
+              ctx.status(200),
+              ctx.body('foo'),
+              ctx.set('ETag', 'etag-value'),
+            );
+          },
+        ),
+      );
+
+      const result = await reader.search(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      expect(result.etag).toBe('etag-value');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.org/backstage-verification/test-template/src/master/template.yaml',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('foo');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketCloudUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketCloudIntegration,
   getBitbucketCloudDefaultBranch,
@@ -163,6 +167,34 @@ export class BitbucketCloudUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.test.ts

@@ -30,6 +30,7 @@ import path from 'path';
 import { NotModifiedError } from '@backstage/errors';
 import { BitbucketServerUrlReader } from './BitbucketServerUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 createMockDirectory({ mockOsTmpDir: true });
 
@@ -277,5 +278,23 @@ describe('BitbucketServerUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs by using readUrl directly', async () => {
+      reader.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+
+      const result = await reader.search(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml',
+      );
+      expect(reader.readUrl).toHaveBeenCalledTimes(1);
+      expect(result.etag).toBe('etag');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/template.yml',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketServerUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketServerIntegration,
   getBitbucketServerDownloadUrl,
@@ -168,6 +172,34 @@ export class BitbucketServerUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.test.ts

@@ -32,6 +32,7 @@ import { NotModifiedError } from '@backstage/errors';
 import { BitbucketUrlReader } from './BitbucketUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
 import getRawBody from 'raw-body';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const logger = mockServices.logger.mock();
 
@@ -613,5 +614,22 @@ describe('BitbucketUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('should work for exact URLs', async () => {
+      hostedBitbucketProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+
+      const result = await hostedBitbucketProcessor.search(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
+      );
+      expect(result.etag).toBe('etag');
+      expect(result.files.length).toBe(1);
+      expect(result.files[0].url).toBe(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs/index.md?at=master',
+      );
+      expect((await result.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/BitbucketUrlReader.ts

@@ -23,7 +23,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   BitbucketIntegration,
   getBitbucketDefaultBranch,
@@ -174,6 +178,34 @@ export class BitbucketUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const matcher = new Minimatch(filepath);
 
     // TODO(freben): For now, read the entire repo and filter through that. In

packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.test.ts

@@ -266,4 +266,34 @@ describe('FetchUrlReader', () => {
       ).rejects.toThrow(Error);
     });
   });
+
+  describe('search', () => {
+    it('should return a file', async () => {
+      const data = await fetchUrlReader.search(
+        `https://backstage.io/some-resource`,
+        { etag: 'etag' },
+      );
+      expect(data.etag).toBe('foo');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(`https://backstage.io/some-resource`);
+      expect((await data.files[0].content()).toString()).toEqual('content foo');
+    });
+
+    it('should return an empty list of file if not found', async () => {
+      const data = await fetchUrlReader.search(
+        `https://backstage.io/not-exists`,
+        { etag: 'etag' },
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        fetchUrlReader.search(`https://backstage.io/some-resource*`, {
+          etag: 'etag',
+        }),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/FetchUrlReader.ts

@@ -19,9 +19,14 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReaderFactory } from './types';
 import path from 'path';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
@@ -162,8 +167,39 @@ export class FetchUrlReader implements UrlReaderService {
     throw new Error('FetchUrlReader does not implement readTree');
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('FetchUrlReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { pathname } = new URL(url);
+
+    if (pathname.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.test.ts

@@ -420,4 +420,59 @@ describe.skip('GerritUrlReader', () => {
       expect(response.etag).toBe(sha);
     });
   });
+
+  describe('search', () => {
+    const responseBuffer = Buffer.from('Apache License');
+
+    it('should return a single file when given an exact URL', async () => {
+      worker.use(
+        rest.get(
+          'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
+          (_, res, ctx) => {
+            return res(
+              ctx.status(200),
+              ctx.body(responseBuffer.toString('base64')),
+            );
+          },
+        ),
+      );
+
+      const data = await gerritProcessor.search(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      worker.use(
+        rest.get(
+          'https://gerrit.com/projects/web%2Fproject/branches/master/files/LICENSE/content',
+          (_, res, ctx) => {
+            return res(ctx.status(404, 'File not found.'));
+          },
+        ),
+      );
+
+      const data = await gerritProcessor.search(
+        'https://gerrit.com/web/project/+/refs/heads/master/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        gerritProcessor.search(
+          'https://gerrit.com/web/project/+/refs/heads/master/*.yaml',
+        ),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GerritUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { Base64Decode } from 'base64-stream';
@@ -39,6 +40,7 @@ import {
   NotFoundError,
   NotModifiedError,
   ResponseError,
+  assertError,
 } from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 
@@ -144,8 +146,39 @@ export class GerritUrlReader implements UrlReaderService {
     return this.readTreeFromGitiles(url, urlRevision, options);
   }
 
-  async search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('GerritReader does not implement search');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { pathname } = new URL(url);
+
+    if (pathname.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.test.ts

@@ -330,4 +330,70 @@ describe('GiteaUrlReader', () => {
       ).rejects.toThrow(NotFoundError);
     });
   });
+
+  describe('search', () => {
+    const responseBuffer = Buffer.from('Apache License');
+    const giteaApiResponse = (content: any) => {
+      return JSON.stringify({
+        encoding: 'base64',
+        content: Buffer.from(content).toString('base64'),
+      });
+    };
+
+    it('should return a single file when given an exact URL', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
+          (req, res, ctx) => {
+            // Test utils prefers matching URL directly but it is part of Gitea's API
+            if (req.url.searchParams.get('ref') === 'branch2') {
+              return res(
+                ctx.status(200),
+                ctx.body(giteaApiResponse(responseBuffer.toString())),
+              );
+            }
+
+            return res(ctx.status(500));
+          },
+        ),
+      );
+
+      const data = await giteaProcessor.search(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      worker.use(
+        rest.get(
+          'https://gitea.com/api/v1/repos/owner/project/contents/LICENSE',
+          (_, res, ctx) => {
+            return res(ctx.status(404, 'File not found.'));
+          },
+        ),
+      );
+
+      const data = await giteaProcessor.search(
+        'https://gitea.com/owner/project/src/branch/branch2/LICENSE',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        giteaProcessor.search(
+          'https://gitea.com/owner/project/src/branch/branch2/*.yaml',
+        ),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GiteaUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import {
@@ -34,12 +35,14 @@ import {
 import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import {
+  assertError,
   AuthenticationError,
   NotFoundError,
   NotModifiedError,
 } from '@backstage/errors';
 import { Readable } from 'stream';
 import { parseLastModified } from './util';
+import parseGitUrl from 'git-url-parse';
 
 /**
  * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for the Gitea v1 api.
@@ -155,8 +158,39 @@ export class GiteaUrlReader implements UrlReaderService {
     });
   }
 
-  search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('GiteaUrlReader search not implemented.');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { filepath } = parseGitUrl(url);
+
+    if (filepath.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.test.ts

@@ -37,6 +37,7 @@ import {
   GithubUrlReader,
 } from './GithubUrlReader';
 import { DefaultReadTreeResponseFactory } from './tree';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const mockDir = createMockDirectory({ mockOsTmpDir: true });
 
@@ -887,9 +888,6 @@ describe('GithubUrlReader', () => {
       expect(r2.etag).toBe('etag123abc');
       expect(r2.files.length).toBe(2);
 
-      const r3 = await reader.search(`${baseUrl}/backstage/mock/tree/main/o`);
-      expect(r3.files.length).toBe(0);
-
       const r4 = await reader.search(
         `${baseUrl}/backstage/mock/tree/main/*docs*`,
       );
@@ -1017,6 +1015,23 @@ describe('GithubUrlReader', () => {
       await runTests(gheProcessor, 'https://ghe.github.com');
     });
 
+    it('uses readUrl when searching for an exact file', async () => {
+      githubProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await githubProcessor.search(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect(githubProcessor.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
+
     it('throws NotModifiedError when same etag', async () => {
       await expect(
         githubProcessor.search(

packages/backend-defaults/src/entrypoints/urlReader/lib/GithubUrlReader.ts

@@ -37,7 +37,11 @@ import fetch, { RequestInit, Response } from 'node-fetch';
 import parseGitUrl from 'git-url-parse';
 import { Minimatch } from 'minimatch';
 import { Readable } from 'stream';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import { parseLastModified } from './util';
@@ -179,6 +183,35 @@ export class GithubUrlReader implements UrlReaderService {
     url: string,
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
+    const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const repoDetails = await this.getRepoDetails(url);
     const commitSha = repoDetails.commitSha;
 
@@ -186,7 +219,6 @@ export class GithubUrlReader implements UrlReaderService {
       throw new NotModifiedError();
     }
 
-    const { filepath } = parseGitUrl(url);
     const { headers } = await this.getCredentials(url, options);
 
     const files = await this.doSearch(

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.test.ts

@@ -31,6 +31,7 @@ import {
   GitLabIntegration,
   readGitLabIntegrationConfig,
 } from '@backstage/integration';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const logger = mockServices.logger.mock();
 
@@ -663,6 +664,23 @@ describe('GitlabUrlReader', () => {
         ),
       ).rejects.toThrow(NotModifiedError);
     });
+
+    it('returns a single file for exact urls', async () => {
+      gitlabProcessor.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await gitlabProcessor.search(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect(gitlabProcessor.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://github.com/backstage/mock/tree/main/o',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
   });
 
   describe('getGitlabFetchUrl', () => {

packages/backend-defaults/src/entrypoints/urlReader/lib/GitlabUrlReader.ts

@@ -26,7 +26,11 @@ import {
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { NotFoundError, NotModifiedError } from '@backstage/errors';
+import {
+  assertError,
+  NotFoundError,
+  NotModifiedError,
+} from '@backstage/errors';
 import {
   GitLabIntegration,
   ScmIntegrations,
@@ -244,6 +248,34 @@ export class GitlabUrlReader implements UrlReaderService {
     options?: UrlReaderServiceSearchOptions,
   ): Promise<UrlReaderServiceSearchResponse> {
     const { filepath } = parseGitUrl(url);
+
+    // If it's a direct URL we use readUrl instead
+    if (!filepath?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     const staticPart = this.getStaticPart(filepath);
     const matcher = new Minimatch(filepath);
     const treeUrl = trimEnd(url.replace(filepath, staticPart), `/`);

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts

@@ -22,6 +22,7 @@ import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 import { UrlReaderPredicateTuple } from './types';
 import packageinfo from '../../../../package.json';
 import { mockServices } from '@backstage/backend-test-utils';
+import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
 
 const bucketGetFilesMock = jest.fn();
 class Bucket {
@@ -130,7 +131,7 @@ describe('GcsUrlReader', () => {
     });
 
     it('throws if search url does not end with *', async () => {
-      const glob = 'https://storage.cloud.google.com/bucket/no-asterisk';
+      const glob = 'https://storage.cloud.google.com/bucket/*no-asterisk';
       await expect(() => reader.search(glob)).rejects.toThrow(
         'GcsUrlReader only supports prefix-based searches',
       );
@@ -167,5 +168,22 @@ describe('GcsUrlReader', () => {
         'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
       );
     });
+
+    it('returns single file if there is no wildcard', async () => {
+      reader.readUrl = jest.fn().mockResolvedValue({
+        buffer: async () => Buffer.from('content'),
+        etag: 'etag',
+      } as UrlReaderServiceReadUrlResponse);
+      const data = await reader.search(
+        'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
+      );
+      expect(reader.readUrl).toHaveBeenCalledTimes(1);
+      expect(data.etag).toBe('etag');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
+      );
+      expect((await data.files[0].content()).toString()).toEqual('content');
+    });
   });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts

@@ -20,6 +20,7 @@ import {
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
+  UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
 import { ReaderFactory } from './types';
@@ -31,6 +32,7 @@ import {
 import { Readable } from 'stream';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import packageinfo from '../../../../package.json';
+import { assertError } from '@backstage/errors';
 
 const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
 
@@ -117,9 +119,39 @@ export class GoogleGcsUrlReader implements UrlReaderService {
     throw new Error('GcsUrlReader does not implement readTree');
   }
 
-  async search(url: string): Promise<UrlReaderServiceSearchResponse> {
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
     const { bucket, key: pattern } = parseURL(url);
 
+    // If it's a direct URL we use readUrl instead
+    if (!pattern?.match(/[*?]/)) {
+      try {
+        const data = await this.readUrl(url, options);
+
+        return {
+          files: [
+            {
+              url: url,
+              content: data.buffer,
+              lastModifiedAt: data.lastModifiedAt,
+            },
+          ],
+          etag: data.etag ?? '',
+        };
+      } catch (error) {
+        assertError(error);
+        if (error.name === 'NotFoundError') {
+          return {
+            files: [],
+            etag: '',
+          };
+        }
+        throw error;
+      }
+    }
+
     if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) {
       throw new Error('GcsUrlReader only supports prefix-based searches');
     }

packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.test.ts

@@ -257,4 +257,36 @@ describe('HarnessUrlReader', () => {
       ).rejects.toThrow(NotFoundError);
     });
   });
+
+  describe('search', () => {
+    it('should return a single file when given an exact URL', async () => {
+      const data = await harnessProcessor.search(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(1);
+      expect(data.files[0].url).toBe(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/buffer.TXT',
+      );
+      expect((await data.files[0].content()).toString()).toEqual(
+        'Apache License',
+      );
+    });
+
+    it('should return empty list of files for not found files.', async () => {
+      const data = await harnessProcessor.search(
+        'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml',
+      );
+      expect(data.etag).toBe('');
+      expect(data.files.length).toBe(0);
+    });
+
+    it('throws if given URL with wildcard', async () => {
+      await expect(
+        harnessProcessor.search(
+          'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/*.yaml',
+        ),
+      ).rejects.toThrow('Unsupported search pattern URL');
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/HarnessUrlReader.ts

@@ -21,6 +21,7 @@ import {
   UrlReaderServiceReadUrlResponse,
   UrlReaderServiceSearchResponse,
   UrlReaderServiceReadTreeOptions,
+  UrlReaderServiceSearchOptions,
 } from '@backstage/backend-plugin-api';
 import {
   getHarnessRequestOptions,
@@ -35,6 +36,7 @@ import { ReadTreeResponseFactory, ReaderFactory } from './types';
 import fetch, { Response } from 'node-fetch';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import {
+  assertError,
   AuthenticationError,
   NotFoundError,
   NotModifiedError,
@@ -154,8 +156,39 @@ export class HarnessUrlReader implements UrlReaderService {
     });
   }
 
-  search(): Promise<UrlReaderServiceSearchResponse> {
-    throw new Error('HarnessUrlReader search not implemented.');
+  async search(
+    url: string,
+    options?: UrlReaderServiceSearchOptions,
+  ): Promise<UrlReaderServiceSearchResponse> {
+    const { path } = parseHarnessUrl(this.integration.config, url);
+
+    if (path.match(/[*?]/)) {
+      throw new Error('Unsupported search pattern URL');
+    }
+
+    try {
+      const data = await this.readUrl(url, options);
+
+      return {
+        files: [
+          {
+            url: url,
+            content: data.buffer,
+            lastModifiedAt: data.lastModifiedAt,
+          },
+        ],
+        etag: data.etag ?? '',
+      };
+    } catch (error) {
+      assertError(error);
+      if (error.name === 'NotFoundError') {
+        return {
+          files: [],
+          etag: '',
+        };
+      }
+      throw error;
+    }
   }
 
   toString() {

packages/create-app/templates/default-app/app-config.yaml.hbs

@@ -103,6 +103,9 @@ catalog:
     #   target: https://github.com/backstage/backstage/blob/master/packages/catalog-model/examples/acme-corp.yaml
     #   rules:
     #     - allow: [User, Group]
+  # Experimental: Always use the search method in UrlReaderProcessor.
+  # New adopters are encouraged to enable it as this behavior will be the default in a future release.
+  useUrlReadersSearch: true
 
 kubernetes:
   # see https://backstage.io/docs/features/kubernetes/configuration for kubernetes configuration options

plugins/catalog-backend/config.d.ts

@@ -202,5 +202,19 @@ export interface Config {
      * housing catalog-info files.
      */
     processingInterval?: HumanDuration | false;
+
+    /**
+     * Defines if the UrlReaderProcessor should always call the search method of the
+     * different UrlReaders.
+     *
+     * If set to false, the UrlReaderProcessor will use the legacy behavior that tries to
+     * parse a Git URL and calls search if there's wildcard patterns and readUrl otherwise.
+     *
+     * If set to true, the UrlReaderProcessor always call the search method and lets each UrlReader
+     * determine if it's a search pattern or not.
+     *
+     * This flag is temporary and will be enabled by default in future releases.
+     */
+    useUrlReadersSearch?: boolean;
   };
 }

plugins/catalog-backend/package.json

@@ -103,6 +103,7 @@
     "@backstage/cli": "workspace:^",
     "@backstage/plugin-permission-common": "workspace:^",
     "@backstage/repo-tools": "workspace:^",
+    "@backstage/test-utils": "workspace:^",
     "@types/core-js": "^2.5.4",
     "@types/git-url-parse": "^9.0.0",
     "@types/glob": "^8.0.0",

plugins/catalog-backend/report.api.md

@@ -499,7 +499,11 @@ export function transformLegacyPolicyToProcessor(
 
 // @public (undocumented)
 export class UrlReaderProcessor implements CatalogProcessor_2 {
-  constructor(options: { reader: UrlReaderService; logger: LoggerService });
+  constructor(options: {
+    reader: UrlReaderService;
+    logger: LoggerService;
+    config?: Config;
+  });
   // (undocumented)
   getProcessorName(): string;
   // (undocumented)

plugins/catalog-backend/src/processors/UrlReaderProcessor.test.ts

@@ -18,7 +18,6 @@ import {
   mockServices,
   registerMswTestHooks,
 } from '@backstage/backend-test-utils';
-import { ConfigReader } from '@backstage/config';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
 import {
@@ -31,6 +30,8 @@ import {
 import { defaultEntityDataParser } from '../util/parse';
 import { UrlReaderProcessor } from './UrlReaderProcessor';
 import { UrlReaders } from '@backstage/backend-defaults/urlReader';
+import { UrlReaderService } from '@backstage/backend-plugin-api';
+import { mockApis } from '@backstage/test-utils';
 
 describe('UrlReaderProcessor', () => {
   const mockApiOrigin = 'http://localhost';
@@ -49,8 +50,10 @@ describe('UrlReaderProcessor', () => {
     const logger = mockServices.logger.mock();
     const reader = UrlReaders.default({
       logger,
-      config: new ConfigReader({
-        backend: { reading: { allow: [{ host: 'localhost' }] } },
+      config: mockServices.rootConfig({
+        data: {
+          backend: { reading: { allow: [{ host: 'localhost' }] } },
+        },
       }),
     });
     const processor = new UrlReaderProcessor({ reader, logger });
@@ -107,8 +110,10 @@ describe('UrlReaderProcessor', () => {
     const logger = mockServices.logger.mock();
     const reader = UrlReaders.default({
       logger,
-      config: new ConfigReader({
-        backend: { reading: { allow: [{ host: 'localhost' }] } },
+      config: mockServices.rootConfig({
+        data: {
+          backend: { reading: { allow: [{ host: 'localhost' }] } },
+        },
       }),
     });
     server.use(
@@ -155,8 +160,10 @@ describe('UrlReaderProcessor', () => {
     const logger = mockServices.logger.mock();
     const reader = UrlReaders.default({
       logger,
-      config: new ConfigReader({
-        backend: { reading: { allow: [{ host: 'localhost' }] } },
+      config: mockServices.rootConfig({
+        data: {
+          backend: { reading: { allow: [{ host: 'localhost' }] } },
+        },
       }),
     });
     const processor = new UrlReaderProcessor({ reader, logger });
@@ -180,7 +187,6 @@ describe('UrlReaderProcessor', () => {
         mockCache,
       ),
     )) as CatalogProcessorErrorResult;
-
     expect(generated.type).toBe('error');
     expect(generated.location).toBe(spec);
     expect(generated.error.name).toBe('NotFoundError');
@@ -210,4 +216,36 @@ describe('UrlReaderProcessor', () => {
 
     expect(reader.search).toHaveBeenCalledTimes(1);
   });
+
+  it('uses search when catalog.useUrlReadersSearch flag is set to true', async () => {
+    const logger = mockServices.logger.mock();
+
+    const reader: jest.Mocked<UrlReaderService> = {
+      readUrl: jest.fn(),
+      readTree: jest.fn(),
+      search: jest.fn().mockImplementation(async () => []),
+    };
+
+    const config = mockApis.config({
+      data: {
+        catalog: {
+          useUrlReadersSearch: true,
+        },
+      },
+    });
+
+    const processor = new UrlReaderProcessor({ reader, logger, config });
+
+    const emit = jest.fn();
+
+    await processor.readLocation(
+      { type: 'url', target: 'https://github.com/a/b/blob/x/b.yaml' },
+      false,
+      emit,
+      defaultEntityDataParser,
+      mockCache,
+    );
+
+    expect(reader.search).toHaveBeenCalledTimes(1);
+  });
 });

plugins/catalog-backend/src/processors/UrlReaderProcessor.ts

@@ -29,6 +29,7 @@ import {
   processingResult,
 } from '@backstage/plugin-catalog-node';
 import { LoggerService, UrlReaderService } from '@backstage/backend-plugin-api';
+import { Config } from '@backstage/config';
 
 const CACHE_KEY = 'v1';
 
@@ -47,14 +48,25 @@ export class UrlReaderProcessor implements CatalogProcessor {
   // This limiter is used for only consuming a limited number of read streams
   // concurrently.
   #limiter: Limit;
+  #useUrlReadersSearch: boolean;
 
   constructor(
     private readonly options: {
       reader: UrlReaderService;
       logger: LoggerService;
+      config?: Config;
     },
   ) {
     this.#limiter = limiterFactory(5);
+
+    this.#useUrlReadersSearch =
+      this.options.config?.getOptionalBoolean('catalog.useUrlReadersSearch') ||
+      false;
+    if (!this.#useUrlReadersSearch) {
+      this.options.logger.warn(
+        'UrlReaderProcessor uses the legacy readUrl/search behavior which will be removed in a future release. Set catalog.useUrlReadersSearch to true to adopt the new behavior.',
+      );
+    }
   }
 
   getProcessorName() {
@@ -80,6 +92,15 @@ export class UrlReaderProcessor implements CatalogProcessor {
         cacheItem?.etag,
       );
 
+      if (response.length === 0 && !optional) {
+        emit(
+          processingResult.notFoundError(
+            location,
+            `Unable to read ${location.type}, no matching files found for ${location.target}`,
+          ),
+        );
+      }
+
       const parseResults: CatalogProcessorResult[] = [];
       for (const item of response) {
         for await (const parseResult of parser({
@@ -128,7 +149,19 @@ export class UrlReaderProcessor implements CatalogProcessor {
     location: string,
     etag?: string,
   ): Promise<{ response: { data: Buffer; url: string }[]; etag?: string }> {
-    // Does it contain globs? I.e. does it contain asterisks or question marks
+    // New behavior: always use the search method
+    if (this.#useUrlReadersSearch) {
+      const response = await this.options.reader.search(location, { etag });
+
+      const output = response.files.map(async file => ({
+        url: file.url,
+        data: await this.#limiter(file.content),
+      }));
+
+      return { response: await Promise.all(output), etag: response.etag };
+    }
+
+    // Old behavior: Does it contain globs? I.e. does it contain asterisks or question marks
     // (no curly braces for now)
 
     const { filepath } = parseGitUrl(location);

plugins/catalog-backend/src/service/CatalogBuilder.ts

@@ -381,7 +381,7 @@ export class CatalogBuilder {
 
     return [
       new FileReaderProcessor(),
-      new UrlReaderProcessor({ reader, logger }),
+      new UrlReaderProcessor({ reader, logger, config }),
       CodeOwnersProcessor.fromConfig(config, { logger, reader }),
       new AnnotateLocationEntityProcessor({ integrations }),
     ];

yarn.lock

@@ -6136,6 +6136,7 @@ __metadata:
     "@backstage/plugin-search-backend-module-catalog": "workspace:^"
     "@backstage/plugin-search-common": "workspace:^"
     "@backstage/repo-tools": "workspace:^"
+    "@backstage/test-utils": "workspace:^"
     "@backstage/types": "workspace:^"
     "@opentelemetry/api": ^1.9.0
     "@types/core-js": ^2.5.4