backend-common: implement UrlReader.search that does glob matching

plugins/catalog-backend/src/ingestion/LocationReaders.ts

@@ -139,6 +139,16 @@ export class LocationReaders implements LocationReader {
       if (emitResult.type === 'relation') {
         throw new Error('readLocation may not emit entity relations');
       }
+      if (
+        emitResult.type === 'location' &&
+        emitResult.location.type === item.location.type &&
+        emitResult.location.target === item.location.target
+      ) {
+        // Ignore self-referential locations silently (this can happen for
+        // example if you use a glob target like "**/*.yaml" in a Location
+        // entity)
+        return;
+      }
       emit(emitResult);
     };
 

plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts

@@ -160,7 +160,7 @@ describe('CodeOwnersProcessor', () => {
       const read = jest
         .fn()
         .mockResolvedValue(mockReadResult({ data: ownersText }));
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
       const result = await findRawCodeOwners(mockLocation(), {
         reader,
         logger,
@@ -170,7 +170,7 @@ describe('CodeOwnersProcessor', () => {
 
     it('should return undefined when no codeowner', async () => {
       const read = jest.fn().mockRejectedValue(mockReadResult());
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
 
       await expect(
         findRawCodeOwners(mockLocation(), { reader, logger }),
@@ -184,7 +184,7 @@ describe('CodeOwnersProcessor', () => {
         .mockImplementationOnce(() => mockReadResult({ error: 'foo' }))
         .mockImplementationOnce(() => mockReadResult({ error: 'bar' }))
         .mockResolvedValue(mockReadResult({ data: ownersText }));
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
 
       const result = await findRawCodeOwners(mockLocation(), {
         reader,
@@ -206,7 +206,7 @@ describe('CodeOwnersProcessor', () => {
       const read = jest
         .fn()
         .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
 
       const owner = await resolveCodeOwner(mockLocation(), { reader, logger });
       expect(owner).toBe('backstage-core');
@@ -216,7 +216,7 @@ describe('CodeOwnersProcessor', () => {
       const read = jest
         .fn()
         .mockImplementation(() => mockReadResult({ error: 'error: foo' }));
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
 
       await expect(
         resolveCodeOwner(mockLocation(), { reader, logger }),
@@ -230,7 +230,7 @@ describe('CodeOwnersProcessor', () => {
       const read = jest
         .fn()
         .mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
-      const reader = { read, readTree: jest.fn() };
+      const reader = { read, readTree: jest.fn(), search: jest.fn() };
       const processor = new CodeOwnersProcessor({ reader, logger });
 
       return { entity, processor, read };

plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts

@@ -27,7 +27,7 @@ import {
 
 describe('PlaceholderProcessor', () => {
   const read: jest.MockedFunction<ResolverRead> = jest.fn();
-  const reader: UrlReader = { read, readTree: jest.fn() };
+  const reader: UrlReader = { read, readTree: jest.fn(), search: jest.fn() };
 
   beforeEach(() => {
     jest.resetAllMocks();

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts

@@ -14,24 +14,29 @@
  * limitations under the License.
  */
 
-import { UrlReaderProcessor } from './UrlReaderProcessor';
-import { getVoidLogger, UrlReaders } from '@backstage/backend-common';
+import {
+  getVoidLogger,
+  UrlReader,
+  UrlReaders,
+} from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
+import { msw } from '@backstage/test-utils';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
-import { msw } from '@backstage/test-utils';
 import {
   CatalogProcessorEntityResult,
   CatalogProcessorErrorResult,
   CatalogProcessorResult,
 } from './types';
+import { UrlReaderProcessor } from './UrlReaderProcessor';
 import { defaultEntityDataParser } from './util/parse';
 
 describe('UrlReaderProcessor', () => {
   const mockApiOrigin = 'http://localhost';
-  const server = setupServer();
 
+  const server = setupServer();
   msw.setupDefaultHandlers(server);
+
   it('should load from url', async () => {
     const logger = getVoidLogger();
     const reader = UrlReaders.default({
@@ -57,7 +62,7 @@ describe('UrlReaderProcessor', () => {
     )) as CatalogProcessorEntityResult;
 
     expect(generated.type).toBe('entity');
-    expect(generated.location).toBe(spec);
+    expect(generated.location).toEqual(spec);
     expect(generated.entity).toEqual({ mock: 'entity' });
   });
 
@@ -92,4 +97,27 @@ describe('UrlReaderProcessor', () => {
       `Unable to read url, NotFoundError: could not read ${mockApiOrigin}/component-notfound.yaml, 404 Not Found`,
     );
   });
+
+  it('uses search when there are globs', async () => {
+    const logger = getVoidLogger();
+
+    const reader: jest.Mocked<UrlReader> = {
+      read: jest.fn(),
+      readTree: jest.fn(),
+      search: jest.fn().mockImplementation(async () => []),
+    };
+
+    const processor = new UrlReaderProcessor({ reader, logger });
+
+    const emit = jest.fn();
+
+    await processor.readLocation(
+      { type: 'url', target: 'https://github.com/a/b/blob/x/**/b.yaml' },
+      false,
+      emit,
+      defaultEntityDataParser,
+    );
+
+    expect(reader.search).toBeCalledTimes(1);
+  });
 });

plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts

@@ -16,6 +16,8 @@
 
 import { UrlReader } from '@backstage/backend-common';
 import { LocationSpec } from '@backstage/catalog-model';
+import parseGitUrl from 'git-url-parse';
+import limiterFactory from 'p-limit';
 import { Logger } from 'winston';
 import * as result from './results';
 import {
@@ -59,10 +61,14 @@ export class UrlReaderProcessor implements CatalogProcessor {
     }
 
     try {
-      const data = await this.options.reader.read(location.target);
-
-      for await (const parseResult of parser({ data, location })) {
-        emit(parseResult);
+      const output = await this.doRead(location.target);
+      for (const item of output) {
+        for await (const parseResult of parser({
+          data: item.data,
+          location: { type: location.type, target: item.url },
+        })) {
+          emit(parseResult);
+        }
       }
     } catch (error) {
       const message = `Unable to read ${location.type}, ${error}`;
@@ -78,4 +84,25 @@ export class UrlReaderProcessor implements CatalogProcessor {
 
     return true;
   }
+
+  private async doRead(
+    location: string,
+  ): Promise<{ data: Buffer; url: string }[]> {
+    // Does it contain globs? I.e. does it contain asterisks or question marks
+    // (no curly braces for now)
+    const { filepath } = parseGitUrl(location);
+    if (filepath?.match(/[*?]/)) {
+      const limiter = limiterFactory(5);
+      const response = await this.options.reader.search(location);
+      const output = response.files.map(async file => ({
+        url: file.url,
+        data: await limiter(file.content),
+      }));
+      return Promise.all(output);
+    }
+
+    // Otherwise do a plain read
+    const data = await this.options.reader.read(location);
+    return [{ url: location, data }];
+  }
 }

plugins/catalog-backend/src/service/CatalogBuilder.test.ts

@@ -44,6 +44,7 @@ describe('CatalogBuilder', () => {
   const reader: jest.Mocked<UrlReader> = {
     read: jest.fn(),
     readTree: jest.fn(),
+    search: jest.fn(),
   };
   const env: CatalogEnvironment = {
     logger: getVoidLogger(),

plugins/techdocs-backend/src/service/standaloneServer.ts

@@ -53,6 +53,7 @@ export async function startStandaloneServer(
   const mockUrlReader: jest.Mocked<UrlReader> = {
     read: jest.fn(),
     readTree: jest.fn(),
+    search: jest.fn(),
   };
 
   logger.debug('Creating application...');