Merge pull request #11174 from backstage/feat/gcs-reader-search

.changeset/backend-common-twist-your-ankle.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Implemented the `UrlReader.search()` method for Google Cloud Storage. Due to limitations in the underlying storage API, only prefix-based searches are supported right now (for example, `https://storage.cloud.google.com/your-bucket/some-path/*`).

packages/backend-common/src/reading/GoogleGcsUrlReader.test.ts

@@ -21,6 +21,24 @@ import { DefaultReadTreeResponseFactory } from './tree';
 import { GoogleGcsUrlReader } from './GoogleGcsUrlReader';
 import { UrlReaderPredicateTuple } from './types';
 
+const bucketGetFilesMock = jest.fn();
+jest.mock('@google-cloud/storage', () => {
+  class Bucket {
+    getFiles(query: any) {
+      return bucketGetFilesMock(query);
+    }
+  }
+  class Storage {
+    bucket() {
+      return new Bucket();
+    }
+  }
+  return {
+    __esModule: true,
+    Storage,
+  };
+});
+
 describe('GcsUrlReader', () => {
   const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
     return GoogleGcsUrlReader.factory({
@@ -92,4 +110,51 @@ describe('GcsUrlReader', () => {
       expect(predicate(new URL('https://a.example.com/test'))).toBe(false);
     });
   });
+
+  describe('search', () => {
+    const { reader } = createReader({ integrations: { googleGcs: {} } })[0];
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('throws if search url does not end with *', async () => {
+      const glob = 'https://storage.cloud.google.com/bucket/no-asterisk';
+      await expect(() => reader.search(glob)).rejects.toThrow(
+        'GcsUrlReader only supports prefix-based searches',
+      );
+    });
+
+    it('throws if search url looks truly glob-y', async () => {
+      const glob = 'https://storage.cloud.google.com/bucket/**/path*';
+      await expect(() => reader.search(glob)).rejects.toThrowError(
+        'GcsUrlReader only supports prefix-based searches',
+      );
+    });
+
+    it('searches with expected prefix and pagination', () => {
+      bucketGetFilesMock.mockResolvedValue([[]]);
+      const glob = 'https://storage.cloud.google.com/bucket/path/some-prefix-*';
+
+      reader.search(glob);
+      expect(bucketGetFilesMock).toHaveBeenCalledWith(
+        expect.objectContaining({
+          autoPaginate: true,
+          prefix: 'path/some-prefix-',
+        }),
+      );
+    });
+
+    it('returns valid SearchResponse object', async () => {
+      const expectedFile = { name: 'path/some-prefix-1.txt' };
+      bucketGetFilesMock.mockResolvedValue([[expectedFile]]);
+      const glob = 'https://storage.cloud.google.com/bucket/path/some-prefix-*';
+
+      const result = await reader.search(glob);
+      expect(result.files).toHaveLength(1);
+      expect(result.files[0].url).toEqual(
+        'https://storage.cloud.google.com/bucket/path/some-prefix-1.txt',
+      );
+    });
+  });
 });

packages/backend-common/src/reading/GoogleGcsUrlReader.ts

@@ -110,8 +110,34 @@ export class GoogleGcsUrlReader implements UrlReader {
     throw new Error('GcsUrlReader does not implement readTree');
   }
 
-  async search(): Promise<SearchResponse> {
-    throw new Error('GcsUrlReader does not implement search');
+  async search(url: string): Promise<SearchResponse> {
+    const { bucket, key: pattern } = parseURL(url);
+
+    if (!pattern.endsWith('*') || pattern.indexOf('*') !== pattern.length - 1) {
+      throw new Error('GcsUrlReader only supports prefix-based searches');
+    }
+
+    const [files] = await this.storage.bucket(bucket).getFiles({
+      autoPaginate: true,
+      prefix: pattern.split('*').join(''),
+    });
+
+    return {
+      files: files.map(file => {
+        const fullUrl = ['https:/', GOOGLE_GCS_HOST, bucket, file.name].join(
+          '/',
+        );
+        return {
+          url: fullUrl,
+          content: async () => {
+            const readResponse = await this.readUrl(fullUrl);
+            return readResponse.buffer();
+          },
+        };
+      }),
+      // TODO etag is not implemented yet.
+      etag: 'NOT/IMPLEMENTED',
+    };
   }
 
   toString() {