feat(integrations): Add googleGcs to ScmIntegrations and implement readTree for the UrlReader (#31998)

* feat: implementing google gcs read tree and scm integrations Signed-off-by: benjdlambert <ben@blam.sh> * feat: added changesets Signed-off-by: benjdlambert <ben@blam.sh> * chore: cleanup Signed-off-by: benjdlambert <ben@blam.sh> Signed-off-by: benjdlambert <ben@blam.sh> * chore: fix test issues Signed-off-by: benjdlambert <ben@blam.sh> * chore: fix code review comments Signed-off-by: benjdlambert <ben@blam.sh> * chore: fix code review comments Signed-off-by: benjdlambert <ben@blam.sh> --------- Signed-off-by: benjdlambert <ben@blam.sh>
2025-12-30 14:20:23 +01:00
parent 4c7bd5b8bc
commit 3afeab42a0
13 changed files with 324 additions and 12 deletions
@@ -23,12 +23,18 @@ import { UrlReaderPredicateTuple } from './types';
 import packageinfo from '../../../../package.json';
 import { mockServices } from '@backstage/backend-test-utils';
 import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
+import { Readable } from 'stream';

 const bucketGetFilesMock = jest.fn();
 class Bucket {
  getFiles(query: any) {
    return bucketGetFilesMock(query);
  }
+  file(_name: string) {
+    return {
+      createReadStream: () => Readable.from(Buffer.from('mock content')),
+    };
+  }
 }
 class Storage {
  bucket() {
@@ -186,4 +192,56 @@ describe('GcsUrlReader', () => {
      expect((await data.files[0].content()).toString()).toEqual('content');
    });
  });
+
+  describe('readTree', () => {
+    const { reader } = createReader({ integrations: { googleGcs: {} } })[0];
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('returns files with relative paths', async () => {
+      const mockFile1 = {
+        name: 'prefix/file1.yaml',
+        metadata: { updated: '2024-01-01T00:00:00Z' },
+        createReadStream: () => Readable.from(Buffer.from('content1')),
+      };
+      const mockFile2 = {
+        name: 'prefix/subdir/file2.yaml',
+        metadata: { updated: '2024-01-02T00:00:00Z' },
+        createReadStream: () => Readable.from(Buffer.from('content2')),
+      };
+      bucketGetFilesMock.mockResolvedValue([[mockFile1, mockFile2]]);
+
+      const result = await reader.readTree(
+        'https://storage.cloud.google.com/bucket/prefix/',
+      );
+      const files = await result.files();
+
+      expect(files).toHaveLength(2);
+      expect(files[0].path).toBe('file1.yaml');
+      expect(files[1].path).toBe('subdir/file2.yaml');
+    });
+
+    it('calls getFiles with correct prefix', async () => {
+      bucketGetFilesMock.mockResolvedValue([[]]);
+
+      await reader.readTree(
+        'https://storage.cloud.google.com/bucket/some/prefix/',
+      );
+
+      expect(bucketGetFilesMock).toHaveBeenCalledWith({
+        autoPaginate: true,
+        prefix: 'some/prefix/',
+      });
+    });
+
+    it('throws if readTree url contains glob pattern', async () => {
+      await expect(
+        reader.readTree('https://storage.cloud.google.com/bucket/path/*'),
+      ).rejects.toThrow(
+        'GcsUrlReader readTree does not support glob patterns, use search instead',
+      );
+    });
+  });
 });
@@ -17,22 +17,25 @@
 import * as GoogleCloud from '@google-cloud/storage';
 import {
  UrlReaderService,
+  UrlReaderServiceReadTreeOptions,
  UrlReaderServiceReadTreeResponse,
  UrlReaderServiceReadUrlOptions,
  UrlReaderServiceReadUrlResponse,
  UrlReaderServiceSearchOptions,
  UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { ReaderFactory } from './types';
+import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import getRawBody from 'raw-body';
 import {
  GoogleGcsIntegrationConfig,
  readGoogleGcsIntegrationConfig,
 } from '@backstage/integration';
+
 import { Readable } from 'stream';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import packageinfo from '../../../../package.json';
 import { assertError } from '@backstage/errors';
+import { relative } from 'path/posix';

 const GOOGLE_GCS_HOST = 'storage.cloud.google.com';

@@ -59,7 +62,7 @@ const parseURL = (
 * @public
 */
 export class GoogleGcsUrlReader implements UrlReaderService {
-  static factory: ReaderFactory = ({ config, logger }) => {
+  static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => {
    if (!config.has('integrations.googleGcs')) {
      return [];
    }
@@ -83,20 +86,29 @@ export class GoogleGcsUrlReader implements UrlReaderService {
        userAgent: `backstage/backend-defaults.GoogleGcsUrlReader/${packageinfo.version}`,
      });
    }
-    const reader = new GoogleGcsUrlReader(gcsConfig, storage);
+    const reader = new GoogleGcsUrlReader(gcsConfig, storage, {
+      treeResponseFactory,
+    });
    const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
    return [{ reader, predicate }];
  };

  private readonly integration: GoogleGcsIntegrationConfig;
  private readonly storage: GoogleCloud.Storage;
+  private readonly deps: {
+    treeResponseFactory: ReadTreeResponseFactory;
+  };

  constructor(
    integration: GoogleGcsIntegrationConfig,
    storage: GoogleCloud.Storage,
+    deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
  ) {
    this.integration = integration;
    this.storage = storage;
+    this.deps = deps;
  }

  private readStreamFromUrl(url: string): Readable {
@@ -121,8 +133,32 @@ export class GoogleGcsUrlReader implements UrlReaderService {
    return ReadUrlResponseFactory.fromReadable(stream);
  }

-  async readTree(): Promise<UrlReaderServiceReadTreeResponse> {
-    throw new Error('GcsUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    _options?: UrlReaderServiceReadTreeOptions,
+  ): Promise<UrlReaderServiceReadTreeResponse> {
+    const { bucket, key } = parseURL(url);
+
+    if (key.match(/[*?]/)) {
+      throw new Error(
+        'GcsUrlReader readTree does not support glob patterns, use search instead',
+      );
+    }
+
+    const [files] = await this.storage.bucket(bucket).getFiles({
+      autoPaginate: true,
+      prefix: key,
+    });
+
+    const responses = files.map(file => ({
+      data: file.createReadStream(),
+      path: relative(key, file.name),
+      lastModifiedAt: file.metadata.updated
+        ? new Date(file.metadata.updated as string)
+        : undefined,
+    }));
+
+    return this.deps.treeResponseFactory.fromReadableArray(responses);
  }

  async search(