feat(integrations): Add googleGcs to ScmIntegrations and implement readTree for the UrlReader (#31998)

* feat: implementing google gcs read tree and scm integrations

Signed-off-by: benjdlambert <ben@blam.sh>

* feat: added changesets

Signed-off-by: benjdlambert <ben@blam.sh>

* chore: cleanup
Signed-off-by: benjdlambert <ben@blam.sh>

Signed-off-by: benjdlambert <ben@blam.sh>

* chore: fix test issues

Signed-off-by: benjdlambert <ben@blam.sh>

* chore: fix code review comments
Signed-off-by: benjdlambert <ben@blam.sh>

* chore: fix code review comments

Signed-off-by: benjdlambert <ben@blam.sh>

---------

Signed-off-by: benjdlambert <ben@blam.sh>

.changeset/clever-streets-roll.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-defaults': patch
+---
+
+Implementing `readTree` for `GoogleGcsReader`

.changeset/hungry-mugs-fall.md

@@ -0,0 +1,5 @@
+---
+'@backstage/integration': patch
+---
+
+Implementing `ScmIntegration` for `GoogleGcs`

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts

@@ -23,12 +23,18 @@ import { UrlReaderPredicateTuple } from './types';
 import packageinfo from '../../../../package.json';
 import { mockServices } from '@backstage/backend-test-utils';
 import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api';
+import { Readable } from 'stream';
 
 const bucketGetFilesMock = jest.fn();
 class Bucket {
   getFiles(query: any) {
     return bucketGetFilesMock(query);
   }
+  file(_name: string) {
+    return {
+      createReadStream: () => Readable.from(Buffer.from('mock content')),
+    };
+  }
 }
 class Storage {
   bucket() {
@@ -186,4 +192,56 @@ describe('GcsUrlReader', () => {
       expect((await data.files[0].content()).toString()).toEqual('content');
     });
   });
+
+  describe('readTree', () => {
+    const { reader } = createReader({ integrations: { googleGcs: {} } })[0];
+
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('returns files with relative paths', async () => {
+      const mockFile1 = {
+        name: 'prefix/file1.yaml',
+        metadata: { updated: '2024-01-01T00:00:00Z' },
+        createReadStream: () => Readable.from(Buffer.from('content1')),
+      };
+      const mockFile2 = {
+        name: 'prefix/subdir/file2.yaml',
+        metadata: { updated: '2024-01-02T00:00:00Z' },
+        createReadStream: () => Readable.from(Buffer.from('content2')),
+      };
+      bucketGetFilesMock.mockResolvedValue([[mockFile1, mockFile2]]);
+
+      const result = await reader.readTree(
+        'https://storage.cloud.google.com/bucket/prefix/',
+      );
+      const files = await result.files();
+
+      expect(files).toHaveLength(2);
+      expect(files[0].path).toBe('file1.yaml');
+      expect(files[1].path).toBe('subdir/file2.yaml');
+    });
+
+    it('calls getFiles with correct prefix', async () => {
+      bucketGetFilesMock.mockResolvedValue([[]]);
+
+      await reader.readTree(
+        'https://storage.cloud.google.com/bucket/some/prefix/',
+      );
+
+      expect(bucketGetFilesMock).toHaveBeenCalledWith({
+        autoPaginate: true,
+        prefix: 'some/prefix/',
+      });
+    });
+
+    it('throws if readTree url contains glob pattern', async () => {
+      await expect(
+        reader.readTree('https://storage.cloud.google.com/bucket/path/*'),
+      ).rejects.toThrow(
+        'GcsUrlReader readTree does not support glob patterns, use search instead',
+      );
+    });
+  });
 });

packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts

@@ -17,22 +17,25 @@
 import * as GoogleCloud from '@google-cloud/storage';
 import {
   UrlReaderService,
+  UrlReaderServiceReadTreeOptions,
   UrlReaderServiceReadTreeResponse,
   UrlReaderServiceReadUrlOptions,
   UrlReaderServiceReadUrlResponse,
   UrlReaderServiceSearchOptions,
   UrlReaderServiceSearchResponse,
 } from '@backstage/backend-plugin-api';
-import { ReaderFactory } from './types';
+import { ReaderFactory, ReadTreeResponseFactory } from './types';
 import getRawBody from 'raw-body';
 import {
   GoogleGcsIntegrationConfig,
   readGoogleGcsIntegrationConfig,
 } from '@backstage/integration';
+
 import { Readable } from 'stream';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import packageinfo from '../../../../package.json';
 import { assertError } from '@backstage/errors';
+import { relative } from 'path/posix';
 
 const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
 
@@ -59,7 +62,7 @@ const parseURL = (
  * @public
  */
 export class GoogleGcsUrlReader implements UrlReaderService {
-  static factory: ReaderFactory = ({ config, logger }) => {
+  static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => {
     if (!config.has('integrations.googleGcs')) {
       return [];
     }
@@ -83,20 +86,29 @@ export class GoogleGcsUrlReader implements UrlReaderService {
         userAgent: `backstage/backend-defaults.GoogleGcsUrlReader/${packageinfo.version}`,
       });
     }
-    const reader = new GoogleGcsUrlReader(gcsConfig, storage);
+    const reader = new GoogleGcsUrlReader(gcsConfig, storage, {
+      treeResponseFactory,
+    });
     const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST;
     return [{ reader, predicate }];
   };
 
   private readonly integration: GoogleGcsIntegrationConfig;
   private readonly storage: GoogleCloud.Storage;
+  private readonly deps: {
+    treeResponseFactory: ReadTreeResponseFactory;
+  };
 
   constructor(
     integration: GoogleGcsIntegrationConfig,
     storage: GoogleCloud.Storage,
+    deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
   ) {
     this.integration = integration;
     this.storage = storage;
+    this.deps = deps;
   }
 
   private readStreamFromUrl(url: string): Readable {
@@ -121,8 +133,32 @@ export class GoogleGcsUrlReader implements UrlReaderService {
     return ReadUrlResponseFactory.fromReadable(stream);
   }
 
-  async readTree(): Promise<UrlReaderServiceReadTreeResponse> {
-    throw new Error('GcsUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    _options?: UrlReaderServiceReadTreeOptions,
+  ): Promise<UrlReaderServiceReadTreeResponse> {
+    const { bucket, key } = parseURL(url);
+
+    if (key.match(/[*?]/)) {
+      throw new Error(
+        'GcsUrlReader readTree does not support glob patterns, use search instead',
+      );
+    }
+
+    const [files] = await this.storage.bucket(bucket).getFiles({
+      autoPaginate: true,
+      prefix: key,
+    });
+
+    const responses = files.map(file => ({
+      data: file.createReadStream(),
+      path: relative(key, file.name),
+      lastModifiedAt: file.metadata.updated
+        ? new Date(file.metadata.updated as string)
+        : undefined,
+    }));
+
+    return this.deps.treeResponseFactory.fromReadableArray(responses);
   }
 
   async search(

packages/integration-react/src/api/ScmIntegrationsApi.test.ts

@@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => {
 
   it('should be instantiated', () => {
     const i = ScmIntegrationsApi.fromConfig(new ConfigReader({}));
-    expect(i.list().length).toBe(7); // The default ones
+    expect(i.list().length).toBe(8); // The default ones
   });
 });

packages/integration/report.api.md

@@ -781,8 +781,30 @@ export type GitLabIntegrationConfig = {
   commitSigningKey?: string;
 };
 
+// @public
+export class GoogleGcsIntegration implements ScmIntegration {
+  constructor(integrationConfig: GoogleGcsIntegrationConfig);
+  // (undocumented)
+  get config(): GoogleGcsIntegrationConfig;
+  // (undocumented)
+  static factory: ScmIntegrationsFactory<GoogleGcsIntegration>;
+  // (undocumented)
+  resolveEditUrl(url: string): string;
+  // (undocumented)
+  resolveUrl(options: {
+    url: string;
+    base: string;
+    lineNumber?: number | undefined;
+  }): string;
+  // (undocumented)
+  get title(): string;
+  // (undocumented)
+  get type(): string;
+}
+
 // @public
 export type GoogleGcsIntegrationConfig = {
+  host: string;
   clientEmail?: string;
   privateKey?: string;
 };
@@ -840,6 +862,8 @@ export interface IntegrationsByType {
   // (undocumented)
   gitlab: ScmIntegrationsGroup<GitLabIntegration>;
   // (undocumented)
+  googleGcs: ScmIntegrationsGroup<GoogleGcsIntegration>;
+  // (undocumented)
   harness: ScmIntegrationsGroup<HarnessIntegration>;
 }
 
@@ -1110,6 +1134,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
   // (undocumented)
   get gitlab(): ScmIntegrationsGroup<GitLabIntegration>;
   // (undocumented)
+  get googleGcs(): ScmIntegrationsGroup<GoogleGcsIntegration>;
+  // (undocumented)
   get harness(): ScmIntegrationsGroup<HarnessIntegration>;
   // (undocumented)
   list(): ScmIntegration[];

packages/integration/src/ScmIntegrations.test.ts

@@ -43,6 +43,7 @@ import {
   AzureBlobStorageIntegrationConfig,
   AzureBlobStorageIntergation,
 } from './azureBlobStorage';
+import { GoogleGcsIntegration, GoogleGcsIntegrationConfig } from './googleGcs';
 
 describe('ScmIntegrations', () => {
   const awsS3 = new AwsS3Integration({
@@ -93,6 +94,10 @@ describe('ScmIntegrations', () => {
     host: 'harness.local',
   } as HarnessIntegrationConfig);
 
+  const googleGcs = new GoogleGcsIntegration({
+    host: 'storage.cloud.google.com',
+  } as GoogleGcsIntegrationConfig);
+
   const i = new ScmIntegrations({
     awsS3: basicIntegrations([awsS3], item => item.config.host),
     awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host),
@@ -108,6 +113,7 @@ describe('ScmIntegrations', () => {
     github: basicIntegrations([github], item => item.config.host),
     gitlab: basicIntegrations([gitlab], item => item.config.host),
     gitea: basicIntegrations([gitea], item => item.config.host),
+    googleGcs: basicIntegrations([googleGcs], item => item.config.host),
     harness: basicIntegrations([harness], item => item.config.host),
   });
 

packages/integration/src/ScmIntegrations.ts

@@ -30,6 +30,7 @@ import { ScmIntegrationRegistry } from './registry';
 import { GiteaIntegration } from './gitea';
 import { HarnessIntegration } from './harness/HarnessIntegration';
 import { AzureBlobStorageIntergation } from './azureBlobStorage';
+import { GoogleGcsIntegration } from './googleGcs/GoogleGcsIntegration';
 
 /**
  * The set of supported integrations.
@@ -51,6 +52,7 @@ export interface IntegrationsByType {
   github: ScmIntegrationsGroup<GithubIntegration>;
   gitlab: ScmIntegrationsGroup<GitLabIntegration>;
   gitea: ScmIntegrationsGroup<GiteaIntegration>;
+  googleGcs: ScmIntegrationsGroup<GoogleGcsIntegration>;
   harness: ScmIntegrationsGroup<HarnessIntegration>;
 }
 
@@ -75,6 +77,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
       github: GithubIntegration.factory({ config }),
       gitlab: GitLabIntegration.factory({ config }),
       gitea: GiteaIntegration.factory({ config }),
+      googleGcs: GoogleGcsIntegration.factory({ config }),
       harness: HarnessIntegration.factory({ config }),
     });
   }
@@ -130,6 +133,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
     return this.byType.gitea;
   }
 
+  get googleGcs(): ScmIntegrationsGroup<GoogleGcsIntegration> {
+    return this.byType.googleGcs;
+  }
+
   get harness(): ScmIntegrationsGroup<HarnessIntegration> {
     return this.byType.harness;
   }

packages/integration/src/googleGcs/GoogleGcsIntegration.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ConfigReader } from '@backstage/config';
+import { GoogleGcsIntegration } from './GoogleGcsIntegration';
+
+describe('GoogleGcsIntegration', () => {
+  it('has a working factory', () => {
+    const integrations = GoogleGcsIntegration.factory({
+      config: new ConfigReader({
+        integrations: {
+          googleGcs: {
+            clientEmail: 'someone@example.com',
+            privateKey: 'fake-key',
+          },
+        },
+      }),
+    });
+    expect(integrations.list().length).toBe(1);
+    expect(integrations.list()[0].config.host).toBe('storage.cloud.google.com');
+  });
+
+  it('returns default integration when no config', () => {
+    const integrations = GoogleGcsIntegration.factory({
+      config: new ConfigReader({
+        integrations: {},
+      }),
+    });
+    expect(integrations.list().length).toBe(1);
+    expect(integrations.list()[0].config.host).toBe('storage.cloud.google.com');
+  });
+
+  it('returns the basics', () => {
+    const integration = new GoogleGcsIntegration({
+      host: 'storage.cloud.google.com',
+    });
+    expect(integration.type).toBe('googleGcs');
+    expect(integration.title).toBe('storage.cloud.google.com');
+  });
+
+  describe('resolveUrl', () => {
+    it('works for valid urls', () => {
+      const integration = new GoogleGcsIntegration({
+        host: 'storage.cloud.google.com',
+      });
+
+      expect(
+        integration.resolveUrl({
+          url: 'https://storage.cloud.google.com/bucket/catalog-info.yaml',
+          base: 'https://storage.cloud.google.com/bucket/catalog-info.yaml',
+        }),
+      ).toBe('https://storage.cloud.google.com/bucket/catalog-info.yaml');
+    });
+  });
+
+  it('resolve edit URL', () => {
+    const integration = new GoogleGcsIntegration({
+      host: 'storage.cloud.google.com',
+    });
+
+    expect(
+      integration.resolveEditUrl(
+        'https://storage.cloud.google.com/bucket/catalog-info.yaml',
+      ),
+    ).toBe('https://storage.cloud.google.com/bucket/catalog-info.yaml');
+  });
+});

packages/integration/src/googleGcs/GoogleGcsIntegration.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
+import { ScmIntegration, ScmIntegrationsFactory } from '../types';
+import {
+  GoogleGcsIntegrationConfig,
+  readGoogleGcsIntegrationConfig,
+  GOOGLE_GCS_HOST,
+} from './config';
+
+/**
+ * A Google Cloud Storage based integration.
+ *
+ * @public
+ */
+export class GoogleGcsIntegration implements ScmIntegration {
+  static factory: ScmIntegrationsFactory<GoogleGcsIntegration> = ({
+    config,
+  }) => {
+    const gcsConfig = config.has('integrations.googleGcs')
+      ? readGoogleGcsIntegrationConfig(
+          config.getConfig('integrations.googleGcs'),
+        )
+      : { host: GOOGLE_GCS_HOST };
+
+    return basicIntegrations(
+      [new GoogleGcsIntegration(gcsConfig)],
+      i => i.config.host,
+    );
+  };
+
+  get type(): string {
+    return 'googleGcs';
+  }
+
+  get title(): string {
+    return this.integrationConfig.host;
+  }
+
+  get config(): GoogleGcsIntegrationConfig {
+    return this.integrationConfig;
+  }
+
+  constructor(private readonly integrationConfig: GoogleGcsIntegrationConfig) {}
+
+  resolveUrl(options: {
+    url: string;
+    base: string;
+    lineNumber?: number | undefined;
+  }): string {
+    return defaultScmResolveUrl(options);
+  }
+
+  resolveEditUrl(url: string): string {
+    return url;
+  }
+}

packages/integration/src/googleGcs/config.test.ts

@@ -33,13 +33,16 @@ describe('readGoogleGcsIntegrationConfig', () => {
       }),
     );
     expect(output).toEqual({
+      host: 'storage.cloud.google.com',
       privateKey: 'fake-key',
       clientEmail: 'someone@example.com',
     });
   });
 
-  it('does not fail when config is not set', () => {
+  it('returns default config when config is not set', () => {
     const output = readGoogleGcsIntegrationConfig(buildConfig({}));
-    expect(output).toEqual({});
+    expect(output).toEqual({
+      host: 'storage.cloud.google.com',
+    });
   });
 });

packages/integration/src/googleGcs/config.ts

@@ -16,12 +16,22 @@
 
 import { Config } from '@backstage/config';
 
+/**
+ * The default Google Cloud Storage host.
+ *
+ */
+export const GOOGLE_GCS_HOST = 'storage.cloud.google.com';
+
 /**
  * The configuration parameters for a single Google Cloud Storage provider.
  *
  * @public
  */
 export type GoogleGcsIntegrationConfig = {
+  /**
+   * The host of the target that this matches on.
+   */
+  host: string;
   /**
    * Service account email used to authenticate requests.
    */
@@ -42,15 +52,19 @@ export function readGoogleGcsIntegrationConfig(
   config: Config,
 ): GoogleGcsIntegrationConfig {
   if (!config) {
-    return {};
+    return { host: GOOGLE_GCS_HOST };
   }
 
   if (!config.has('clientEmail') && !config.has('privateKey')) {
-    return {};
+    return { host: GOOGLE_GCS_HOST };
   }
 
   const privateKey = config.getString('privateKey').split('\\n').join('\n');
-
   const clientEmail = config.getString('clientEmail');
-  return { clientEmail: clientEmail, privateKey: privateKey };
+
+  return {
+    host: GOOGLE_GCS_HOST,
+    clientEmail,
+    privateKey,
+  };
 }

packages/integration/src/googleGcs/index.ts

@@ -16,3 +16,4 @@
 
 export { readGoogleGcsIntegrationConfig } from './config';
 export type { GoogleGcsIntegrationConfig } from './config';
+export { GoogleGcsIntegration } from './GoogleGcsIntegration';