From 3afeab42a04e1d092880a804ec1d98008cdebac7 Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Tue, 30 Dec 2025 14:20:23 +0100 Subject: [PATCH] `feat(integrations)`: Add `googleGcs` to `ScmIntegrations` and implement `readTree` for the `UrlReader` (#31998) * feat: implementing google gcs read tree and scm integrations Signed-off-by: benjdlambert * feat: added changesets Signed-off-by: benjdlambert * chore: cleanup Signed-off-by: benjdlambert Signed-off-by: benjdlambert * chore: fix test issues Signed-off-by: benjdlambert * chore: fix code review comments Signed-off-by: benjdlambert * chore: fix code review comments Signed-off-by: benjdlambert --------- Signed-off-by: benjdlambert --- .changeset/clever-streets-roll.md | 5 ++ .changeset/hungry-mugs-fall.md | 5 ++ .../urlReader/lib/GoogleGcsUrlReader.test.ts | 58 ++++++++++++++ .../urlReader/lib/GoogleGcsUrlReader.ts | 46 +++++++++-- .../src/api/ScmIntegrationsApi.test.ts | 2 +- packages/integration/report.api.md | 26 ++++++ .../integration/src/ScmIntegrations.test.ts | 6 ++ packages/integration/src/ScmIntegrations.ts | 7 ++ .../googleGcs/GoogleGcsIntegration.test.ts | 80 +++++++++++++++++++ .../src/googleGcs/GoogleGcsIntegration.ts | 71 ++++++++++++++++ .../integration/src/googleGcs/config.test.ts | 7 +- packages/integration/src/googleGcs/config.ts | 22 ++++- packages/integration/src/googleGcs/index.ts | 1 + 13 files changed, 324 insertions(+), 12 deletions(-) create mode 100644 .changeset/clever-streets-roll.md create mode 100644 .changeset/hungry-mugs-fall.md create mode 100644 packages/integration/src/googleGcs/GoogleGcsIntegration.test.ts create mode 100644 packages/integration/src/googleGcs/GoogleGcsIntegration.ts diff --git a/.changeset/clever-streets-roll.md b/.changeset/clever-streets-roll.md new file mode 100644 index 0000000000..7efa961533 --- /dev/null +++ b/.changeset/clever-streets-roll.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Implementing `readTree` for `GoogleGcsReader` diff --git a/.changeset/hungry-mugs-fall.md b/.changeset/hungry-mugs-fall.md new file mode 100644 index 0000000000..4332d92cc3 --- /dev/null +++ b/.changeset/hungry-mugs-fall.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Implementing `ScmIntegration` for `GoogleGcs` diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts index 7429bae94f..52d9660632 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.test.ts @@ -23,12 +23,18 @@ import { UrlReaderPredicateTuple } from './types'; import packageinfo from '../../../../package.json'; import { mockServices } from '@backstage/backend-test-utils'; import { UrlReaderServiceReadUrlResponse } from '@backstage/backend-plugin-api'; +import { Readable } from 'stream'; const bucketGetFilesMock = jest.fn(); class Bucket { getFiles(query: any) { return bucketGetFilesMock(query); } + file(_name: string) { + return { + createReadStream: () => Readable.from(Buffer.from('mock content')), + }; + } } class Storage { bucket() { @@ -186,4 +192,56 @@ describe('GcsUrlReader', () => { expect((await data.files[0].content()).toString()).toEqual('content'); }); }); + + describe('readTree', () => { + const { reader } = createReader({ integrations: { googleGcs: {} } })[0]; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns files with relative paths', async () => { + const mockFile1 = { + name: 'prefix/file1.yaml', + metadata: { updated: '2024-01-01T00:00:00Z' }, + createReadStream: () => Readable.from(Buffer.from('content1')), + }; + const mockFile2 = { + name: 'prefix/subdir/file2.yaml', + metadata: { updated: '2024-01-02T00:00:00Z' }, + createReadStream: () => Readable.from(Buffer.from('content2')), + }; + bucketGetFilesMock.mockResolvedValue([[mockFile1, mockFile2]]); + + const result = await reader.readTree( + 'https://storage.cloud.google.com/bucket/prefix/', + ); + const files = await result.files(); + + expect(files).toHaveLength(2); + expect(files[0].path).toBe('file1.yaml'); + expect(files[1].path).toBe('subdir/file2.yaml'); + }); + + it('calls getFiles with correct prefix', async () => { + bucketGetFilesMock.mockResolvedValue([[]]); + + await reader.readTree( + 'https://storage.cloud.google.com/bucket/some/prefix/', + ); + + expect(bucketGetFilesMock).toHaveBeenCalledWith({ + autoPaginate: true, + prefix: 'some/prefix/', + }); + }); + + it('throws if readTree url contains glob pattern', async () => { + await expect( + reader.readTree('https://storage.cloud.google.com/bucket/path/*'), + ).rejects.toThrow( + 'GcsUrlReader readTree does not support glob patterns, use search instead', + ); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts index 0bc5add43f..a29b98eb51 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/GoogleGcsUrlReader.ts @@ -17,22 +17,25 @@ import * as GoogleCloud from '@google-cloud/storage'; import { UrlReaderService, + UrlReaderServiceReadTreeOptions, UrlReaderServiceReadTreeResponse, UrlReaderServiceReadUrlOptions, UrlReaderServiceReadUrlResponse, UrlReaderServiceSearchOptions, UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -import { ReaderFactory } from './types'; +import { ReaderFactory, ReadTreeResponseFactory } from './types'; import getRawBody from 'raw-body'; import { GoogleGcsIntegrationConfig, readGoogleGcsIntegrationConfig, } from '@backstage/integration'; + import { Readable } from 'stream'; import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; import packageinfo from '../../../../package.json'; import { assertError } from '@backstage/errors'; +import { relative } from 'path/posix'; const GOOGLE_GCS_HOST = 'storage.cloud.google.com'; @@ -59,7 +62,7 @@ const parseURL = ( * @public */ export class GoogleGcsUrlReader implements UrlReaderService { - static factory: ReaderFactory = ({ config, logger }) => { + static factory: ReaderFactory = ({ config, logger, treeResponseFactory }) => { if (!config.has('integrations.googleGcs')) { return []; } @@ -83,20 +86,29 @@ export class GoogleGcsUrlReader implements UrlReaderService { userAgent: `backstage/backend-defaults.GoogleGcsUrlReader/${packageinfo.version}`, }); } - const reader = new GoogleGcsUrlReader(gcsConfig, storage); + const reader = new GoogleGcsUrlReader(gcsConfig, storage, { + treeResponseFactory, + }); const predicate = (url: URL) => url.host === GOOGLE_GCS_HOST; return [{ reader, predicate }]; }; private readonly integration: GoogleGcsIntegrationConfig; private readonly storage: GoogleCloud.Storage; + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }; constructor( integration: GoogleGcsIntegrationConfig, storage: GoogleCloud.Storage, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, ) { this.integration = integration; this.storage = storage; + this.deps = deps; } private readStreamFromUrl(url: string): Readable { @@ -121,8 +133,32 @@ export class GoogleGcsUrlReader implements UrlReaderService { return ReadUrlResponseFactory.fromReadable(stream); } - async readTree(): Promise { - throw new Error('GcsUrlReader does not implement readTree'); + async readTree( + url: string, + _options?: UrlReaderServiceReadTreeOptions, + ): Promise { + const { bucket, key } = parseURL(url); + + if (key.match(/[*?]/)) { + throw new Error( + 'GcsUrlReader readTree does not support glob patterns, use search instead', + ); + } + + const [files] = await this.storage.bucket(bucket).getFiles({ + autoPaginate: true, + prefix: key, + }); + + const responses = files.map(file => ({ + data: file.createReadStream(), + path: relative(key, file.name), + lastModifiedAt: file.metadata.updated + ? new Date(file.metadata.updated as string) + : undefined, + })); + + return this.deps.treeResponseFactory.fromReadableArray(responses); } async search( diff --git a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts index 0a1f1b1b8f..55acd29eec 100644 --- a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts +++ b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts @@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => { it('should be instantiated', () => { const i = ScmIntegrationsApi.fromConfig(new ConfigReader({})); - expect(i.list().length).toBe(7); // The default ones + expect(i.list().length).toBe(8); // The default ones }); }); diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index 561dea979d..abcdb8898b 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -781,8 +781,30 @@ export type GitLabIntegrationConfig = { commitSigningKey?: string; }; +// @public +export class GoogleGcsIntegration implements ScmIntegration { + constructor(integrationConfig: GoogleGcsIntegrationConfig); + // (undocumented) + get config(): GoogleGcsIntegrationConfig; + // (undocumented) + static factory: ScmIntegrationsFactory; + // (undocumented) + resolveEditUrl(url: string): string; + // (undocumented) + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string; + // (undocumented) + get title(): string; + // (undocumented) + get type(): string; +} + // @public export type GoogleGcsIntegrationConfig = { + host: string; clientEmail?: string; privateKey?: string; }; @@ -840,6 +862,8 @@ export interface IntegrationsByType { // (undocumented) gitlab: ScmIntegrationsGroup; // (undocumented) + googleGcs: ScmIntegrationsGroup; + // (undocumented) harness: ScmIntegrationsGroup; } @@ -1110,6 +1134,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry { // (undocumented) get gitlab(): ScmIntegrationsGroup; // (undocumented) + get googleGcs(): ScmIntegrationsGroup; + // (undocumented) get harness(): ScmIntegrationsGroup; // (undocumented) list(): ScmIntegration[]; diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index 0e0c181861..7f75ab44fc 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -43,6 +43,7 @@ import { AzureBlobStorageIntegrationConfig, AzureBlobStorageIntergation, } from './azureBlobStorage'; +import { GoogleGcsIntegration, GoogleGcsIntegrationConfig } from './googleGcs'; describe('ScmIntegrations', () => { const awsS3 = new AwsS3Integration({ @@ -93,6 +94,10 @@ describe('ScmIntegrations', () => { host: 'harness.local', } as HarnessIntegrationConfig); + const googleGcs = new GoogleGcsIntegration({ + host: 'storage.cloud.google.com', + } as GoogleGcsIntegrationConfig); + const i = new ScmIntegrations({ awsS3: basicIntegrations([awsS3], item => item.config.host), awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host), @@ -108,6 +113,7 @@ describe('ScmIntegrations', () => { github: basicIntegrations([github], item => item.config.host), gitlab: basicIntegrations([gitlab], item => item.config.host), gitea: basicIntegrations([gitea], item => item.config.host), + googleGcs: basicIntegrations([googleGcs], item => item.config.host), harness: basicIntegrations([harness], item => item.config.host), }); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index f11cb3c17e..573975da0e 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -30,6 +30,7 @@ import { ScmIntegrationRegistry } from './registry'; import { GiteaIntegration } from './gitea'; import { HarnessIntegration } from './harness/HarnessIntegration'; import { AzureBlobStorageIntergation } from './azureBlobStorage'; +import { GoogleGcsIntegration } from './googleGcs/GoogleGcsIntegration'; /** * The set of supported integrations. @@ -51,6 +52,7 @@ export interface IntegrationsByType { github: ScmIntegrationsGroup; gitlab: ScmIntegrationsGroup; gitea: ScmIntegrationsGroup; + googleGcs: ScmIntegrationsGroup; harness: ScmIntegrationsGroup; } @@ -75,6 +77,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { github: GithubIntegration.factory({ config }), gitlab: GitLabIntegration.factory({ config }), gitea: GiteaIntegration.factory({ config }), + googleGcs: GoogleGcsIntegration.factory({ config }), harness: HarnessIntegration.factory({ config }), }); } @@ -130,6 +133,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return this.byType.gitea; } + get googleGcs(): ScmIntegrationsGroup { + return this.byType.googleGcs; + } + get harness(): ScmIntegrationsGroup { return this.byType.harness; } diff --git a/packages/integration/src/googleGcs/GoogleGcsIntegration.test.ts b/packages/integration/src/googleGcs/GoogleGcsIntegration.test.ts new file mode 100644 index 0000000000..5666a0aa5e --- /dev/null +++ b/packages/integration/src/googleGcs/GoogleGcsIntegration.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { GoogleGcsIntegration } from './GoogleGcsIntegration'; + +describe('GoogleGcsIntegration', () => { + it('has a working factory', () => { + const integrations = GoogleGcsIntegration.factory({ + config: new ConfigReader({ + integrations: { + googleGcs: { + clientEmail: 'someone@example.com', + privateKey: 'fake-key', + }, + }, + }), + }); + expect(integrations.list().length).toBe(1); + expect(integrations.list()[0].config.host).toBe('storage.cloud.google.com'); + }); + + it('returns default integration when no config', () => { + const integrations = GoogleGcsIntegration.factory({ + config: new ConfigReader({ + integrations: {}, + }), + }); + expect(integrations.list().length).toBe(1); + expect(integrations.list()[0].config.host).toBe('storage.cloud.google.com'); + }); + + it('returns the basics', () => { + const integration = new GoogleGcsIntegration({ + host: 'storage.cloud.google.com', + }); + expect(integration.type).toBe('googleGcs'); + expect(integration.title).toBe('storage.cloud.google.com'); + }); + + describe('resolveUrl', () => { + it('works for valid urls', () => { + const integration = new GoogleGcsIntegration({ + host: 'storage.cloud.google.com', + }); + + expect( + integration.resolveUrl({ + url: 'https://storage.cloud.google.com/bucket/catalog-info.yaml', + base: 'https://storage.cloud.google.com/bucket/catalog-info.yaml', + }), + ).toBe('https://storage.cloud.google.com/bucket/catalog-info.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new GoogleGcsIntegration({ + host: 'storage.cloud.google.com', + }); + + expect( + integration.resolveEditUrl( + 'https://storage.cloud.google.com/bucket/catalog-info.yaml', + ), + ).toBe('https://storage.cloud.google.com/bucket/catalog-info.yaml'); + }); +}); diff --git a/packages/integration/src/googleGcs/GoogleGcsIntegration.ts b/packages/integration/src/googleGcs/GoogleGcsIntegration.ts new file mode 100644 index 0000000000..11642a231c --- /dev/null +++ b/packages/integration/src/googleGcs/GoogleGcsIntegration.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { + GoogleGcsIntegrationConfig, + readGoogleGcsIntegrationConfig, + GOOGLE_GCS_HOST, +} from './config'; + +/** + * A Google Cloud Storage based integration. + * + * @public + */ +export class GoogleGcsIntegration implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ + config, + }) => { + const gcsConfig = config.has('integrations.googleGcs') + ? readGoogleGcsIntegrationConfig( + config.getConfig('integrations.googleGcs'), + ) + : { host: GOOGLE_GCS_HOST }; + + return basicIntegrations( + [new GoogleGcsIntegration(gcsConfig)], + i => i.config.host, + ); + }; + + get type(): string { + return 'googleGcs'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): GoogleGcsIntegrationConfig { + return this.integrationConfig; + } + + constructor(private readonly integrationConfig: GoogleGcsIntegrationConfig) {} + + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + return defaultScmResolveUrl(options); + } + + resolveEditUrl(url: string): string { + return url; + } +} diff --git a/packages/integration/src/googleGcs/config.test.ts b/packages/integration/src/googleGcs/config.test.ts index 892440c64a..d074cb7129 100644 --- a/packages/integration/src/googleGcs/config.test.ts +++ b/packages/integration/src/googleGcs/config.test.ts @@ -33,13 +33,16 @@ describe('readGoogleGcsIntegrationConfig', () => { }), ); expect(output).toEqual({ + host: 'storage.cloud.google.com', privateKey: 'fake-key', clientEmail: 'someone@example.com', }); }); - it('does not fail when config is not set', () => { + it('returns default config when config is not set', () => { const output = readGoogleGcsIntegrationConfig(buildConfig({})); - expect(output).toEqual({}); + expect(output).toEqual({ + host: 'storage.cloud.google.com', + }); }); }); diff --git a/packages/integration/src/googleGcs/config.ts b/packages/integration/src/googleGcs/config.ts index 3c03204e4c..9421b9dfba 100644 --- a/packages/integration/src/googleGcs/config.ts +++ b/packages/integration/src/googleGcs/config.ts @@ -16,12 +16,22 @@ import { Config } from '@backstage/config'; +/** + * The default Google Cloud Storage host. + * + */ +export const GOOGLE_GCS_HOST = 'storage.cloud.google.com'; + /** * The configuration parameters for a single Google Cloud Storage provider. * * @public */ export type GoogleGcsIntegrationConfig = { + /** + * The host of the target that this matches on. + */ + host: string; /** * Service account email used to authenticate requests. */ @@ -42,15 +52,19 @@ export function readGoogleGcsIntegrationConfig( config: Config, ): GoogleGcsIntegrationConfig { if (!config) { - return {}; + return { host: GOOGLE_GCS_HOST }; } if (!config.has('clientEmail') && !config.has('privateKey')) { - return {}; + return { host: GOOGLE_GCS_HOST }; } const privateKey = config.getString('privateKey').split('\\n').join('\n'); - const clientEmail = config.getString('clientEmail'); - return { clientEmail: clientEmail, privateKey: privateKey }; + + return { + host: GOOGLE_GCS_HOST, + clientEmail, + privateKey, + }; } diff --git a/packages/integration/src/googleGcs/index.ts b/packages/integration/src/googleGcs/index.ts index 3d7beae59d..c30138fc3e 100644 --- a/packages/integration/src/googleGcs/index.ts +++ b/packages/integration/src/googleGcs/index.ts @@ -16,3 +16,4 @@ export { readGoogleGcsIntegrationConfig } from './config'; export type { GoogleGcsIntegrationConfig } from './config'; +export { GoogleGcsIntegration } from './GoogleGcsIntegration';