From fbdc63116c9bf982f30f9f33fbc3c00b49e5a4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Kosi=C5=84ski?= Date: Tue, 1 Oct 2024 19:40:47 +0200 Subject: [PATCH] feat(techdocs): Allow to pass options to GCS publisher (#26836) * feat(techdocs): Allow to pass options to GCS publisher Signed-off-by: Adrian Kosinski --------- Signed-off-by: Adrian Kosinski --- .changeset/tough-fireants-itch.md | 6 ++++ docs/features/techdocs/using-cloud-storage.md | 30 +++++++++++++++++++ plugins/techdocs-backend/src/plugin.ts | 9 ++++++ plugins/techdocs-node/report.api.md | 25 ++++++++++++---- plugins/techdocs-node/src/extensions.ts | 5 ++++ .../src/stages/publish/googleStorage.test.ts | 24 ++++++++++++++- .../src/stages/publish/googleStorage.ts | 8 +++-- .../techdocs-node/src/stages/publish/index.ts | 1 + .../src/stages/publish/publish.ts | 6 +++- .../techdocs-node/src/stages/publish/types.ts | 10 +++++++ 10 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 .changeset/tough-fireants-itch.md diff --git a/.changeset/tough-fireants-itch.md b/.changeset/tough-fireants-itch.md new file mode 100644 index 0000000000..3567ac82d0 --- /dev/null +++ b/.changeset/tough-fireants-itch.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-techdocs-node': patch +--- + +Allow to pass StorageOptions to GCS Publisher diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 6ba826ea62..bdafb3532c 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -120,6 +120,36 @@ techdocs: Your Backstage app is now ready to use Google Cloud Storage for TechDocs, to store and read the static generated documentation files. +### Extending default Storage configuration + +If you need a non-standard configuration of Google Cloud Storage client, +`TechdocsPublisherExtensionPoint` is something you should look at. +You can register custom `StorageOptions` that will be used to configure the client. To do so, you +need to register publisher settings inside your module init, like in the following example: + +```typescript +export const gcsPublisherCustomizer = createBackendModule({ + pluginId: 'techdocs', + moduleId: 'gcs-publisher-customizer', + register(reg) { + reg.registerInit({ + deps: { + techdocsExtensionPoint: techdocsPublisherExtensionPoint, + }, + async init({ techdocsExtensionPoint }) { + const customOptions: StorageOptions = { + userAgent: 'my-custom-user-agent', + }; + techdocsExtensionPoint.registerPublisherSettings( + 'googleGcs', + customOptions, + ); + }, + }); + }, +}); +``` + ## Configuring AWS S3 Bucket with TechDocs **1. Set `techdocs.publisher.type` config in your `app-config.yaml`** diff --git a/plugins/techdocs-backend/src/plugin.ts b/plugins/techdocs-backend/src/plugin.ts index 0f3def79b1..9b5ae21a52 100644 --- a/plugins/techdocs-backend/src/plugin.ts +++ b/plugins/techdocs-backend/src/plugin.ts @@ -29,6 +29,7 @@ import { Preparers, Publisher, PublisherBase, + PublisherSettings, PublisherType, RemoteProtocol, techdocsBuildsExtensionPoint, @@ -89,6 +90,7 @@ export const techdocsPlugin = createBackendPlugin({ }); let customTechdocsPublisher: PublisherBase | undefined; + const publisherSettings: PublisherSettings = {}; env.registerExtensionPoint(techdocsPublisherExtensionPoint, { registerPublisher(type: PublisherType, publisher: PublisherBase) { if (customTechdocsPublisher) { @@ -96,6 +98,12 @@ export const techdocsPlugin = createBackendPlugin({ } customTechdocsPublisher = publisher; }, + registerPublisherSettings( + publisher: T, + settings: PublisherSettings[T], + ) { + publisherSettings[publisher] = settings; + }, }); env.registerInit({ @@ -144,6 +152,7 @@ export const techdocsPlugin = createBackendPlugin({ logger: winstonLogger, discovery: discovery, customPublisher: customTechdocsPublisher, + publisherSettings, }); // checks if the publisher is working and logs the result diff --git a/plugins/techdocs-node/report.api.md b/plugins/techdocs-node/report.api.md index ad6e01636f..1bd93d1b57 100644 --- a/plugins/techdocs-node/report.api.md +++ b/plugins/techdocs-node/report.api.md @@ -15,6 +15,7 @@ import { IndexableDocument } from '@backstage/plugin-search-common'; import { Logger } from 'winston'; import { LoggerService } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; +import { StorageOptions } from '@google-cloud/storage'; import { UrlReaderService } from '@backstage/backend-plugin-api'; import * as winston from 'winston'; import { Writable } from 'stream'; @@ -217,8 +218,15 @@ export type PublisherFactory = { logger: LoggerService; discovery: DiscoveryService; customPublisher?: PublisherBase | undefined; + publisherSettings?: PublisherSettings; }; +// @public +export interface PublisherSettings { + // (undocumented) + googleGcs?: StorageOptions; +} + // @public export type PublisherType = | 'local' @@ -344,6 +352,11 @@ export const techdocsPreparerExtensionPoint: ExtensionPoint( + publisher: T, + settings: PublisherSettings[T], + ): void; } // @public @@ -368,13 +381,15 @@ export class UrlPreparer implements PreparerBase { // Warnings were encountered during analysis: // -// src/extensions.d.ts:10:5 - (ae-undocumented) Missing documentation for "setBuildStrategy". -// src/extensions.d.ts:11:5 - (ae-undocumented) Missing documentation for "setBuildLogTransport". -// src/extensions.d.ts:25:5 - (ae-undocumented) Missing documentation for "setTechdocsGenerator". -// src/extensions.d.ts:39:5 - (ae-undocumented) Missing documentation for "registerPreparer". -// src/extensions.d.ts:53:5 - (ae-undocumented) Missing documentation for "registerPublisher". +// src/extensions.d.ts:11:5 - (ae-undocumented) Missing documentation for "setBuildStrategy". +// src/extensions.d.ts:12:5 - (ae-undocumented) Missing documentation for "setBuildLogTransport". +// src/extensions.d.ts:26:5 - (ae-undocumented) Missing documentation for "setTechdocsGenerator". +// src/extensions.d.ts:40:5 - (ae-undocumented) Missing documentation for "registerPreparer". +// src/extensions.d.ts:54:5 - (ae-undocumented) Missing documentation for "registerPublisher". +// src/extensions.d.ts:55:5 - (ae-undocumented) Missing documentation for "registerPublisherSettings". // src/stages/generate/index.d.ts:10:22 - (ae-undocumented) Missing documentation for "getMkDocsYml". // src/stages/publish/publish.d.ts:10:5 - (ae-undocumented) Missing documentation for "register". // src/stages/publish/publish.d.ts:11:5 - (ae-undocumented) Missing documentation for "get". +// src/stages/publish/types.d.ts:21:5 - (ae-undocumented) Missing documentation for "googleGcs". // src/techdocsTypes.d.ts:39:5 - (ae-undocumented) Missing documentation for "shouldBuild". ``` diff --git a/plugins/techdocs-node/src/extensions.ts b/plugins/techdocs-node/src/extensions.ts index e9a9b1d731..60498746ad 100644 --- a/plugins/techdocs-node/src/extensions.ts +++ b/plugins/techdocs-node/src/extensions.ts @@ -23,6 +23,7 @@ import { TechdocsGenerator, } from './stages'; import * as winston from 'winston'; +import { PublisherSettings } from './stages/publish/types'; /** * Extension point type for configuring TechDocs builds. @@ -89,6 +90,10 @@ export const techdocsPreparerExtensionPoint = */ export interface TechdocsPublisherExtensionPoint { registerPublisher(type: PublisherType, publisher: PublisherBase): void; + registerPublisherSettings( + publisher: T, + settings: PublisherSettings[T], + ): void; } /** diff --git a/plugins/techdocs-node/src/stages/publish/googleStorage.test.ts b/plugins/techdocs-node/src/stages/publish/googleStorage.test.ts index f09ba160d9..322149660e 100644 --- a/plugins/techdocs-node/src/stages/publish/googleStorage.test.ts +++ b/plugins/techdocs-node/src/stages/publish/googleStorage.test.ts @@ -26,9 +26,12 @@ import { createMockDirectory, mockServices, } from '@backstage/backend-test-utils'; +import { StorageOptions } from '@google-cloud/storage'; const mockDir = createMockDirectory(); +let createdStorageOptions: Array = []; + jest.mock('@google-cloud/storage', () => { class GCSFile { constructor(private readonly filePath: string) {} @@ -118,6 +121,10 @@ jest.mock('@google-cloud/storage', () => { } class Storage { + constructor(readonly options?: StorageOptions) { + createdStorageOptions.push(options); + } + bucket(bucketName: string) { return new Bucket(bucketName); } @@ -144,10 +151,12 @@ const createPublisherFromConfig = ({ bucketName = 'bucketName', bucketRootPath = '/', legacyUseCaseSensitiveTripletPaths = false, + storageOptions = {}, }: { bucketName?: string; bucketRootPath?: string; legacyUseCaseSensitiveTripletPaths?: boolean; + storageOptions?: StorageOptions; } = {}) => { const config = new ConfigReader({ techdocs: { @@ -162,7 +171,7 @@ const createPublisherFromConfig = ({ legacyUseCaseSensitiveTripletPaths, }, }); - return GoogleGCSPublish.fromConfig(config, logger); + return GoogleGCSPublish.fromConfig(config, logger, storageOptions); }; describe('GoogleGCSPublish', () => { @@ -211,11 +220,24 @@ describe('GoogleGCSPublish', () => { }; beforeEach(() => { + createdStorageOptions = []; mockDir.setContent({ [directory]: files, }); }); + it('should pass options to storage', () => { + createPublisherFromConfig({ + storageOptions: { + userAgent: 'Test-UA', + }, + }); + + expect(createdStorageOptions.map(opt => opt?.userAgent)).toContain( + 'Test-UA', + ); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { const publisher = createPublisherFromConfig(); diff --git a/plugins/techdocs-node/src/stages/publish/googleStorage.ts b/plugins/techdocs-node/src/stages/publish/googleStorage.ts index 519b0f8c6b..3eac9834d2 100644 --- a/plugins/techdocs-node/src/stages/publish/googleStorage.ts +++ b/plugins/techdocs-node/src/stages/publish/googleStorage.ts @@ -68,7 +68,11 @@ export class GoogleGCSPublish implements PublisherBase { this.bucketRootPath = options.bucketRootPath; } - static fromConfig(config: Config, logger: LoggerService): PublisherBase { + static fromConfig( + config: Config, + logger: LoggerService, + options?: StorageOptions, + ): PublisherBase { let bucketName = ''; try { bucketName = config.getString('techdocs.publisher.googleGcs.bucketName'); @@ -103,7 +107,7 @@ export class GoogleGCSPublish implements PublisherBase { } } - const clientOpts: StorageOptions = {}; + const clientOpts: StorageOptions = options ?? {}; if (projectId) { clientOpts.projectId = projectId; } diff --git a/plugins/techdocs-node/src/stages/publish/index.ts b/plugins/techdocs-node/src/stages/publish/index.ts index 7f6859367f..8d528b4db6 100644 --- a/plugins/techdocs-node/src/stages/publish/index.ts +++ b/plugins/techdocs-node/src/stages/publish/index.ts @@ -24,4 +24,5 @@ export type { MigrateRequest, ReadinessResponse, TechDocsMetadata, + PublisherSettings, } from './types'; diff --git a/plugins/techdocs-node/src/stages/publish/publish.ts b/plugins/techdocs-node/src/stages/publish/publish.ts index 09a6d9b375..67315165a9 100644 --- a/plugins/techdocs-node/src/stages/publish/publish.ts +++ b/plugins/techdocs-node/src/stages/publish/publish.ts @@ -86,7 +86,11 @@ export class Publisher implements PublisherBuilder { logger.info('Creating Google Storage Bucket publisher for TechDocs'); publishers.register( publisherType, - GoogleGCSPublish.fromConfig(config, logger), + GoogleGCSPublish.fromConfig( + config, + logger, + options.publisherSettings?.googleGcs, + ), ); break; case 'awsS3': diff --git a/plugins/techdocs-node/src/stages/publish/types.ts b/plugins/techdocs-node/src/stages/publish/types.ts index 720f1b218a..b5b65c8e08 100644 --- a/plugins/techdocs-node/src/stages/publish/types.ts +++ b/plugins/techdocs-node/src/stages/publish/types.ts @@ -17,6 +17,7 @@ import express from 'express'; import { Config } from '@backstage/config'; import { DiscoveryService, LoggerService } from '@backstage/backend-plugin-api'; import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; +import { StorageOptions } from '@google-cloud/storage'; /** * Options for building publishers @@ -26,8 +27,17 @@ export type PublisherFactory = { logger: LoggerService; discovery: DiscoveryService; customPublisher?: PublisherBase | undefined; + publisherSettings?: PublisherSettings; }; +/** + * Additional configurations for publishers. + * @public + */ +export interface PublisherSettings { + googleGcs?: StorageOptions; +} + /** * Key for all the different types of TechDocs publishers that are supported. * @public