diff --git a/.changeset/friendly-hairs-happen.md b/.changeset/friendly-hairs-happen.md new file mode 100644 index 0000000000..219d5efbcd --- /dev/null +++ b/.changeset/friendly-hairs-happen.md @@ -0,0 +1,76 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +Add a new provider `AwsS3EntityProvider` as replacement for `AwsS3DiscoveryProcessor`. + +In order to migrate from the `AwsS3DiscoveryProcessor` you need to apply +the following changes: + +**Before:** + +```yaml +# app-config.yaml + +catalog: + locations: + - type: s3-discovery + target: https://sample-bucket.s3.us-east-2.amazonaws.com/prefix/ +``` + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-aws'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors ... */ +builder.addProcessor(new AwsS3DiscoveryProcessor(env.reader)); +``` + +**After:** + +```yaml +# app-config.yaml + +catalog: + providers: + awsS3: + yourProviderId: # identifies your dataset / provider independent of config changes + bucketName: sample-bucket + prefix: prefix/ # optional + region: us-east-2 # optional, uses the default region otherwise +``` + +```ts +/* packages/backend/src/plugins/catalog.ts */ + +import { AwsS3EntityProvider } from '@backstage/plugin-catalog-backend-module-aws'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +builder.addEntityProvider( + ...AwsS3EntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: Duration.fromObject({ minutes: 30 }), + timeout: Duration.fromObject({ minutes: 3 }), + }), + }), +); +``` + +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. + +```yaml +# app-config.yaml + +catalog: + providers: + awsS3: + # uses "default" as provider ID + bucketName: sample-bucket + prefix: prefix/ # optional + region: us-east-2 # optional, uses the default region otherwise +``` diff --git a/docs/integrations/aws-s3/discovery.md b/docs/integrations/aws-s3/discovery.md index 2c25aaaf05..316b5c5014 100644 --- a/docs/integrations/aws-s3/discovery.md +++ b/docs/integrations/aws-s3/discovery.md @@ -6,32 +6,55 @@ sidebar_label: Discovery description: Automatically discovering catalog entities from an AWS S3 Bucket --- -The AWS S3 integration has a special discovery processor for discovering catalog +The AWS S3 integration has a special entity provider for discovering catalog entities located in an S3 Bucket. If you have a bucket that contains multiple -catalog-info files and want to automatically discover them, you can use this -processor. The processor will crawl your S3 bucket and register entities +catalog files, and you want to automatically discover them, you can use this +provider. The provider will crawl your S3 bucket and register entities matching the configured path. This can be useful as an alternative to static locations or manually adding things to the catalog. -To use the discovery processor, you'll need an AWS S3 integration -[set up](locations.md) with an `AWS_ACCESS_KEY`, `AWS_SECRET_ACCESS_KEY`, and -optionally a `roleArn`. Then you can add a location target to the catalog -configuration: +To use the entity provider, you'll need an AWS S3 integration +[set up](locations.md) with `accessKeyId` and `secretAccessKey`, and/or +a `roleArn` or none of these (e.g., profile- or instance-based credentials). + +At production deployments, you likely manage these with the permissions attached +to your instance. + +At your configuration, you add a provider config per bucket: ```yaml +# app-config.yaml + catalog: - locations: - - type: s3-discovery - target: https://sample-bucket.s3.us-east-2.amazonaws.com/ + providers: + awsS3: + yourProviderId: # identifies your dataset / provider independent of config changes + bucketName: sample-bucket + prefix: prefix/ # optional + region: us-east-2 # optional, uses the default region otherwise ``` -Note the `s3-discovery` type, as this is not a regular `url` processor. +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. -As this processor is not one of the default providers, you will first need to install the AWS catalog plugin: +```yaml +# app-config.yaml + +catalog: + providers: + awsS3: + # uses "default" as provider ID + bucketName: sample-bucket + prefix: prefix/ # optional + region: us-east-2 # optional, uses the default region otherwise +``` + +As this provider is not one of the default providers, you will first need to install +the AWS catalog plugin: ```bash # From the Backstage root directory -yarn install --cwd packages/backend @backstage/plugin-catalog-backend-module-aws +yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-aws ``` Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: @@ -39,6 +62,38 @@ Once you've done that, you'll also need to add the segment below to `packages/ba ```ts /* packages/backend/src/plugins/catalog.ts */ +import { AwsS3EntityProvider } from '@backstage/plugin-catalog-backend-module-aws'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +builder.addEntityProvider( + ...AwsS3EntityProvider.fromConfig(env.config, { + logger: env.logger, + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: Duration.fromObject({ minutes: 30 }), + timeout: Duration.fromObject({ minutes: 3 }), + }), + }), +); +``` + +## Alternative Processor + +As alternative to the entity provider `AwsS3EntityProvider` +you can still use the `AwsS3DiscoveryProcessor`. + +```yaml +# app-config.yaml + +catalog: + locations: + - type: s3-discovery + target: https://sample-bucket.s3.us-east-2.amazonaws.com/prefix/ +``` + +```ts +/* packages/backend/src/plugins/catalog.ts */ + import { AwsS3DiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-aws'; const builder = await CatalogBuilder.create(env); diff --git a/plugins/catalog-backend-module-aws/api-report.md b/plugins/catalog-backend-module-aws/api-report.md index 876562fc8d..c2c18431b9 100644 --- a/plugins/catalog-backend-module-aws/api-report.md +++ b/plugins/catalog-backend-module-aws/api-report.md @@ -7,8 +7,11 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend'; import { CatalogProcessorParser } from '@backstage/plugin-catalog-backend'; import { Config } from '@backstage/config'; +import { EntityProvider } from '@backstage/plugin-catalog-backend'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; +import { TaskRunner } from '@backstage/backend-tasks'; import { UrlReader } from '@backstage/backend-common'; // @public @@ -43,4 +46,22 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { parser: CatalogProcessorParser, ): Promise; } + +// @public +export class AwsS3EntityProvider implements EntityProvider { + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + configRoot: Config, + options: { + logger: Logger; + schedule: TaskRunner; + }, + ): AwsS3EntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(logger: Logger): Promise; +} ``` diff --git a/plugins/catalog-backend-module-aws/config.d.ts b/plugins/catalog-backend-module-aws/config.d.ts index bb416a576e..2e41d60aac 100644 --- a/plugins/catalog-backend-module-aws/config.d.ts +++ b/plugins/catalog-backend-module-aws/config.d.ts @@ -14,6 +14,27 @@ * limitations under the License. */ +interface AwsS3Config { + /** + * (Required) AWS S3 Bucket Name + * @visibility backend + */ + bucketName: string; + /** + * (Optional) AWS S3 Object key prefix + * If not set, all keys will be accepted, no filtering will be applied. + * @visibility backend + */ + prefix?: string; + /** + * (Optional) AWS Region. + * If not set, AWS_REGION environment variable or aws config file will be used. + * @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html + * @visibility backend + */ + region?: string; +} + export interface Config { catalog?: { /** @@ -32,5 +53,16 @@ export interface Config { }; }; }; + /** + * List of provider-specific options and attributes + */ + providers?: { + /** + * AwsS3EntityProvider configuration + * + * Uses "default" as default id for the single config variant. + */ + awsS3?: AwsS3Config | Record; + }; }; } diff --git a/plugins/catalog-backend-module-aws/package.json b/plugins/catalog-backend-module-aws/package.json index 23b5055e6a..a115efeacb 100644 --- a/plugins/catalog-backend-module-aws/package.json +++ b/plugins/catalog-backend-module-aws/package.json @@ -34,14 +34,17 @@ }, "dependencies": { "@backstage/backend-common": "^0.13.2-next.1", + "@backstage/backend-tasks": "^0.3.0-next.1", "@backstage/catalog-model": "^1.0.1-next.0", "@backstage/config": "^1.0.0", "@backstage/errors": "^1.0.0", + "@backstage/integration": "^1.1.0-next.1", "@backstage/plugin-catalog-backend": "^1.1.0-next.1", "@backstage/types": "^1.0.0", "aws-sdk": "^2.840.0", "lodash": "^4.17.21", "p-limit": "^3.0.2", + "uuid": "^8.0.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts b/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts new file mode 100644 index 0000000000..ffb84d1c34 --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import aws, { Credentials } from 'aws-sdk'; +import { CredentialsOptions } from 'aws-sdk/lib/credentials'; + +export class AwsCredentials { + /** + * If accessKeyId and secretAccessKey are missing, the DefaultAWSCredentialsProviderChain will be used: + * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html + */ + static create( + config: { + accessKeyId?: string; + secretAccessKey?: string; + roleArn?: string; + }, + roleSessionName: string, + ): Credentials | CredentialsOptions | undefined { + if (!config) { + return undefined; + } + + const accessKeyId = config.accessKeyId; + const secretAccessKey = config.secretAccessKey; + let explicitCredentials: Credentials | undefined; + + if (accessKeyId && secretAccessKey) { + explicitCredentials = new Credentials({ + accessKeyId, + secretAccessKey, + }); + } + + const roleArn = config.roleArn; + if (roleArn) { + return new aws.ChainableTemporaryCredentials({ + masterCredentials: explicitCredentials, + params: { + RoleArn: roleArn, + RoleSessionName: roleSessionName, + }, + }); + } + + return explicitCredentials; + } +} diff --git a/plugins/catalog-backend-module-aws/src/index.ts b/plugins/catalog-backend-module-aws/src/index.ts index 62c10a0f6d..1d8e844895 100644 --- a/plugins/catalog-backend-module-aws/src/index.ts +++ b/plugins/catalog-backend-module-aws/src/index.ts @@ -21,3 +21,4 @@ */ export * from './processors'; +export * from './providers'; diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts new file mode 100644 index 0000000000..5e5901a770 --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.test.ts @@ -0,0 +1,196 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { AwsS3EntityProvider } from './AwsS3EntityProvider'; +import aws from 'aws-sdk'; +import AWSMock from 'aws-sdk-mock'; + +class PersistingTaskRunner implements TaskRunner { + private tasks: TaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: TaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const logger = getVoidLogger(); + +describe('AwsS3EntityProvider', () => { + const config = new ConfigReader({ + catalog: { + providers: { + awsS3: { + anyProviderId: { + bucketName: 'bucket-1', + region: 'us-east-1', + prefix: 'sub/dir/', + }, + }, + }, + }, + }); + + const schedule = new PersistingTaskRunner(); + + AWSMock.setSDKInstance(aws); + const createObjectList = (...keys: string[]): aws.S3.ObjectList => { + const objects = keys.map(key => { + return { + Key: key, + } as aws.S3.Types.Object; + }); + + return objects as aws.S3.ObjectList; + }; + + AWSMock.mock('S3', 'listObjectsV2', async req => { + const prefix = req.Prefix ?? ''; + + if (!req.ContinuationToken) { + return { + Contents: createObjectList(`${prefix}key1.yaml`, `${prefix}key2.yaml`), + NextContinuationToken: 'next-token', + } as aws.S3.Types.ListObjectsV2Output; + } + + return { + Contents: createObjectList(`${prefix}key3.yaml`, `${prefix}key4.yaml`), + } as aws.S3.Types.ListObjectsV2Output; + }); + + afterEach(() => jest.resetAllMocks()); + + it('apply full update on scheduled execution', async () => { + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + }; + + const provider = AwsS3EntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + expect(provider.getProviderName()).toEqual('awsS3-provider:anyProviderId'); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('awsS3-provider:anyProviderId:refresh'); + await (taskDef.fn as () => Promise)(); + + expect(entityProviderConnection.applyMutation).toBeCalledWith({ + type: 'full', + entities: [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml', + }, + name: 'generated-980e6ad47fbfbfeead708a9c7c87331b7540296a', + }, + spec: { + presence: 'required', + target: + 'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml', + type: 'url', + }, + }, + locationKey: 'awsS3-provider:anyProviderId', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml', + }, + name: 'generated-266794d8e789089dddba2b42cd79e70b149aa61c', + }, + spec: { + presence: 'required', + target: + 'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml', + type: 'url', + }, + }, + locationKey: 'awsS3-provider:anyProviderId', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml', + }, + name: 'generated-96f0cdcd7e33aa687c19d160ec7d5b1975cb9ea1', + }, + spec: { + presence: 'required', + target: + 'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml', + type: 'url', + }, + }, + locationKey: 'awsS3-provider:anyProviderId', + }, + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml', + 'backstage.io/managed-by-origin-location': + 'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml', + }, + name: 'generated-cd1a799b5ecfc055a0c672654420af3afeb648d3', + }, + spec: { + presence: 'required', + target: + 'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml', + type: 'url', + }, + }, + locationKey: 'awsS3-provider:anyProviderId', + }, + ], + }); + }); +}); diff --git a/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts new file mode 100644 index 0000000000..f068e40130 --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/AwsS3EntityProvider.ts @@ -0,0 +1,208 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TaskRunner } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { AwsS3Integration, ScmIntegrations } from '@backstage/integration'; +import { + EntityProvider, + EntityProviderConnection, + LocationSpec, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-backend'; +import { AwsCredentials } from '../credentials/AwsCredentials'; +import { readAwsS3Configs } from './config'; +import { AwsS3Config } from './types'; +import { S3 } from 'aws-sdk'; +import { ListObjectsV2Output } from 'aws-sdk/clients/s3'; +import * as uuid from 'uuid'; +import { Logger } from 'winston'; + +// TODO: event-based updates using S3 events (+ queue like SQS)? +/** + * Provider which discovers catalog files (any name) within an S3 bucket. + * + * Use `AwsS3EntityProvider.fromConfig(...)` to create instances. + * + * @public + */ +export class AwsS3EntityProvider implements EntityProvider { + private readonly logger: Logger; + private readonly s3: S3; + private readonly scheduleFn: () => Promise; + private connection?: EntityProviderConnection; + + static fromConfig( + configRoot: Config, + options: { + logger: Logger; + schedule: TaskRunner; + }, + ): AwsS3EntityProvider[] { + const providerConfigs = readAwsS3Configs(configRoot); + + // Even though the awsS3 integration allows a config array + // there is no *real* support for multiple configs. + // Usually, there will be just the integration for the default host. + // In case, a config custom endpoint is used, the host from this endpoint + // will be extracted and used as host (e.g., localhost when used with LocalStack) + // and the default integration will be added as second integration. + // In this case, we still want the first one though, but have no means to select it + // just from the bucket name (and region). + const integration = ScmIntegrations.fromConfig(configRoot).awsS3.list()[0]; + + return providerConfigs.map( + providerConfig => + new AwsS3EntityProvider( + providerConfig, + integration, + options.logger, + options.schedule, + ), + ); + } + + private constructor( + private readonly config: AwsS3Config, + private readonly integration: AwsS3Integration, + logger: Logger, + schedule: TaskRunner, + ) { + this.logger = logger.child({ + target: this.getProviderName(), + }); + + this.s3 = new S3({ + apiVersion: '2006-03-01', + credentials: AwsCredentials.create( + integration.config, + 'backstage-aws-s3-provider', + ), + endpoint: integration.config.endpoint, + region: this.config.region, + s3ForcePathStyle: integration.config.s3ForcePathStyle, + }); + + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn(schedule: TaskRunner): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return schedule.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: AwsS3EntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */ + getProviderName(): string { + return `awsS3-provider:${this.config.id}`; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + await this.scheduleFn(); + } + + async refresh(logger: Logger) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering AWS S3 objects'); + + const keys = await this.listAllObjectKeys(); + logger.info(`Discovered ${keys.length} AWS S3 objects`); + + const locations = keys.map(key => this.createLocationSpec(key)); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }), + }); + + logger.info(`Committed ${locations.length} Locations for AWS S3 objects`); + } + + private async listAllObjectKeys(): Promise { + const keys: string[] = []; + + let continuationToken: string | undefined = undefined; + let output: ListObjectsV2Output; + do { + const request = this.s3.listObjectsV2({ + Bucket: this.config.bucketName, + ContinuationToken: continuationToken, + Prefix: this.config.prefix, + }); + + output = await request.promise(); + if (output.Contents) { + output.Contents.forEach(item => { + if (item.Key && !item.Key.endsWith('/')) { + keys.push(item.Key); + } + }); + } + continuationToken = output.NextContinuationToken; + } while (continuationToken); + + return keys; + } + + private createLocationSpec(key: string): LocationSpec { + return { + type: 'url', + target: this.createObjectUrl(key), + presence: 'required', + }; + } + + private createObjectUrl(key: string): string { + const bucketName = this.config.bucketName; + const endpoint = this.integration.config.endpoint; + + if (endpoint) { + if (endpoint.startsWith(`https://${bucketName}.`)) { + return `${endpoint}/${key}`; + } + + return `${endpoint}/${bucketName}/${key}`; + } + + return `https://${bucketName}.s3.${this.config.region}.amazonaws.com/${key}`; + } +} diff --git a/plugins/catalog-backend-module-aws/src/providers/config.test.ts b/plugins/catalog-backend-module-aws/src/providers/config.test.ts new file mode 100644 index 0000000000..534d4da35e --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/config.test.ts @@ -0,0 +1,98 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { readAwsS3Configs } from './config'; + +describe('readAwsS3Configs', () => { + it('reads single provider config', () => { + const provider = { + bucketName: 'bucket-1', + region: 'us-east-1', + prefix: 'sub/dir/', + }; + const config = { + catalog: { + providers: { + awsS3: provider, + }, + }, + }; + + const actual = readAwsS3Configs(new ConfigReader(config)); + + expect(actual).toHaveLength(1); + expect(actual[0]).toEqual({ + ...provider, + id: 'default', + }); + }); + + it('reads all provider configs', () => { + const provider1 = { + bucketName: 'bucket-1', + region: 'us-east-1', + prefix: 'sub/dir/', + }; + const provider2 = { + bucketName: 'bucket-2', + region: 'eu-west-1', + }; + const provider3 = { + bucketName: 'bucket-3', + }; + const config = { + catalog: { + providers: { + awsS3: { provider1, provider2, provider3 }, + }, + }, + }; + + const actual = readAwsS3Configs(new ConfigReader(config)); + + expect(actual).toHaveLength(3); + expect(actual[0]).toEqual({ + ...provider1, + id: 'provider1', + }); + expect(actual[1]).toEqual({ + ...provider2, + id: 'provider2', + }); + expect(actual[2]).toEqual({ + ...provider3, + id: 'provider3', + }); + }); + + it('fails if bucketName is missing', () => { + const provider = { + region: 'us-east-1', + }; + const config = { + catalog: { + providers: { + awsS3: { provider }, + }, + }, + }; + + expect(() => readAwsS3Configs(new ConfigReader(config))).toThrow( + "Missing required config value at 'catalog.providers.awsS3.provider.bucketName'", + ); + }); +}); diff --git a/plugins/catalog-backend-module-aws/src/providers/config.ts b/plugins/catalog-backend-module-aws/src/providers/config.ts new file mode 100644 index 0000000000..5fdf2351c3 --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/config.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { AwsS3Config } from './types'; + +const DEFAULT_PROVIDER_ID = 'default'; + +export function readAwsS3Configs(config: Config): AwsS3Config[] { + const configs: AwsS3Config[] = []; + + const providerConfigs = config.getOptionalConfig('catalog.providers.awsS3'); + if (!providerConfigs) { + return configs; + } + + if (providerConfigs.has('bucketName')) { + // simple/single config variant + configs.push(readAwsS3Config(DEFAULT_PROVIDER_ID, providerConfigs)); + + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readAwsS3Config(id, providerConfigs.getConfig(id))); + } + + return configs; +} + +function readAwsS3Config(id: string, config: Config): AwsS3Config { + const bucketName = config.getString('bucketName'); + const region = config.getOptionalString('region'); + const prefix = config.getOptionalString('prefix'); + + return { + id, + bucketName, + region, + prefix, + }; +} diff --git a/plugins/catalog-backend-module-aws/src/providers/index.ts b/plugins/catalog-backend-module-aws/src/providers/index.ts new file mode 100644 index 0000000000..7bc8b4f265 --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { AwsS3EntityProvider } from './AwsS3EntityProvider'; diff --git a/plugins/catalog-backend-module-aws/src/providers/types.ts b/plugins/catalog-backend-module-aws/src/providers/types.ts new file mode 100644 index 0000000000..3508bf919c --- /dev/null +++ b/plugins/catalog-backend-module-aws/src/providers/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type AwsS3Config = { + id: string; + bucketName: string; + prefix?: string; + region?: string; +};