feat: add AwsS3EntityProvider as replacement for AwsS3DiscoveryProcessor

Add a new provider `AwsS3EntityProvider` as a replacement for the now deprecated
`AwsS3DiscoveryProcessor`.

The new provider will scan configured S3 buckets (with optional) prefix and
add `Location` entities for all discovered catalog files.

These `Location` entities will then be processed as usual.

At each execution, the provider will apply a full mutation, replacing all previous
entities with the new entities/state.

Relates-to: #10183
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-03-25 22:47:51 +01:00
parent db21d78179
commit 5969c4b65c
13 changed files with 858 additions and 13 deletions
@@ -7,8 +7,11 @@ import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
import { CatalogProcessorParser } from '@backstage/plugin-catalog-backend';
import { Config } from '@backstage/config';
import { EntityProvider } from '@backstage/plugin-catalog-backend';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import { TaskRunner } from '@backstage/backend-tasks';
import { UrlReader } from '@backstage/backend-common';
// @public
@@ -43,4 +46,22 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor {
parser: CatalogProcessorParser,
): Promise<boolean>;
}
// @public
export class AwsS3EntityProvider implements EntityProvider {
// (undocumented)
connect(connection: EntityProviderConnection): Promise<void>;
// (undocumented)
static fromConfig(
configRoot: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): AwsS3EntityProvider[];
// (undocumented)
getProviderName(): string;
// (undocumented)
refresh(logger: Logger): Promise<void>;
}
```
+32
View File
@@ -14,6 +14,27 @@
* limitations under the License.
*/
interface AwsS3Config {
/**
* (Required) AWS S3 Bucket Name
* @visibility backend
*/
bucketName: string;
/**
* (Optional) AWS S3 Object key prefix
* If not set, all keys will be accepted, no filtering will be applied.
* @visibility backend
*/
prefix?: string;
/**
* (Optional) AWS Region.
* If not set, AWS_REGION environment variable or aws config file will be used.
* @see https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
* @visibility backend
*/
region?: string;
}
export interface Config {
catalog?: {
/**
@@ -32,5 +53,16 @@ export interface Config {
};
};
};
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* AwsS3EntityProvider configuration
*
* Uses "default" as default id for the single config variant.
*/
awsS3?: AwsS3Config | Record<string, AwsS3Config>;
};
};
}
@@ -34,14 +34,17 @@
},
"dependencies": {
"@backstage/backend-common": "^0.13.2-next.1",
"@backstage/backend-tasks": "^0.3.0-next.1",
"@backstage/catalog-model": "^1.0.1-next.0",
"@backstage/config": "^1.0.0",
"@backstage/errors": "^1.0.0",
"@backstage/integration": "^1.1.0-next.1",
"@backstage/plugin-catalog-backend": "^1.1.0-next.1",
"@backstage/types": "^1.0.0",
"aws-sdk": "^2.840.0",
"lodash": "^4.17.21",
"p-limit": "^3.0.2",
"uuid": "^8.0.0",
"winston": "^3.2.1"
},
"devDependencies": {
@@ -0,0 +1,61 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import aws, { Credentials } from 'aws-sdk';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
export class AwsCredentials {
/**
* If accessKeyId and secretAccessKey are missing, the DefaultAWSCredentialsProviderChain will be used:
* https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
*/
static create(
config: {
accessKeyId?: string;
secretAccessKey?: string;
roleArn?: string;
},
roleSessionName: string,
): Credentials | CredentialsOptions | undefined {
if (!config) {
return undefined;
}
const accessKeyId = config.accessKeyId;
const secretAccessKey = config.secretAccessKey;
let explicitCredentials: Credentials | undefined;
if (accessKeyId && secretAccessKey) {
explicitCredentials = new Credentials({
accessKeyId,
secretAccessKey,
});
}
const roleArn = config.roleArn;
if (roleArn) {
return new aws.ChainableTemporaryCredentials({
masterCredentials: explicitCredentials,
params: {
RoleArn: roleArn,
RoleSessionName: roleSessionName,
},
});
}
return explicitCredentials;
}
}
@@ -21,3 +21,4 @@
*/
export * from './processors';
export * from './providers';
@@ -0,0 +1,196 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { AwsS3EntityProvider } from './AwsS3EntityProvider';
import aws from 'aws-sdk';
import AWSMock from 'aws-sdk-mock';
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
getTasks() {
return this.tasks;
}
run(task: TaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
const logger = getVoidLogger();
describe('AwsS3EntityProvider', () => {
const config = new ConfigReader({
catalog: {
providers: {
awsS3: {
anyProviderId: {
bucketName: 'bucket-1',
region: 'us-east-1',
prefix: 'sub/dir/',
},
},
},
},
});
const schedule = new PersistingTaskRunner();
AWSMock.setSDKInstance(aws);
const createObjectList = (...keys: string[]): aws.S3.ObjectList => {
const objects = keys.map(key => {
return {
Key: key,
} as aws.S3.Types.Object;
});
return objects as aws.S3.ObjectList;
};
AWSMock.mock('S3', 'listObjectsV2', async req => {
const prefix = req.Prefix ?? '';
if (!req.ContinuationToken) {
return {
Contents: createObjectList(`${prefix}key1.yaml`, `${prefix}key2.yaml`),
NextContinuationToken: 'next-token',
} as aws.S3.Types.ListObjectsV2Output;
}
return {
Contents: createObjectList(`${prefix}key3.yaml`, `${prefix}key4.yaml`),
} as aws.S3.Types.ListObjectsV2Output;
});
afterEach(() => jest.resetAllMocks());
it('apply full update on scheduled execution', async () => {
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = AwsS3EntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
expect(provider.getProviderName()).toEqual('awsS3-provider:anyProviderId');
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('awsS3-provider:anyProviderId:refresh');
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toBeCalledWith({
type: 'full',
entities: [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml',
'backstage.io/managed-by-origin-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml',
},
name: 'generated-980e6ad47fbfbfeead708a9c7c87331b7540296a',
},
spec: {
presence: 'required',
target:
'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key1.yaml',
type: 'url',
},
},
locationKey: 'awsS3-provider:anyProviderId',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml',
'backstage.io/managed-by-origin-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml',
},
name: 'generated-266794d8e789089dddba2b42cd79e70b149aa61c',
},
spec: {
presence: 'required',
target:
'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key2.yaml',
type: 'url',
},
},
locationKey: 'awsS3-provider:anyProviderId',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml',
'backstage.io/managed-by-origin-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml',
},
name: 'generated-96f0cdcd7e33aa687c19d160ec7d5b1975cb9ea1',
},
spec: {
presence: 'required',
target:
'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key3.yaml',
type: 'url',
},
},
locationKey: 'awsS3-provider:anyProviderId',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml',
'backstage.io/managed-by-origin-location':
'url:https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml',
},
name: 'generated-cd1a799b5ecfc055a0c672654420af3afeb648d3',
},
spec: {
presence: 'required',
target:
'https://bucket-1.s3.us-east-1.amazonaws.com/sub/dir/key4.yaml',
type: 'url',
},
},
locationKey: 'awsS3-provider:anyProviderId',
},
],
});
});
});
@@ -0,0 +1,208 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
import {
EntityProvider,
EntityProviderConnection,
LocationSpec,
locationSpecToLocationEntity,
} from '@backstage/plugin-catalog-backend';
import { AwsCredentials } from '../credentials/AwsCredentials';
import { readAwsS3Configs } from './config';
import { AwsS3Config } from './types';
import { S3 } from 'aws-sdk';
import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
import * as uuid from 'uuid';
import { Logger } from 'winston';
// TODO: event-based updates using S3 events (+ queue like SQS)?
/**
* Provider which discovers catalog files (any name) within an S3 bucket.
*
* Use `AwsS3EntityProvider.fromConfig(...)` to create instances.
*
* @public
*/
export class AwsS3EntityProvider implements EntityProvider {
private readonly logger: Logger;
private readonly s3: S3;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
static fromConfig(
configRoot: Config,
options: {
logger: Logger;
schedule: TaskRunner;
},
): AwsS3EntityProvider[] {
const providerConfigs = readAwsS3Configs(configRoot);
// Even though the awsS3 integration allows a config array
// there is no *real* support for multiple configs.
// Usually, there will be just the integration for the default host.
// In case, a config custom endpoint is used, the host from this endpoint
// will be extracted and used as host (e.g., localhost when used with LocalStack)
// and the default integration will be added as second integration.
// In this case, we still want the first one though, but have no means to select it
// just from the bucket name (and region).
const integration = ScmIntegrations.fromConfig(configRoot).awsS3.list()[0];
return providerConfigs.map(
providerConfig =>
new AwsS3EntityProvider(
providerConfig,
integration,
options.logger,
options.schedule,
),
);
}
private constructor(
private readonly config: AwsS3Config,
private readonly integration: AwsS3Integration,
logger: Logger,
schedule: TaskRunner,
) {
this.logger = logger.child({
target: this.getProviderName(),
});
this.s3 = new S3({
apiVersion: '2006-03-01',
credentials: AwsCredentials.create(
integration.config,
'backstage-aws-s3-provider',
),
endpoint: integration.config.endpoint,
region: this.config.region,
s3ForcePathStyle: integration.config.s3ForcePathStyle,
});
this.scheduleFn = this.createScheduleFn(schedule);
}
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
return async () => {
const taskId = `${this.getProviderName()}:refresh`;
return schedule.run({
id: taskId,
fn: async () => {
const logger = this.logger.child({
class: AwsS3EntityProvider.prototype.constructor.name,
taskId,
taskInstanceId: uuid.v4(),
});
try {
await this.refresh(logger);
} catch (error) {
logger.error(error);
}
},
});
};
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
getProviderName(): string {
return `awsS3-provider:${this.config.id}`;
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
await this.scheduleFn();
}
async refresh(logger: Logger) {
if (!this.connection) {
throw new Error('Not initialized');
}
logger.info('Discovering AWS S3 objects');
const keys = await this.listAllObjectKeys();
logger.info(`Discovered ${keys.length} AWS S3 objects`);
const locations = keys.map(key => this.createLocationSpec(key));
await this.connection.applyMutation({
type: 'full',
entities: locations.map(location => {
return {
locationKey: this.getProviderName(),
entity: locationSpecToLocationEntity({ location }),
};
}),
});
logger.info(`Committed ${locations.length} Locations for AWS S3 objects`);
}
private async listAllObjectKeys(): Promise<string[]> {
const keys: string[] = [];
let continuationToken: string | undefined = undefined;
let output: ListObjectsV2Output;
do {
const request = this.s3.listObjectsV2({
Bucket: this.config.bucketName,
ContinuationToken: continuationToken,
Prefix: this.config.prefix,
});
output = await request.promise();
if (output.Contents) {
output.Contents.forEach(item => {
if (item.Key && !item.Key.endsWith('/')) {
keys.push(item.Key);
}
});
}
continuationToken = output.NextContinuationToken;
} while (continuationToken);
return keys;
}
private createLocationSpec(key: string): LocationSpec {
return {
type: 'url',
target: this.createObjectUrl(key),
presence: 'required',
};
}
private createObjectUrl(key: string): string {
const bucketName = this.config.bucketName;
const endpoint = this.integration.config.endpoint;
if (endpoint) {
if (endpoint.startsWith(`https://${bucketName}.`)) {
return `${endpoint}/${key}`;
}
return `${endpoint}/${bucketName}/${key}`;
}
return `https://${bucketName}.s3.${this.config.region}.amazonaws.com/${key}`;
}
}
@@ -0,0 +1,98 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { readAwsS3Configs } from './config';
describe('readAwsS3Configs', () => {
it('reads single provider config', () => {
const provider = {
bucketName: 'bucket-1',
region: 'us-east-1',
prefix: 'sub/dir/',
};
const config = {
catalog: {
providers: {
awsS3: provider,
},
},
};
const actual = readAwsS3Configs(new ConfigReader(config));
expect(actual).toHaveLength(1);
expect(actual[0]).toEqual({
...provider,
id: 'default',
});
});
it('reads all provider configs', () => {
const provider1 = {
bucketName: 'bucket-1',
region: 'us-east-1',
prefix: 'sub/dir/',
};
const provider2 = {
bucketName: 'bucket-2',
region: 'eu-west-1',
};
const provider3 = {
bucketName: 'bucket-3',
};
const config = {
catalog: {
providers: {
awsS3: { provider1, provider2, provider3 },
},
},
};
const actual = readAwsS3Configs(new ConfigReader(config));
expect(actual).toHaveLength(3);
expect(actual[0]).toEqual({
...provider1,
id: 'provider1',
});
expect(actual[1]).toEqual({
...provider2,
id: 'provider2',
});
expect(actual[2]).toEqual({
...provider3,
id: 'provider3',
});
});
it('fails if bucketName is missing', () => {
const provider = {
region: 'us-east-1',
};
const config = {
catalog: {
providers: {
awsS3: { provider },
},
},
};
expect(() => readAwsS3Configs(new ConfigReader(config))).toThrow(
"Missing required config value at 'catalog.providers.awsS3.provider.bucketName'",
);
});
});
@@ -0,0 +1,55 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { AwsS3Config } from './types';
const DEFAULT_PROVIDER_ID = 'default';
export function readAwsS3Configs(config: Config): AwsS3Config[] {
const configs: AwsS3Config[] = [];
const providerConfigs = config.getOptionalConfig('catalog.providers.awsS3');
if (!providerConfigs) {
return configs;
}
if (providerConfigs.has('bucketName')) {
// simple/single config variant
configs.push(readAwsS3Config(DEFAULT_PROVIDER_ID, providerConfigs));
return configs;
}
for (const id of providerConfigs.keys()) {
configs.push(readAwsS3Config(id, providerConfigs.getConfig(id)));
}
return configs;
}
function readAwsS3Config(id: string, config: Config): AwsS3Config {
const bucketName = config.getString('bucketName');
const region = config.getOptionalString('region');
const prefix = config.getOptionalString('prefix');
return {
id,
bucketName,
region,
prefix,
};
}
@@ -0,0 +1,17 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { AwsS3EntityProvider } from './AwsS3EntityProvider';
@@ -0,0 +1,22 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type AwsS3Config = {
id: string;
bucketName: string;
prefix?: string;
region?: string;
};