paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 49 Packages Projects Releases Wiki Activity

Make change to AwsS3 Integration to support list

Browse Source 
Signed-off-by: Sean Tan <seant@splunk.com>
This commit is contained in:
Sean Tan
2021-07-27 15:21:54 -07:00
parent c0e5f6d6dd
commit 2e5e2decc7
13 changed files with 293 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+4 -3
View File
@@ -163,9 +163,10 @@ integrations:
# googleGcs:
# clientEmail: 'example@example.com'
# privateKey: ${GCS_PRIVATE_KEY}
# awsS3:
# accessKeyId: ${AWS_ACCESS_KEY_ID}
# secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
awsS3:
- host: amazonaws.com
accessKeyId: ${AWS_ACCESS_KEY_ID}
secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
catalog:
rules:
docs/integrations/aws-s3/locations.md
+3 -3
View File
@@ -18,9 +18,9 @@ plugin.
To use this integration, add configuration to your root `app-config.yaml`:
```yaml
integrations:
awsS3:
accessKeyID: ${AWS_ACCESS_KEY_ID}
awsS3:
- host: amazonaws.com
accessKeyId: ${AWS_ACCESS_KEY_ID}
secretAccessKey: ${AWS_SECRET_ACCESS_KEY}
```
packages/backend-common/src/reading/AwsS3UrlReader.test.ts
+17 -10
View File
@@ -30,38 +30,45 @@ describe('AwsS3UrlReader', () => {
});
};
it('does not create a reader without the awsS3 field', () => {
it('creates a dummy reader without the awsS3 field', () => {
const entries = createReader({
integrations: {},
});
expect(entries).toHaveLength(0);
expect(entries).toHaveLength(1);
});
it('creates a reader with credentials correctly configured', () => {
const awsS3Integrations = [];
awsS3Integrations.push({
host: 'amazonaws.com',
accessKeyId: 'fakekey',
secretAccessKey: 'fakekey',
});
const entries = createReader({
integrations: {
awsS3: {
accessKeyId: 'fakekey',
secretAccessKey: 'fakekey',
},
awsS3: awsS3Integrations,
},
});
expect(entries).toHaveLength(1);
expect(entries).toHaveLength(2);
});
it('creates a reader with default credentials provider', () => {
const awsS3Integrations = [];
awsS3Integrations.push({
host: 'amazonaws.com',
});
const entries = createReader({
integrations: {
awsS3: {},
awsS3: awsS3Integrations,
},
});
expect(entries).toHaveLength(1);
expect(entries).toHaveLength(2);
});
describe('predicates', () => {
const readers = createReader({
integrations: {
awsS3: {},
awsS3: [{}],
},
});
const predicate = readers[0].predicate;
packages/backend-common/src/reading/AwsS3UrlReader.ts
+33 -34
View File
@@ -24,12 +24,7 @@ import {
UrlReader,
} from './types';
import getRawBody from 'raw-body';
import {
AwsS3IntegrationConfig,
readAwsS3IntegrationConfig,
} from '@backstage/integration';
const AMAZON_AWS_HOST = '.amazonaws.com';
import { AwsS3Integration, ScmIntegrations } from '@backstage/integration';
const parseURL = (
url: string,
@@ -66,35 +61,37 @@ const parseURL = (
export class AwsS3UrlReader implements UrlReader {
static factory: ReaderFactory = ({ config, logger }) => {
if (!config.has('integrations.awsS3')) {
return [];
}
const awsS3Config = readAwsS3IntegrationConfig(
config.getConfig('integrations.awsS3'),
);
let s3: S3;
if (!awsS3Config.accessKeyId || !awsS3Config.secretAccessKey) {
logger.debug(
'integrations.awsS3 not found in app config. AWS S3 integration will use default AWS credentials if set in environment.',
);
s3 = new S3({});
} else {
const creds = new Credentials({
accessKeyId: awsS3Config.accessKeyId,
secretAccessKey: awsS3Config.secretAccessKey,
});
s3 = new S3({
apiVersion: '2006-03-01',
credentials: creds,
});
}
const reader = new AwsS3UrlReader(awsS3Config, s3);
const predicate = (url: URL) => url.host.endsWith(AMAZON_AWS_HOST);
return [{ reader, predicate }];
const integrations = ScmIntegrations.fromConfig(config);
return integrations.awsS3.list().map(integration => {
let s3: S3;
if (
!integration.config.accessKeyId ||
!integration.config.secretAccessKey
) {
logger.info(
'awsS3 credentials not found in config. Using default credentials provider.',
);
s3 = new S3({});
} else {
const creds = new Credentials({
accessKeyId: integration.config.accessKeyId,
secretAccessKey: integration.config.secretAccessKey,
});
s3 = new S3({
apiVersion: '2006-03-01',
credentials: creds,
});
}
const reader = new AwsS3UrlReader(integration, s3);
const predicate = (url: URL) =>
url.host.endsWith(integration.config.host);
return { reader, predicate };
});
};
constructor(
private readonly integration: AwsS3IntegrationConfig,
private readonly integration: AwsS3Integration,
private readonly s3: S3,
) {}
@@ -147,7 +144,9 @@ export class AwsS3UrlReader implements UrlReader {
}
toString() {
const secretAccessKey = this.integration.secretAccessKey;
return `awsS3{host=${AMAZON_AWS_HOST},authed=${Boolean(secretAccessKey)}}`;
const secretAccessKey = this.integration.config.secretAccessKey;
return `awsS3{host=${this.integration.config.host},authed=${Boolean(
secretAccessKey,
)}}`;
}
}
packages/integration/config.d.ts
Vendored
+9 -2
View File
@@ -165,7 +165,14 @@ export interface Config {
};
/** Integration configuration for AWS S3 Service */
awsS3?: {
awsS3?: Array<{
/**
* The host of the target that this matches on, e.g. "amazonaws.com".
*
* @visibility frontend
*/
host: string;
/**
/**
* Account access key used to authenticate requests.
* @visibility backend
@@ -176,6 +183,6 @@ export interface Config {
* @visibility secret
*/
secretAccessKey?: string;
};
}>;
};
}
packages/integration/src/ScmIntegrations.test.ts
+11 -2
View File
@@ -13,7 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AwsS3IntegrationConfig } from './awsS3';
import { AwsS3Integration } from './awsS3/AwsS3Integration';
import { AzureIntegrationConfig } from './azure';
import { AzureIntegration } from './azure/AzureIntegration';
import { BitbucketIntegrationConfig } from './bitbucket';
@@ -26,6 +27,10 @@ import { basicIntegrations } from './helpers';
import { ScmIntegrations } from './ScmIntegrations';
describe('ScmIntegrations', () => {
const awsS3 = new AwsS3Integration({
host: 'awss3.local',
} as AwsS3IntegrationConfig);
const azure = new AzureIntegration({
host: 'azure.local',
} as AzureIntegrationConfig);
@@ -43,6 +48,7 @@ describe('ScmIntegrations', () => {
} as GitLabIntegrationConfig);
const i = new ScmIntegrations({
awsS3: basicIntegrations([awsS3], item => item.config.host),
azure: basicIntegrations([azure], item => item.config.host),
bitbucket: basicIntegrations([bitbucket], item => item.config.host),
github: basicIntegrations([github], item => item.config.host),
@@ -50,6 +56,7 @@ describe('ScmIntegrations', () => {
});
it('can get the specifics', () => {
expect(i.awsS3.byUrl('https://awss3.local')).toBe(awsS3);
expect(i.azure.byUrl('https://azure.local')).toBe(azure);
expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket);
expect(i.github.byUrl('https://github.local')).toBe(github);
@@ -58,16 +65,18 @@ describe('ScmIntegrations', () => {
it('can list', () => {
expect(i.list()).toEqual(
expect.arrayContaining([azure, bitbucket, github, gitlab]),
expect.arrayContaining([awsS3, azure, bitbucket, github, gitlab]),
);
});
it('can select by url and host', () => {
expect(i.byUrl('https://awss3.local')).toBe(awsS3);
expect(i.byUrl('https://azure.local')).toBe(azure);
expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket);
expect(i.byUrl('https://github.local')).toBe(github);
expect(i.byUrl('https://gitlab.local')).toBe(gitlab);
expect(i.byHost('awss3.local')).toBe(awsS3);
expect(i.byHost('azure.local')).toBe(azure);
expect(i.byHost('bitbucket.local')).toBe(bitbucket);
expect(i.byHost('github.local')).toBe(github);
packages/integration/src/ScmIntegrations.ts
+7
View File
@@ -15,6 +15,7 @@
*/
import { Config } from '@backstage/config';
import { AwsS3Integration } from './awsS3/AwsS3Integration';
import { AzureIntegration } from './azure/AzureIntegration';
import { BitbucketIntegration } from './bitbucket/BitbucketIntegration';
import { GitHubIntegration } from './github/GitHubIntegration';
@@ -24,6 +25,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types';
import { ScmIntegrationRegistry } from './registry';
type IntegrationsByType = {
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
azure: ScmIntegrationsGroup<AzureIntegration>;
bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
github: ScmIntegrationsGroup<GitHubIntegration>;
@@ -35,6 +37,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
static fromConfig(config: Config): ScmIntegrations {
return new ScmIntegrations({
awsS3: AwsS3Integration.factory({ config }),
azure: AzureIntegration.factory({ config }),
bitbucket: BitbucketIntegration.factory({ config }),
github: GitHubIntegration.factory({ config }),
@@ -46,6 +49,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
this.byType = integrationsByType;
}
get awsS3(): ScmIntegrationsGroup<AwsS3Integration> {
return this.byType.awsS3;
}
get azure(): ScmIntegrationsGroup<AzureIntegration> {
return this.byType.azure;
}
packages/integration/src/awsS3/AwsS3Integration.test.ts
+72
View File
@@ -0,0 +1,72 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import { AwsS3Integration } from './AwsS3Integration';
describe('AwsS3Integration', () => {
it('has a working factory', () => {
const integrations = AwsS3Integration.factory({
config: new ConfigReader({
integrations: {
awsS3: [
{
host: 'a.com',
accessKeyId: 'access key',
secretAccessKey: 'secret key',
},
],
},
}),
});
expect(integrations.list().length).toBe(2); // including default
expect(integrations.list()[0].config.host).toBe('a.com');
expect(integrations.list()[1].config.host).toBe('.amazonaws.com');
});
it('returns the basics', () => {
const integration = new AwsS3Integration({ host: 'a.com' } as any);
expect(integration.type).toBe('awsS3');
expect(integration.title).toBe('a.com');
});
describe('resolveUrl', () => {
it('works for valid urls', () => {
const integration = new AwsS3Integration({
host: 'amazonaws.com',
} as any);
expect(
integration.resolveUrl({
url: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
base: 'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
}),
).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml');
});
});
it('resolve edit URL', () => {
const integration = new AwsS3Integration({ host: 'a.com' } as any);
// TODO: The Aws S3 integration doesn't support resolving an edit URL,
// instead we keep the input URL.
expect(
integration.resolveEditUrl(
'https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml',
),
).toBe('https://mytest.s3.us-east-2.amazonaws.com/catalog-info.yaml');
});
});
packages/integration/src/awsS3/AwsS3Integration.ts
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { basicIntegrations, defaultScmResolveUrl } from '../helpers';
import { ScmIntegration, ScmIntegrationsFactory } from '../types';
import { AwsS3IntegrationConfig, readAwsS3IntegrationConfigs } from './config';
export class AwsS3Integration implements ScmIntegration {
static factory: ScmIntegrationsFactory<AwsS3Integration> = ({ config }) => {
const configs = readAwsS3IntegrationConfigs(
config.getOptionalConfigArray('integrations.awsS3') ?? [],
);
return basicIntegrations(
configs.map(c => new AwsS3Integration(c)),
i => i.config.host,
);
};
get type(): string {
return 'awsS3';
}
get title(): string {
return this.integrationConfig.host;
}
get config(): AwsS3IntegrationConfig {
return this.integrationConfig;
}
constructor(private readonly integrationConfig: AwsS3IntegrationConfig) {}
resolveUrl(options: {
url: string;
base: string;
lineNumber?: number | undefined;
}): string {
const resolved = defaultScmResolveUrl(options);
return resolved;
}
resolveEditUrl(url: string): string {
// TODO: Implement edit URL for awsS3
return url;
}
}
packages/integration/src/awsS3/config.tests.ts → packages/integration/src/awsS3/config.test.ts
+37 -4
View File
@@ -15,7 +15,11 @@
*/
import { Config, ConfigReader } from '@backstage/config';
import { AwsS3IntegrationConfig, readAwsS3IntegrationConfig } from './config';
import {
AwsS3IntegrationConfig,
readAwsS3IntegrationConfig,
readAwsS3IntegrationConfigs,
} from './config';
describe('readAwsS3IntegrationConfig', () => {
function buildConfig(data: Partial<AwsS3IntegrationConfig>): Config {
@@ -25,18 +29,47 @@ describe('readAwsS3IntegrationConfig', () => {
it('reads all values', () => {
const output = readAwsS3IntegrationConfig(
buildConfig({
host: '.amazonaws.com',
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
}),
);
expect(output).toEqual({
host: '.amazonaws.com',
accessKeyId: 'fake-key',
secretAccessKey: 'fake-secret-key',
});
});
});
it('does not fail when config is not set', () => {
const output = readAwsS3IntegrationConfig(buildConfig({}));
expect(output).toEqual({});
describe('readAwsS3IntegrationConfigs', () => {
function buildConfig(data: Partial<AwsS3IntegrationConfig>[]): Config[] {
return data.map(item => new ConfigReader(item));
}
it('reads all values', () => {
const output = readAwsS3IntegrationConfigs(
buildConfig([
{
host: '.amazonaws.com',
accessKeyId: 'key',
secretAccessKey: 'secret',
},
]),
);
expect(output).toContainEqual({
host: '.amazonaws.com',
accessKeyId: 'key',
secretAccessKey: 'secret',
});
});
it('adds a default entry when missing', () => {
const output = readAwsS3IntegrationConfigs(buildConfig([]));
expect(output).toEqual([
{
host: '.amazonaws.com',
},
]);
});
});
packages/integration/src/awsS3/config.ts
+35 -11
View File
@@ -15,12 +15,22 @@
*/
import { Config } from '@backstage/config';
import { isValidHost } from '../helpers';
const AMAZON_AWS_HOST = '.amazonaws.com';
/**
* The configuration parameters for a single AWS S3 provider.
*/
export type AwsS3IntegrationConfig = {
/**
* The host of the target that this matches on, e.g. "amazonaws.com"
*
* Currently only "amazonaws.com" is supported.
*/
host: string;
/**
* accessKeyId
*/
@@ -41,17 +51,31 @@ export type AwsS3IntegrationConfig = {
export function readAwsS3IntegrationConfig(
config: Config,
): AwsS3IntegrationConfig {
if (!config) {
return {};
const host = config.getOptionalString('host') ?? AMAZON_AWS_HOST;
const accessKeyId = config.getOptionalString('accessKeyId');
const secretAccessKey = config.getOptionalString('secretAccessKey');
if (!isValidHost(host)) {
throw new Error(
`Invalid awsS3 integration config, '${host}' is not a valid host`,
);
}
if (!config.has('accessKeyId') && !config.has('secretAccessKey')) {
return {};
}
const accessKeyId = config.getString('accessKeyId');
const secretAccessKey = config.getString('secretAccessKey');
return { accessKeyId, secretAccessKey };
return { host, accessKeyId, secretAccessKey };
}
export function readAwsS3IntegrationConfigs(
configs: Config[],
): AwsS3IntegrationConfig[] {
// First read all the explicit integrations
const result = configs.map(readAwsS3IntegrationConfig);
// If no explicit amazonaws.com integration was added, put one in the list as
// a convenience
if (!result.some(c => c.host === AMAZON_AWS_HOST)) {
result.push({
host: AMAZON_AWS_HOST,
});
}
return result;
}
packages/integration/src/awsS3/index.ts
+5 -2
View File
@@ -13,6 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { readAwsS3IntegrationConfig } from './config';
export { AwsS3Integration } from './AwsS3Integration';
export {
readAwsS3IntegrationConfig,
readAwsS3IntegrationConfigs,
} from './config';
export type { AwsS3IntegrationConfig } from './config';
packages/integration/src/registry.ts
+2
View File
@@ -15,6 +15,7 @@
*/
import { ScmIntegration, ScmIntegrationsGroup } from './types';
import { AwsS3Integration } from './awsS3/AwsS3Integration';
import { AzureIntegration } from './azure/AzureIntegration';
import { BitbucketIntegration } from './bitbucket/BitbucketIntegration';
import { GitHubIntegration } from './github/GitHubIntegration';
@@ -25,6 +26,7 @@ import { GitLabIntegration } from './gitlab/GitLabIntegration';
*/
export interface ScmIntegrationRegistry
extends ScmIntegrationsGroup<ScmIntegration> {
awsS3: ScmIntegrationsGroup<AwsS3Integration>;
azure: ScmIntegrationsGroup<AzureIntegration>;
bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
github: ScmIntegrationsGroup<GitHubIntegration>;