introduce a catalog aws module

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2022-02-18 10:19:01 +01:00
parent ba68ea4450
commit 25e97e7242
26 changed files with 351 additions and 157 deletions
+33
View File
@@ -0,0 +1,33 @@
---
'@backstage/plugin-catalog-backend': minor
---
**BREAKING**: Removed `AwsOrganizationCloudAccountProcessor` from the default
set of builtin processors, and instead moved it into its own module
`@backstage/plugin-catalog-backend-module-aws`.
If you were using this processor, through making use of the location type
`aws-cloud-accounts` and/or using the configuration key
`catalog.processors.awsOrganization`, you will from now on have to add the
processor manually to your catalog.
First, add the `@backstage/plugin-catalog-backend-module-aws` dependency to your
`packages/backend` package.
Then, in `packages/backend/src/plugins/catalog.ts`:
```diff
+import { AwsOrganizationCloudAccountProcessor } from '@backstage/plugin-catalog-backend-module-aws';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
+ builder.addProcessor(
+ AwsOrganizationCloudAccountProcessor.fromConfig(
+ env.config,
+ { logger: env.logger }
+ )
+ );
// ...
```
+33
View File
@@ -0,0 +1,33 @@
---
'@backstage/plugin-catalog-backend-module-aws': minor
---
Added this new catalog module, initially containing only the
`AwsOrganizationCloudAccountProcessor`.
Note that this was moved over from the catalog backend itself, and therefore is
no longer part of its builtin set of processors. If you were using this
processor, through making use of the location type `aws-cloud-accounts` and/or
using the configuration key `catalog.processors.awsOrganization`, you will from
now on have to add the processor manually to your catalog.
First, add the `@backstage/plugin-catalog-backend-module-aws` dependency to your
`packages/backend` package.
Then, in `packages/backend/src/plugins/catalog.ts`:
```diff
+import { AwsOrganizationCloudAccountProcessor } from '@backstage/plugin-catalog-backend-module-aws';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
+ builder.addProcessor(
+ AwsOrganizationCloudAccountProcessor.fromConfig(
+ env.config,
+ { logger: env.logger }
+ )
+ );
// ...
```
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/plugin-catalog-backend-module-ldap': patch
'@backstage/plugin-catalog-backend-module-msgraph': patch
---
Minor wording update
+1
View File
@@ -17,6 +17,7 @@ autoscaling
Autoscaling
autoselect
Avro
aws
backrub
backported
backporting
@@ -0,0 +1,3 @@
module.exports = {
extends: [require.resolve('@backstage/cli/config/eslint.backend')],
};
@@ -0,0 +1,10 @@
# Catalog Backend Module for LDAP
This is an extension module to the plugin-catalog-backend plugin, providing an
`AwsOrganizationCloudAccountProcessor` that can be used to ingest cloud accounts
as `Resource` kind entities.
## Getting started
See [Backstage documentation](https://backstage.io/docs/integrations/ldap/org) for details on how to install
and configure the plugin.
@@ -0,0 +1,30 @@
## API Report File for "@backstage/plugin-catalog-backend-module-aws"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
import { Config } from '@backstage/config';
import { LocationSpec } from '@backstage/catalog-model';
import { Logger as Logger_2 } from 'winston';
// @public
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger_2;
},
): AwsOrganizationCloudAccountProcessor;
// (undocumented)
getProcessorName(): string;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
```
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface Config {
catalog?: {
/**
* List of processor-specific options and attributes
*/
processors?: {
/**
* AwsOrganizationCloudAccountProcessor configuration
*/
awsOrganization?: {
provider: {
/**
* The role to be assumed by this processor
*/
roleArn?: string;
};
};
};
};
}
@@ -0,0 +1,55 @@
{
"name": "@backstage/plugin-catalog-backend-module-aws",
"description": "A Backstage catalog backend module that helps integrate towards AWS",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin-module"
},
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/catalog-backend-module-aws"
},
"keywords": [
"backstage"
],
"scripts": {
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack",
"clean": "backstage-cli package clean",
"start": "backstage-cli package start"
},
"dependencies": {
"@backstage/catalog-model": "^0.10.0",
"@backstage/config": "^0.1.14",
"@backstage/errors": "^0.2.1",
"@backstage/plugin-catalog-backend": "^0.21.4",
"@backstage/types": "^0.1.2",
"aws-sdk": "^2.840.0",
"lodash": "^4.17.21",
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/cli": "^0.14.0",
"@types/lodash": "^4.14.151",
"aws-sdk-mock": "^5.2.1"
},
"files": [
"dist",
"config.d.ts"
],
"configSchema": "config.d.ts"
}
@@ -0,0 +1,23 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A Backstage catalog backend module that helps integrate towards AWS
*
* @packageDocumentation
*/
export * from './processors';
@@ -15,13 +15,11 @@
*/
import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
import * as winston from 'winston';
describe('AwsOrganizationCloudAccountProcessor', () => {
describe('readLocation', () => {
const processor = new AwsOrganizationCloudAccountProcessor({
const processor = new (AwsOrganizationCloudAccountProcessor as any)({
provider: {},
logger: winston.createLogger(),
});
const location = { type: 'aws-cloud-accounts', target: '' };
const emit = jest.fn();
@@ -13,35 +13,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec, ResourceEntityV1alpha1 } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
CatalogProcessor,
CatalogProcessorEmit,
results,
} from '@backstage/plugin-catalog-backend';
import AWS, { Credentials, Organizations } from 'aws-sdk';
import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations';
import * as results from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import {
AwsOrganizationProviderConfig,
readAwsOrganizationConfig,
} from './awsOrganization/config';
} from '../awsOrganization/config';
const AWS_ORGANIZATION_REGION = 'us-east-1';
const LOCATION_TYPE = 'aws-cloud-accounts';
const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id';
const ARN_ANNOTATION: string = 'amazonaws.com/arn';
const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id';
const ACCOUNTID_ANNOTATION = 'amazonaws.com/account-id';
const ARN_ANNOTATION = 'amazonaws.com/arn';
const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id';
/**
* A processor for ingesting AWS Accounts from AWS Organizations.
*
* If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object.
* If custom authentication is needed, it can be achieved by configuring the
* global AWS.credentials object.
*
* @public
*/
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
logger: Logger;
organizations: Organizations;
provider: AwsOrganizationProviderConfig;
private readonly organizations: Organizations;
private readonly provider: AwsOrganizationProviderConfig;
static fromConfig(config: Config, options: { logger: Logger }) {
const c = config.getOptionalConfig('catalog.processors.awsOrganization');
@@ -67,12 +72,8 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
});
}
constructor(options: {
provider: AwsOrganizationProviderConfig;
logger: Logger;
}) {
private constructor(options: { provider: AwsOrganizationProviderConfig }) {
this.provider = options.provider;
this.logger = options.logger;
const credentials = AwsOrganizationCloudAccountProcessor.buildCredentials(
this.provider,
);
@@ -81,70 +82,11 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
region: AWS_ORGANIZATION_REGION,
}); // Only available in us-east-1
}
getProcessorName(): string {
return 'AwsOrganizationCloudAccountProcessor';
}
normalizeName(name: string): string {
return name
.trim()
.toLocaleLowerCase()
.replace(/[^a-zA-Z0-9\-]/g, '-');
}
extractInformationFromArn(arn: string): {
accountId: string;
organizationId: string;
} {
const parts = arn.split('/');
return {
accountId: parts[parts.length - 1],
organizationId: parts[parts.length - 2],
};
}
async getAwsAccounts(): Promise<Account[]> {
let awsAccounts: Account[] = [];
let isInitialAttempt = true;
let nextToken = undefined;
while (isInitialAttempt || nextToken) {
isInitialAttempt = false;
const orgAccounts: ListAccountsResponse = await this.organizations
.listAccounts({ NextToken: nextToken })
.promise();
if (orgAccounts.Accounts) {
awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
}
nextToken = orgAccounts.NextToken;
}
return awsAccounts;
}
mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {
const { accountId, organizationId } = this.extractInformationFromArn(
account.Arn as string,
);
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Resource',
metadata: {
annotations: {
[ACCOUNTID_ANNOTATION]: accountId,
[ARN_ANNOTATION]: account.Arn || '',
[ORGANIZATION_ANNOTATION]: organizationId,
},
name: this.normalizeName(account.Name || ''),
namespace: 'default',
},
spec: {
type: 'cloud-account',
owner: 'unknown',
},
};
}
async readLocation(
location: LocationSpec,
_optional: boolean,
@@ -168,10 +110,70 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
}
return true;
})
.forEach((entity: ResourceEntityV1alpha1) => {
.forEach(entity => {
emit(results.entity(location, entity));
});
return true;
}
private normalizeName(name: string): string {
return name
.trim()
.toLocaleLowerCase('en-US')
.replace(/[^a-zA-Z0-9\-]/g, '-');
}
private extractInformationFromArn(arn: string): {
accountId: string;
organizationId: string;
} {
const parts = arn.split('/');
return {
accountId: parts[parts.length - 1],
organizationId: parts[parts.length - 2],
};
}
private async getAwsAccounts(): Promise<Account[]> {
let awsAccounts: Account[] = [];
let isInitialAttempt = true;
let nextToken = undefined;
while (isInitialAttempt || nextToken) {
isInitialAttempt = false;
const orgAccounts: ListAccountsResponse = await this.organizations
.listAccounts({ NextToken: nextToken })
.promise();
if (orgAccounts.Accounts) {
awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
}
nextToken = orgAccounts.NextToken;
}
return awsAccounts;
}
private mapAccountToComponent(account: Account): ResourceEntityV1alpha1 {
const { accountId, organizationId } = this.extractInformationFromArn(
account.Arn as string,
);
return {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Resource',
metadata: {
annotations: {
[ACCOUNTID_ANNOTATION]: accountId,
[ARN_ANNOTATION]: account.Arn || '',
[ORGANIZATION_ANNOTATION]: organizationId,
},
name: this.normalizeName(account.Name || ''),
namespace: 'default',
},
spec: {
type: 'cloud-account',
owner: 'unknown',
},
};
}
}
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
@@ -0,0 +1,17 @@
/*
* Copyright 2020 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};
+1 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { JsonValue } from '@backstage/config';
import { JsonValue } from '@backstage/types';
export interface Config {
/**
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-catalog-backend-module-ldap",
"description": "A Backstage catalog backend modules that helps integrate towards LDAP",
"description": "A Backstage catalog backend module that helps integrate towards LDAP",
"version": "0.3.13",
"main": "src/index.ts",
"types": "src/index.ts",
@@ -15,7 +15,7 @@
*/
/**
* A Backstage catalog backend modules that helps integrate towards LDAP
* A Backstage catalog backend module that helps integrate towards LDAP
*
* @packageDocumentation
*/
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-catalog-backend-module-msgraph",
"description": "A Backstage catalog backend modules that helps integrate towards Microsoft Graph",
"description": "A Backstage catalog backend module that helps integrate towards Microsoft Graph",
"version": "0.2.16",
"main": "src/index.ts",
"types": "src/index.ts",
@@ -15,7 +15,7 @@
*/
/**
* A Backstage catalog backend modules that helps integrate towards Microsoft Graph
* A Backstage catalog backend module that helps integrate towards Microsoft Graph
*
* @packageDocumentation
*/
-52
View File
@@ -5,7 +5,6 @@
```ts
/// <reference types="node" />
import { Account } from 'aws-sdk/clients/organizations';
import { BitbucketIntegration } from '@backstage/integration';
import { CatalogApi } from '@backstage/catalog-client';
import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node';
@@ -25,7 +24,6 @@ import { JsonValue } from '@backstage/types';
import { Location as Location_2 } from '@backstage/catalog-client';
import { LocationSpec } from '@backstage/catalog-model';
import { Logger as Logger_2 } from 'winston';
import { Organizations } from 'aws-sdk';
import { Permission } from '@backstage/plugin-permission-common';
import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
import { PermissionCondition } from '@backstage/plugin-permission-common';
@@ -33,7 +31,6 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common';
import { PermissionRule } from '@backstage/plugin-permission-node';
import { PluginDatabaseManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { ResourceEntityV1alpha1 } from '@backstage/catalog-model';
import { Router } from 'express';
import { ScmIntegrationRegistry } from '@backstage/integration';
import { TokenManager } from '@backstage/backend-common';
@@ -115,55 +112,6 @@ export class AnnotateScmSlugEntityProcessor implements CatalogProcessor {
preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
}
// Warning: (ae-missing-release-tag) "AwsOrganizationCloudAccountProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor {
constructor(options: {
provider: AwsOrganizationProviderConfig;
logger: Logger_2;
});
// (undocumented)
extractInformationFromArn(arn: string): {
accountId: string;
organizationId: string;
};
// (undocumented)
static fromConfig(
config: Config,
options: {
logger: Logger_2;
},
): AwsOrganizationCloudAccountProcessor;
// (undocumented)
getAwsAccounts(): Promise<Account[]>;
// (undocumented)
getProcessorName(): string;
// (undocumented)
logger: Logger_2;
// (undocumented)
mapAccountToComponent(account: Account): ResourceEntityV1alpha1;
// (undocumented)
normalizeName(name: string): string;
// (undocumented)
organizations: Organizations;
// (undocumented)
provider: AwsOrganizationProviderConfig;
// (undocumented)
readLocation(
location: LocationSpec,
_optional: boolean,
emit: CatalogProcessorEmit,
): Promise<boolean>;
}
// Warning: (ae-missing-release-tag) "AwsOrganizationProviderConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export type AwsOrganizationProviderConfig = {
roleArn?: string;
};
// Warning: (ae-missing-release-tag) "AwsS3DiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
-15
View File
@@ -14,8 +14,6 @@
* limitations under the License.
*/
import { JsonValue } from '@backstage/config';
export interface Config {
/**
* Configuration options for the catalog plugin.
@@ -172,19 +170,6 @@ export interface Config {
userNamespace?: string;
}>;
};
/**
* AwsOrganizationCloudAccountProcessor configuration
*/
awsOrganization?: {
provider: {
/**
* The role to be assumed by this processor
*
*/
roleArn?: string;
};
};
};
};
}
-2
View File
@@ -47,7 +47,6 @@
"@backstage/types": "^0.1.2",
"@octokit/graphql": "^4.5.8",
"@types/express": "^4.17.6",
"aws-sdk": "^2.840.0",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
"express": "^4.17.1",
@@ -79,7 +78,6 @@
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
"@vscode/sqlite3": "^5.0.7",
"aws-sdk-mock": "^5.2.1",
"msw": "^0.35.0",
"supertest": "^6.1.3",
"wait-for-expect": "^3.0.2",
@@ -18,8 +18,6 @@ import * as results from './results';
export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor';
export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor';
export type { AwsOrganizationProviderConfig } from './awsOrganization/config';
export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor';
export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
+1
View File
@@ -219,6 +219,7 @@ const NO_WARNING_PACKAGES = [
'packages/release-manifests',
'packages/version-bridge',
'plugins/auth-node',
'plugins/catalog-backend-module-aws',
'plugins/catalog-backend-module-ldap',
'plugins/catalog-backend-module-msgraph',
'plugins/catalog-common',