diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 5ebe1c3e41..1a43dd6d62 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -36,6 +36,7 @@ "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", + "aws-sdk": "^2.817.0", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts new file mode 100644 index 0000000000..4fb189e433 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; + +describe('AwsOrganizationProcessor', () => { + describe('readLocation', () => { + const processor = new AwsOrganizationProcessor(); + const location = { type: 'aws-organization', target: 'b' }; + const emit = jest.fn(); + const listAccounts = jest.fn(); + + processor.organizations.listAccounts = listAccounts; + afterEach(() => jest.resetAllMocks()); + + it('generates component entities for accounts', async () => { + listAccounts.mockImplementation(() => { + return { + promise: async function () { + return { + Accounts: [ + { + Arn: + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + Name: 'testaccount', + }, + ], + NextToken: undefined, + }; + }, + }; + }); + await processor.readLocation(location, false, emit); + expect(emit).toBeCalledWith({ + type: 'entity', + location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + 'amazonaws.com/account-id': '957140518395', + 'amazonaws.com/organization-id': 'o-1vl18kc5a3', + }, + name: 'testaccount', + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts new file mode 100644 index 0000000000..70a74e8443 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + ComponentEntityV1alpha1, + Entity, + LocationSpec, +} from '@backstage/catalog-model'; +import AWS, { Organizations } from 'aws-sdk'; +import { Account } from 'aws-sdk/clients/organizations'; + +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +const AWS_ORGANIZATION_REGION = 'us-east-1'; +const LOCATION_TYPE = 'aws-organization'; + +export class AwsOrganizationProcessor implements CatalogProcessor { + organizations: Organizations; + constructor() { + this.organizations = new AWS.Organizations({ + region: AWS_ORGANIZATION_REGION, + }); // Only available in us-east-1 + } + + async handleError(): Promise { + return undefined; + } + + async postProcessEntity(entity: Entity): Promise { + return entity; + } + + async preProcessEntity(entity: Entity): Promise { + return entity; + } + + normalizeName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9\-]/g, '-'); + } + + extractInformationFromArn( + arn: string, + ): { accountId: string; organizationId: string } { + const parts = arn.split('/'); + + return { + accountId: parts[parts.length - 1], + organizationId: parts[parts.length - 2], + }; + } + + async getAwsAccounts(): Promise { + let awsAccounts: Account[] = []; + let isInitialAttempt = true; + let NextToken = undefined; + while (isInitialAttempt || NextToken) { + isInitialAttempt = false; + const orgAccounts = await this.organizations + .listAccounts({ NextToken }) + .promise(); + if (orgAccounts.Accounts) { + awsAccounts = awsAccounts.concat(orgAccounts.Accounts); + NextToken = orgAccounts.NextToken; + } + } + + return awsAccounts; + } + + mapAccountToComponent(account: Account): ComponentEntityV1alpha1 { + const { accountId, organizationId } = this.extractInformationFromArn( + account.Arn as string, + ); + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': account.Arn || '', + 'amazonaws.com/account-id': accountId, + 'amazonaws.com/organization-id': organizationId, + }, + name: this.normalizeName(account.Name || ''), + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== LOCATION_TYPE) { + return false; + } + + (await this.getAwsAccounts()) + .map(account => this.mapAccountToComponent(account)) + .forEach((entity: ComponentEntityV1alpha1) => { + emit(results.entity(location, entity)); + }); + + return true; + } + + async validateEntityKind(): Promise { + return false; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 2b477f18f9..7818c083ee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -17,6 +17,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; +export { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; diff --git a/yarn.lock b/yarn.lock index 195f746693..a8405333af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1666,10 +1666,10 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.4.1" + version "0.4.2" dependencies: "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.6" + "@backstage/core-api" "^0.2.7" "@backstage/theme" "^0.2.2" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -7712,6 +7712,21 @@ autoprefixer@^9.7.2: postcss "^7.0.32" postcss-value-parser "^4.1.0" +aws-sdk@^2.817.0: + version "2.817.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.817.0.tgz#3a97b690b0ec494cf8ee927affb3973cf26abcc8" + integrity sha512-DZIdWpkcqbqsCz0MEskHsyFaqc6Tk9XIFqXAg1AKHbOgC8nU45bz+Y2osX77pU01JkS/G7OhGtGmlKDrOPvFwg== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -8604,7 +8619,7 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -12025,6 +12040,11 @@ eventemitter3@^4.0.0: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb" integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + events@3.1.0, events@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/events/-/events-3.1.0.tgz#84279af1b34cb75aa88bf5ff291f6d0bd9b31a59" @@ -14304,6 +14324,11 @@ identity-obj-proxy@3.0.0: dependencies: harmony-reflect "^1.4.6" +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -15706,6 +15731,11 @@ jest@^26.0.1: import-local "^3.0.2" jest-cli "^26.6.3" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + jose@^1.27.1: version "1.27.1" resolved "https://registry.npmjs.org/jose/-/jose-1.27.1.tgz#a1de2ecb5b3ae1ae28f0d9d0cc536349ada27ec8" @@ -22019,6 +22049,11 @@ sanitize-html@^1.27.0: srcset "^2.0.1" xtend "^4.0.1" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24563,6 +24598,14 @@ url-parse@^1.4.3, url-parse@^1.4.7: querystringify "^2.1.1" requires-port "^1.0.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0, url@~0.11.0: version "0.11.0" resolved "https://registry.npmjs.org/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24661,6 +24704,11 @@ utils-merge@1.0.1, utils-merge@1.x.x: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -25392,6 +25440,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + xml2js@0.4.x: version "0.4.23" resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25405,6 +25461,11 @@ xmlbuilder@^11.0.0, xmlbuilder@~11.0.0: resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"