feat: add ingestion processor for indexing AWS accounts

Signed-off-by: Jonah Back <jback@legalzoom.com>
2020-12-23 23:22:30 -08:00
parent 3d3c9bd5e9
commit b67efe7441
5 changed files with 269 additions and 3 deletions
@@ -36,6 +36,7 @@
    "@octokit/graphql": "^4.5.6",
    "@types/express": "^4.17.6",
    "@types/ldapjs": "^1.0.9",
+    "aws-sdk": "^2.817.0",
    "codeowners-utils": "^1.0.2",
    "core-js": "^3.6.5",
    "cross-fetch": "^3.0.6",
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AwsOrganizationProcessor } from './AwsOrganizationProcessor';
+
+describe('AwsOrganizationProcessor', () => {
+  describe('readLocation', () => {
+    const processor = new AwsOrganizationProcessor();
+    const location = { type: 'aws-organization', target: 'b' };
+    const emit = jest.fn();
+    const listAccounts = jest.fn();
+
+    processor.organizations.listAccounts = listAccounts;
+    afterEach(() => jest.resetAllMocks());
+
+    it('generates component entities for accounts', async () => {
+      listAccounts.mockImplementation(() => {
+        return {
+          promise: async function () {
+            return {
+              Accounts: [
+                {
+                  Arn:
+                    'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+                  Name: 'testaccount',
+                },
+              ],
+              NextToken: undefined,
+            };
+          },
+        };
+      });
+      await processor.readLocation(location, false, emit);
+      expect(emit).toBeCalledWith({
+        type: 'entity',
+        location,
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Component',
+          metadata: {
+            annotations: {
+              'amazonaws.com/arn':
+                'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+              'amazonaws.com/account-id': '957140518395',
+              'amazonaws.com/organization-id': 'o-1vl18kc5a3',
+            },
+            name: 'testaccount',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'cloud-account',
+            lifecycle: 'unknown',
+            owner: 'unknown',
+          },
+        },
+      });
+    });
+  });
+});
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  ComponentEntityV1alpha1,
+  Entity,
+  LocationSpec,
+} from '@backstage/catalog-model';
+import AWS, { Organizations } from 'aws-sdk';
+import { Account } from 'aws-sdk/clients/organizations';
+
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+const AWS_ORGANIZATION_REGION = 'us-east-1';
+const LOCATION_TYPE = 'aws-organization';
+
+export class AwsOrganizationProcessor implements CatalogProcessor {
+  organizations: Organizations;
+  constructor() {
+    this.organizations = new AWS.Organizations({
+      region: AWS_ORGANIZATION_REGION,
+    }); // Only available in us-east-1
+  }
+
+  async handleError(): Promise<void> {
+    return undefined;
+  }
+
+  async postProcessEntity(entity: Entity): Promise<Entity> {
+    return entity;
+  }
+
+  async preProcessEntity(entity: Entity): Promise<Entity> {
+    return entity;
+  }
+
+  normalizeName(name: string): string {
+    return name
+      .trim()
+      .toLocaleLowerCase()
+      .replace(/[^a-zA-Z0-9\-]/g, '-');
+  }
+
+  extractInformationFromArn(
+    arn: string,
+  ): { accountId: string; organizationId: string } {
+    const parts = arn.split('/');
+
+    return {
+      accountId: parts[parts.length - 1],
+      organizationId: parts[parts.length - 2],
+    };
+  }
+
+  async getAwsAccounts(): Promise<Account[]> {
+    let awsAccounts: Account[] = [];
+    let isInitialAttempt = true;
+    let NextToken = undefined;
+    while (isInitialAttempt || NextToken) {
+      isInitialAttempt = false;
+      const orgAccounts = await this.organizations
+        .listAccounts({ NextToken })
+        .promise();
+      if (orgAccounts.Accounts) {
+        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
+        NextToken = orgAccounts.NextToken;
+      }
+    }
+
+    return awsAccounts;
+  }
+
+  mapAccountToComponent(account: Account): ComponentEntityV1alpha1 {
+    const { accountId, organizationId } = this.extractInformationFromArn(
+      account.Arn as string,
+    );
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        annotations: {
+          'amazonaws.com/arn': account.Arn || '',
+          'amazonaws.com/account-id': accountId,
+          'amazonaws.com/organization-id': organizationId,
+        },
+        name: this.normalizeName(account.Name || ''),
+        namespace: 'default',
+      },
+      spec: {
+        type: 'cloud-account',
+        lifecycle: 'unknown',
+        owner: 'unknown',
+      },
+    };
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== LOCATION_TYPE) {
+      return false;
+    }
+
+    (await this.getAwsAccounts())
+      .map(account => this.mapAccountToComponent(account))
+      .forEach((entity: ComponentEntityV1alpha1) => {
+        emit(results.entity(location, entity));
+      });
+
+    return true;
+  }
+
+  async validateEntityKind(): Promise<boolean> {
+    return false;
+  }
+}
@@ -17,6 +17,7 @@
 import * as results from './results';

 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
+export { AwsOrganizationProcessor } from './AwsOrganizationProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
@@ -1666,10 +1666,10 @@
    yup "^0.29.3"

 "@backstage/core@^0.3.0":
-  version "0.4.1"
+  version "0.4.2"
  dependencies:
    "@backstage/config" "^0.1.2"
-    "@backstage/core-api" "^0.2.6"
+    "@backstage/core-api" "^0.2.7"
    "@backstage/theme" "^0.2.2"
    "@material-ui/core" "^4.11.0"
    "@material-ui/icons" "^4.9.1"
@@ -7712,6 +7712,21 @@ autoprefixer@^9.7.2:
    postcss "^7.0.32"
    postcss-value-parser "^4.1.0"

+aws-sdk@^2.817.0:
+  version "2.817.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.817.0.tgz#3a97b690b0ec494cf8ee927affb3973cf26abcc8"
+  integrity sha512-DZIdWpkcqbqsCz0MEskHsyFaqc6Tk9XIFqXAg1AKHbOgC8nU45bz+Y2osX77pU01JkS/G7OhGtGmlKDrOPvFwg==
+  dependencies:
+    buffer "4.9.2"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 aws-sign2@~0.7.0:
  version "0.7.0"
  resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -8604,7 +8619,7 @@ buffer-xor@^1.0.3:
  resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
  integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=

-buffer@^4.3.0:
+buffer@4.9.2, buffer@^4.3.0:
  version "4.9.2"
  resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
  integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
@@ -12025,6 +12040,11 @@ eventemitter3@^4.0.0:
  resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb"
  integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==

+events@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 events@3.1.0, events@^3.0.0:
  version "3.1.0"
  resolved "https://registry.npmjs.org/events/-/events-3.1.0.tgz#84279af1b34cb75aa88bf5ff291f6d0bd9b31a59"
@@ -14304,6 +14324,11 @@ identity-obj-proxy@3.0.0:
  dependencies:
    harmony-reflect "^1.4.6"

+ieee754@1.1.13:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
+  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
+
 ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1:
  version "1.2.1"
  resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
@@ -15706,6 +15731,11 @@ jest@^26.0.1:
    import-local "^3.0.2"
    jest-cli "^26.6.3"

+jmespath@0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
+  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
+
 jose@^1.27.1:
  version "1.27.1"
  resolved "https://registry.npmjs.org/jose/-/jose-1.27.1.tgz#a1de2ecb5b3ae1ae28f0d9d0cc536349ada27ec8"
@@ -22019,6 +22049,11 @@ sanitize-html@^1.27.0:
    srcset "^2.0.1"
    xtend "^4.0.1"

+sax@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
+  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
+
 sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4:
  version "1.2.4"
  resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -24563,6 +24598,14 @@ url-parse@^1.4.3, url-parse@^1.4.7:
    querystringify "^2.1.1"
    requires-port "^1.0.0"

+url@0.10.3:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
+  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 url@^0.11.0, url@~0.11.0:
  version "0.11.0"
  resolved "https://registry.npmjs.org/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"
@@ -24661,6 +24704,11 @@ utils-merge@1.0.1, utils-merge@1.x.x:
  resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
  integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=

+uuid@3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
+  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
+
 uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2, uuid@^3.4.0:
  version "3.4.0"
  resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@@ -25392,6 +25440,14 @@ xml-name-validator@^3.0.0:
  resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a"
  integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==

+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
 xml2js@0.4.x:
  version "0.4.23"
  resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
@@ -25405,6 +25461,11 @@ xmlbuilder@^11.0.0, xmlbuilder@~11.0.0:
  resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
  integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==

+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
+
 xmlchars@^2.2.0:
  version "2.2.0"
  resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"