feat: add ingestion processor for indexing AWS accounts

Signed-off-by: Jonah Back <jback@legalzoom.com>

plugins/catalog-backend/package.json

@@ -36,6 +36,7 @@
     "@octokit/graphql": "^4.5.6",
     "@types/express": "^4.17.6",
     "@types/ldapjs": "^1.0.9",
+    "aws-sdk": "^2.817.0",
     "codeowners-utils": "^1.0.2",
     "core-js": "^3.6.5",
     "cross-fetch": "^3.0.6",

plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { AwsOrganizationProcessor } from './AwsOrganizationProcessor';
+
+describe('AwsOrganizationProcessor', () => {
+  describe('readLocation', () => {
+    const processor = new AwsOrganizationProcessor();
+    const location = { type: 'aws-organization', target: 'b' };
+    const emit = jest.fn();
+    const listAccounts = jest.fn();
+
+    processor.organizations.listAccounts = listAccounts;
+    afterEach(() => jest.resetAllMocks());
+
+    it('generates component entities for accounts', async () => {
+      listAccounts.mockImplementation(() => {
+        return {
+          promise: async function () {
+            return {
+              Accounts: [
+                {
+                  Arn:
+                    'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+                  Name: 'testaccount',
+                },
+              ],
+              NextToken: undefined,
+            };
+          },
+        };
+      });
+      await processor.readLocation(location, false, emit);
+      expect(emit).toBeCalledWith({
+        type: 'entity',
+        location,
+        entity: {
+          apiVersion: 'backstage.io/v1alpha1',
+          kind: 'Component',
+          metadata: {
+            annotations: {
+              'amazonaws.com/arn':
+                'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395',
+              'amazonaws.com/account-id': '957140518395',
+              'amazonaws.com/organization-id': 'o-1vl18kc5a3',
+            },
+            name: 'testaccount',
+            namespace: 'default',
+          },
+          spec: {
+            type: 'cloud-account',
+            lifecycle: 'unknown',
+            owner: 'unknown',
+          },
+        },
+      });
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  ComponentEntityV1alpha1,
+  Entity,
+  LocationSpec,
+} from '@backstage/catalog-model';
+import AWS, { Organizations } from 'aws-sdk';
+import { Account } from 'aws-sdk/clients/organizations';
+
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+
+const AWS_ORGANIZATION_REGION = 'us-east-1';
+const LOCATION_TYPE = 'aws-organization';
+
+export class AwsOrganizationProcessor implements CatalogProcessor {
+  organizations: Organizations;
+  constructor() {
+    this.organizations = new AWS.Organizations({
+      region: AWS_ORGANIZATION_REGION,
+    }); // Only available in us-east-1
+  }
+
+  async handleError(): Promise<void> {
+    return undefined;
+  }
+
+  async postProcessEntity(entity: Entity): Promise<Entity> {
+    return entity;
+  }
+
+  async preProcessEntity(entity: Entity): Promise<Entity> {
+    return entity;
+  }
+
+  normalizeName(name: string): string {
+    return name
+      .trim()
+      .toLocaleLowerCase()
+      .replace(/[^a-zA-Z0-9\-]/g, '-');
+  }
+
+  extractInformationFromArn(
+    arn: string,
+  ): { accountId: string; organizationId: string } {
+    const parts = arn.split('/');
+
+    return {
+      accountId: parts[parts.length - 1],
+      organizationId: parts[parts.length - 2],
+    };
+  }
+
+  async getAwsAccounts(): Promise<Account[]> {
+    let awsAccounts: Account[] = [];
+    let isInitialAttempt = true;
+    let NextToken = undefined;
+    while (isInitialAttempt || NextToken) {
+      isInitialAttempt = false;
+      const orgAccounts = await this.organizations
+        .listAccounts({ NextToken })
+        .promise();
+      if (orgAccounts.Accounts) {
+        awsAccounts = awsAccounts.concat(orgAccounts.Accounts);
+        NextToken = orgAccounts.NextToken;
+      }
+    }
+
+    return awsAccounts;
+  }
+
+  mapAccountToComponent(account: Account): ComponentEntityV1alpha1 {
+    const { accountId, organizationId } = this.extractInformationFromArn(
+      account.Arn as string,
+    );
+    return {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        annotations: {
+          'amazonaws.com/arn': account.Arn || '',
+          'amazonaws.com/account-id': accountId,
+          'amazonaws.com/organization-id': organizationId,
+        },
+        name: this.normalizeName(account.Name || ''),
+        namespace: 'default',
+      },
+      spec: {
+        type: 'cloud-account',
+        lifecycle: 'unknown',
+        owner: 'unknown',
+      },
+    };
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== LOCATION_TYPE) {
+      return false;
+    }
+
+    (await this.getAwsAccounts())
+      .map(account => this.mapAccountToComponent(account))
+      .forEach((entity: ComponentEntityV1alpha1) => {
+        emit(results.entity(location, entity));
+      });
+
+    return true;
+  }
+
+  async validateEntityKind(): Promise<boolean> {
+    return false;
+  }
+}

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -17,6 +17,7 @@
 import * as results from './results';
 
 export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor';
+export { AwsOrganizationProcessor } from './AwsOrganizationProcessor';
 export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';