From 53d790027fa957ba44062aff98bfcd09e1769941 Mon Sep 17 00:00:00 2001 From: Omer Farooq <17722640+o-farooq@users.noreply.github.com> Date: Sun, 1 Nov 2020 19:01:34 +1300 Subject: [PATCH 01/39] add techradar entry history data structure --- plugins/tech-radar/src/api.ts | 12 ++- .../src/components/RadarComponent.tsx | 24 ++++- plugins/tech-radar/src/sampleData.ts | 101 +++++++++++++----- plugins/tech-radar/src/utils/types.ts | 8 ++ 4 files changed, 114 insertions(+), 31 deletions(-) diff --git a/plugins/tech-radar/src/api.ts b/plugins/tech-radar/src/api.ts index 7a6e790136..10b8f480f2 100644 --- a/plugins/tech-radar/src/api.ts +++ b/plugins/tech-radar/src/api.ts @@ -34,11 +34,17 @@ export interface RadarQuadrant { export interface RadarEntry { key: string; // react key id: string; - moved: MovedState; - quadrant: RadarQuadrant; - ring: RadarRing; + quadrant: string; title: string; url: string; + history: Array; +} + +export interface RadarEntryHistory { + date: Date; + ring: string; + description?: string; + moved?: MovedState; } /** diff --git a/plugins/tech-radar/src/components/RadarComponent.tsx b/plugins/tech-radar/src/components/RadarComponent.tsx index 99190202b9..525949d040 100644 --- a/plugins/tech-radar/src/components/RadarComponent.tsx +++ b/plugins/tech-radar/src/components/RadarComponent.tsx @@ -20,6 +20,7 @@ import { useAsync } from 'react-use'; import Radar from '../components/Radar'; import { TechRadarComponentProps, TechRadarLoaderResponse } from '../api'; import getSampleData from '../sampleData'; +import { Entry } from '../utils/types'; const useTechRadarLoader = (props: TechRadarComponentProps) => { const errorApi = useApi(errorApiRef); @@ -47,6 +48,27 @@ const useTechRadarLoader = (props: TechRadarComponentProps) => { const RadarComponent = (props: TechRadarComponentProps): JSX.Element => { const { loading, error, value: data } = useTechRadarLoader(props); + const mapToEntries = ( + loaderResponse: TechRadarLoaderResponse | undefined, + ): Array => { + return loaderResponse!.entries.map(entry => { + return { + id: entry.key, + quadrant: loaderResponse!.quadrants.find(q => q.id === entry.quadrant)!, + title: entry.title, + ring: loaderResponse!.rings.find(r => r.id === entry.history[0].ring)!, + history: entry.history.map(e => { + return { + date: e.date, + ring: loaderResponse!.rings.find(a => a.id === e.ring)!, + description: e.description, + moved: e.moved, + }; + }), + }; + }); + }; + return ( <> {loading && } @@ -55,7 +77,7 @@ const RadarComponent = (props: TechRadarComponentProps): JSX.Element => { {...props} rings={data!.rings} quadrants={data!.quadrants} - entries={data!.entries} + entries={mapToEntries(data)} /> )} diff --git a/plugins/tech-radar/src/sampleData.ts b/plugins/tech-radar/src/sampleData.ts index a9fa7efd7b..7d4ff6198f 100644 --- a/plugins/tech-radar/src/sampleData.ts +++ b/plugins/tech-radar/src/sampleData.ts @@ -35,85 +35,132 @@ quadrants.push({ id: 'process', name: 'Process' }); const entries = new Array(); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + description: + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua', + }, + ], url: '#', key: 'javascript', id: 'javascript', title: 'JavaScript', - quadrant: { id: 'languages', name: 'Languages' }, + quadrant: 'languages', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + description: + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat', + }, + ], url: '#', key: 'typescript', id: 'typescript', title: 'TypeScript', - quadrant: { id: 'languages', name: 'Languages' }, + quadrant: 'languages', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'webpack', id: 'webpack', title: 'Webpack', - quadrant: { id: 'frameworks', name: 'Frameworks' }, + quadrant: 'frameworks', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'react', id: 'react', title: 'React', - quadrant: { id: 'frameworks', name: 'Frameworks' }, + quadrant: 'frameworks', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'code-reviews', id: 'code-reviews', title: 'Code Reviews', - quadrant: { id: 'process', name: 'Process' }, + quadrant: 'process', }); entries.push({ - moved: 0, + history: [ + { + moved: 0, + ring: 'assess', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'mob-programming', id: 'mob-programming', title: 'Mob Programming', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'assess', name: 'ASSESS', color: '#fbdb84' }, + quadrant: 'process', }); entries.push({ - moved: 0, + history: [ + { + moved: 0, + ring: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'docs-like-code', id: 'docs-like-code', title: 'Docs-like-code', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + quadrant: 'process', }); entries.push({ - moved: 0, + history: [ + { + ring: 'hold', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'force-push', id: 'force-push', title: 'Force push to master', - quadrant: { id: 'process', name: 'Process' }, - ring: { id: 'hold', name: 'HOLD', color: '#93c47d' }, + quadrant: 'process', }); entries.push({ - moved: 0, - ring: { id: 'use', name: 'USE', color: '#93c47d' }, + history: [ + { + ring: 'use', + date: new Date('2020-08-06'), + }, + ], url: '#', key: 'github-actions', id: 'github-actions', title: 'GitHub Actions', - quadrant: { id: 'infrastructure', name: 'Infrastructure' }, + quadrant: 'infrastructure', }); export default function getSampleData(): Promise { diff --git a/plugins/tech-radar/src/utils/types.ts b/plugins/tech-radar/src/utils/types.ts index f3933ee584..65c14f38c6 100644 --- a/plugins/tech-radar/src/utils/types.ts +++ b/plugins/tech-radar/src/utils/types.ts @@ -69,6 +69,14 @@ export type Entry = { // How this entry has recently moved; -1 for "down", +1 for "up", 0 for not moved moved?: MovedState; active?: boolean; + history?: Array; +}; + +export type EntryHistory = { + date: Date; + ring: Ring; + description?: string; + moved?: MovedState; }; // The same as Entry except quadrant/ring are declared by their string ID instead of being the actual objects From 59e2551801954c1a81e947b4390be143af095726 Mon Sep 17 00:00:00 2001 From: Omer Farooq <17722640+o-farooq@users.noreply.github.com> Date: Sat, 7 Nov 2020 23:10:41 +1300 Subject: [PATCH 02/39] updated names and added changeset --- .changeset/wicked-impalas-tan.md | 5 +++ plugins/tech-radar/src/api.ts | 6 ++-- .../src/components/RadarComponent.tsx | 8 +++-- plugins/tech-radar/src/sampleData.ts | 36 +++++++++---------- plugins/tech-radar/src/utils/types.ts | 4 +-- 5 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 .changeset/wicked-impalas-tan.md diff --git a/.changeset/wicked-impalas-tan.md b/.changeset/wicked-impalas-tan.md new file mode 100644 index 0000000000..ea68175d74 --- /dev/null +++ b/.changeset/wicked-impalas-tan.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': minor +--- + +Added tech radar blip history backend support and normalized the datastructure diff --git a/plugins/tech-radar/src/api.ts b/plugins/tech-radar/src/api.ts index 10b8f480f2..0c30c9374c 100644 --- a/plugins/tech-radar/src/api.ts +++ b/plugins/tech-radar/src/api.ts @@ -37,12 +37,12 @@ export interface RadarEntry { quadrant: string; title: string; url: string; - history: Array; + timeline: Array; } -export interface RadarEntryHistory { +export interface RadarEntrySnapshot { date: Date; - ring: string; + ringId: string; description?: string; moved?: MovedState; } diff --git a/plugins/tech-radar/src/components/RadarComponent.tsx b/plugins/tech-radar/src/components/RadarComponent.tsx index 525949d040..583a7c91e0 100644 --- a/plugins/tech-radar/src/components/RadarComponent.tsx +++ b/plugins/tech-radar/src/components/RadarComponent.tsx @@ -56,11 +56,13 @@ const RadarComponent = (props: TechRadarComponentProps): JSX.Element => { id: entry.key, quadrant: loaderResponse!.quadrants.find(q => q.id === entry.quadrant)!, title: entry.title, - ring: loaderResponse!.rings.find(r => r.id === entry.history[0].ring)!, - history: entry.history.map(e => { + ring: loaderResponse!.rings.find( + r => r.id === entry.timeline[0].ringId, + )!, + history: entry.timeline.map(e => { return { date: e.date, - ring: loaderResponse!.rings.find(a => a.id === e.ring)!, + ring: loaderResponse!.rings.find(a => a.id === e.ringId)!, description: e.description, moved: e.moved, }; diff --git a/plugins/tech-radar/src/sampleData.ts b/plugins/tech-radar/src/sampleData.ts index 7d4ff6198f..914bc30931 100644 --- a/plugins/tech-radar/src/sampleData.ts +++ b/plugins/tech-radar/src/sampleData.ts @@ -35,10 +35,10 @@ quadrants.push({ id: 'process', name: 'Process' }); const entries = new Array(); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), description: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua', @@ -51,10 +51,10 @@ entries.push({ quadrant: 'languages', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), description: 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat', @@ -67,10 +67,10 @@ entries.push({ quadrant: 'languages', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), }, ], @@ -81,10 +81,10 @@ entries.push({ quadrant: 'frameworks', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), }, ], @@ -95,10 +95,10 @@ entries.push({ quadrant: 'frameworks', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), }, ], @@ -109,10 +109,10 @@ entries.push({ quadrant: 'process', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'assess', + ringId: 'assess', date: new Date('2020-08-06'), }, ], @@ -123,10 +123,10 @@ entries.push({ quadrant: 'process', }); entries.push({ - history: [ + timeline: [ { moved: 0, - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), }, ], @@ -137,9 +137,9 @@ entries.push({ quadrant: 'process', }); entries.push({ - history: [ + timeline: [ { - ring: 'hold', + ringId: 'hold', date: new Date('2020-08-06'), }, ], @@ -150,9 +150,9 @@ entries.push({ quadrant: 'process', }); entries.push({ - history: [ + timeline: [ { - ring: 'use', + ringId: 'use', date: new Date('2020-08-06'), }, ], diff --git a/plugins/tech-radar/src/utils/types.ts b/plugins/tech-radar/src/utils/types.ts index 65c14f38c6..3322a67dcb 100644 --- a/plugins/tech-radar/src/utils/types.ts +++ b/plugins/tech-radar/src/utils/types.ts @@ -69,10 +69,10 @@ export type Entry = { // How this entry has recently moved; -1 for "down", +1 for "up", 0 for not moved moved?: MovedState; active?: boolean; - history?: Array; + timeline?: Array; }; -export type EntryHistory = { +export type EntrySnapshot = { date: Date; ring: Ring; description?: string; From 40bb4a2c7441eb9f2cb00eb467918c5197758fb5 Mon Sep 17 00:00:00 2001 From: Omer Farooq <17722640+o-farooq@users.noreply.github.com> Date: Sat, 7 Nov 2020 23:15:14 +1300 Subject: [PATCH 03/39] fix docs --- .changeset/wicked-impalas-tan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/wicked-impalas-tan.md b/.changeset/wicked-impalas-tan.md index ea68175d74..8cfe919f5c 100644 --- a/.changeset/wicked-impalas-tan.md +++ b/.changeset/wicked-impalas-tan.md @@ -2,4 +2,4 @@ '@backstage/plugin-tech-radar': minor --- -Added tech radar blip history backend support and normalized the datastructure +Added tech radar blip history backend support and normalized the data structure From b67efe7441f1f421cbe38817f561b60102e5ff74 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Wed, 23 Dec 2020 23:22:30 -0800 Subject: [PATCH 04/39] feat: add ingestion processor for indexing AWS accounts Signed-off-by: Jonah Back --- plugins/catalog-backend/package.json | 1 + .../AwsOrganizationProcessor.test.ts | 72 ++++++++++ .../processors/AwsOrganizationProcessor.ts | 131 ++++++++++++++++++ .../src/ingestion/processors/index.ts | 1 + yarn.lock | 67 ++++++++- 5 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 5ebe1c3e41..1a43dd6d62 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -36,6 +36,7 @@ "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", + "aws-sdk": "^2.817.0", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts new file mode 100644 index 0000000000..4fb189e433 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; + +describe('AwsOrganizationProcessor', () => { + describe('readLocation', () => { + const processor = new AwsOrganizationProcessor(); + const location = { type: 'aws-organization', target: 'b' }; + const emit = jest.fn(); + const listAccounts = jest.fn(); + + processor.organizations.listAccounts = listAccounts; + afterEach(() => jest.resetAllMocks()); + + it('generates component entities for accounts', async () => { + listAccounts.mockImplementation(() => { + return { + promise: async function () { + return { + Accounts: [ + { + Arn: + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + Name: 'testaccount', + }, + ], + NextToken: undefined, + }; + }, + }; + }); + await processor.readLocation(location, false, emit); + expect(emit).toBeCalledWith({ + type: 'entity', + location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + 'amazonaws.com/account-id': '957140518395', + 'amazonaws.com/organization-id': 'o-1vl18kc5a3', + }, + name: 'testaccount', + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts new file mode 100644 index 0000000000..70a74e8443 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + ComponentEntityV1alpha1, + Entity, + LocationSpec, +} from '@backstage/catalog-model'; +import AWS, { Organizations } from 'aws-sdk'; +import { Account } from 'aws-sdk/clients/organizations'; + +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +const AWS_ORGANIZATION_REGION = 'us-east-1'; +const LOCATION_TYPE = 'aws-organization'; + +export class AwsOrganizationProcessor implements CatalogProcessor { + organizations: Organizations; + constructor() { + this.organizations = new AWS.Organizations({ + region: AWS_ORGANIZATION_REGION, + }); // Only available in us-east-1 + } + + async handleError(): Promise { + return undefined; + } + + async postProcessEntity(entity: Entity): Promise { + return entity; + } + + async preProcessEntity(entity: Entity): Promise { + return entity; + } + + normalizeName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9\-]/g, '-'); + } + + extractInformationFromArn( + arn: string, + ): { accountId: string; organizationId: string } { + const parts = arn.split('/'); + + return { + accountId: parts[parts.length - 1], + organizationId: parts[parts.length - 2], + }; + } + + async getAwsAccounts(): Promise { + let awsAccounts: Account[] = []; + let isInitialAttempt = true; + let NextToken = undefined; + while (isInitialAttempt || NextToken) { + isInitialAttempt = false; + const orgAccounts = await this.organizations + .listAccounts({ NextToken }) + .promise(); + if (orgAccounts.Accounts) { + awsAccounts = awsAccounts.concat(orgAccounts.Accounts); + NextToken = orgAccounts.NextToken; + } + } + + return awsAccounts; + } + + mapAccountToComponent(account: Account): ComponentEntityV1alpha1 { + const { accountId, organizationId } = this.extractInformationFromArn( + account.Arn as string, + ); + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': account.Arn || '', + 'amazonaws.com/account-id': accountId, + 'amazonaws.com/organization-id': organizationId, + }, + name: this.normalizeName(account.Name || ''), + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== LOCATION_TYPE) { + return false; + } + + (await this.getAwsAccounts()) + .map(account => this.mapAccountToComponent(account)) + .forEach((entity: ComponentEntityV1alpha1) => { + emit(results.entity(location, entity)); + }); + + return true; + } + + async validateEntityKind(): Promise { + return false; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 2b477f18f9..7818c083ee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -17,6 +17,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; +export { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; diff --git a/yarn.lock b/yarn.lock index 195f746693..a8405333af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1666,10 +1666,10 @@ yup "^0.29.3" "@backstage/core@^0.3.0": - version "0.4.1" + version "0.4.2" dependencies: "@backstage/config" "^0.1.2" - "@backstage/core-api" "^0.2.6" + "@backstage/core-api" "^0.2.7" "@backstage/theme" "^0.2.2" "@material-ui/core" "^4.11.0" "@material-ui/icons" "^4.9.1" @@ -7712,6 +7712,21 @@ autoprefixer@^9.7.2: postcss "^7.0.32" postcss-value-parser "^4.1.0" +aws-sdk@^2.817.0: + version "2.817.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.817.0.tgz#3a97b690b0ec494cf8ee927affb3973cf26abcc8" + integrity sha512-DZIdWpkcqbqsCz0MEskHsyFaqc6Tk9XIFqXAg1AKHbOgC8nU45bz+Y2osX77pU01JkS/G7OhGtGmlKDrOPvFwg== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -8604,7 +8619,7 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -12025,6 +12040,11 @@ eventemitter3@^4.0.0: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb" integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + events@3.1.0, events@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/events/-/events-3.1.0.tgz#84279af1b34cb75aa88bf5ff291f6d0bd9b31a59" @@ -14304,6 +14324,11 @@ identity-obj-proxy@3.0.0: dependencies: harmony-reflect "^1.4.6" +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -15706,6 +15731,11 @@ jest@^26.0.1: import-local "^3.0.2" jest-cli "^26.6.3" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + jose@^1.27.1: version "1.27.1" resolved "https://registry.npmjs.org/jose/-/jose-1.27.1.tgz#a1de2ecb5b3ae1ae28f0d9d0cc536349ada27ec8" @@ -22019,6 +22049,11 @@ sanitize-html@^1.27.0: srcset "^2.0.1" xtend "^4.0.1" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24563,6 +24598,14 @@ url-parse@^1.4.3, url-parse@^1.4.7: querystringify "^2.1.1" requires-port "^1.0.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0, url@~0.11.0: version "0.11.0" resolved "https://registry.npmjs.org/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24661,6 +24704,11 @@ utils-merge@1.0.1, utils-merge@1.x.x: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -25392,6 +25440,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + xml2js@0.4.x: version "0.4.23" resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25405,6 +25461,11 @@ xmlbuilder@^11.0.0, xmlbuilder@~11.0.0: resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 48d78aaf19202613a149774ea7c607d8016c3d99 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 27 Dec 2020 23:37:24 -0500 Subject: [PATCH 05/39] Clarify local instructions --- packages/create-app/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/create-app/README.md b/packages/create-app/README.md index 7533110372..dafc01e365 100644 --- a/packages/create-app/README.md +++ b/packages/create-app/README.md @@ -1,9 +1,10 @@ # @backstage/create-app -This package provides a CLI for creating apps. +This package provides a CLI for creating a copy of the Backstage app. + You can use the flag `--skip-install` to skip the install. -## Installation +## Usage With `npx`: @@ -11,6 +12,12 @@ With `npx`: $ npx @backstage/create-app ``` +With a local clone of this repo, from the main `create-app/` folder, run: + +```sh +$ yarn backstage-create-app +``` + ## Documentation - [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) From 65632b3a362b31a5bdf2cece8b0b2ca29b4be6bf Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Sun, 27 Dec 2020 23:44:51 -0500 Subject: [PATCH 06/39] Add catalog-import plugin --- packages/create-app/package.json | 2 +- packages/create-app/src/lib/versions.ts | 4 ++-- .../templates/default-app/packages/app/package.json.hbs | 2 +- .../templates/default-app/packages/app/src/App.tsx | 6 +++--- .../templates/default-app/packages/app/src/plugins.ts | 8 ++++---- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/create-app/package.json b/packages/create-app/package.json index 23a0b1a584..d787f012db 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -47,12 +47,12 @@ "@backstage/plugin-auth-backend": "^0.2.7", "@backstage/plugin-catalog": "^0.2.8", "@backstage/plugin-catalog-backend": "^0.5.1", + "@backstage/plugin-catalog-import": "^0.3.2", "@backstage/plugin-circleci": "^0.2.5", "@backstage/plugin-explore": "^0.2.2", "@backstage/plugin-github-actions": "^0.2.6", "@backstage/plugin-lighthouse": "^0.2.6", "@backstage/plugin-proxy-backend": "^0.2.3", - "@backstage/plugin-register-component": "^0.2.5", "@backstage/plugin-rollbar-backend": "^0.1.5", "@backstage/plugin-scaffolder": "^0.3.5", "@backstage/plugin-search": "^0.2.4", diff --git a/packages/create-app/src/lib/versions.ts b/packages/create-app/src/lib/versions.ts index df8b9c01ea..5cbf6c77ee 100644 --- a/packages/create-app/src/lib/versions.ts +++ b/packages/create-app/src/lib/versions.ts @@ -39,12 +39,12 @@ import { version as pluginAppBackend } from '@backstage/plugin-app-backend/packa import { version as pluginAuthBackend } from '@backstage/plugin-auth-backend/package.json'; import { version as pluginCatalog } from '@backstage/plugin-catalog/package.json'; import { version as pluginCatalogBackend } from '@backstage/plugin-catalog-backend/package.json'; +import { version as pluginCatalogImport } from '@backstage/plugin-catalog-import/package.json'; import { version as pluginCircleci } from '@backstage/plugin-circleci/package.json'; import { version as pluginExplore } from '@backstage/plugin-explore/package.json'; import { version as pluginGithubActions } from '@backstage/plugin-github-actions/package.json'; import { version as pluginLighthouse } from '@backstage/plugin-lighthouse/package.json'; import { version as pluginProxyBackend } from '@backstage/plugin-proxy-backend/package.json'; -import { version as pluginRegisterComponent } from '@backstage/plugin-register-component/package.json'; import { version as pluginRollbarBackend } from '@backstage/plugin-rollbar-backend/package.json'; import { version as pluginScaffolder } from '@backstage/plugin-scaffolder/package.json'; import { version as pluginScaffolderBackend } from '@backstage/plugin-scaffolder-backend/package.json'; @@ -67,12 +67,12 @@ export const packageVersions = { '@backstage/plugin-auth-backend': pluginAuthBackend, '@backstage/plugin-catalog': pluginCatalog, '@backstage/plugin-catalog-backend': pluginCatalogBackend, + '@backstage/plugin-catalog-import': pluginCatalogImport, '@backstage/plugin-circleci': pluginCircleci, '@backstage/plugin-explore': pluginExplore, '@backstage/plugin-github-actions': pluginGithubActions, '@backstage/plugin-lighthouse': pluginLighthouse, '@backstage/plugin-proxy-backend': pluginProxyBackend, - '@backstage/plugin-register-component': pluginRegisterComponent, '@backstage/plugin-rollbar-backend': pluginRollbarBackend, '@backstage/plugin-scaffolder': pluginScaffolder, '@backstage/plugin-scaffolder-backend': pluginScaffolderBackend, diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs index 1746f171c9..40e520496a 100644 --- a/packages/create-app/templates/default-app/packages/app/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs @@ -10,7 +10,7 @@ "@backstage/core": "^{{version '@backstage/core'}}", "@backstage/plugin-api-docs": "^{{version '@backstage/plugin-api-docs'}}", "@backstage/plugin-catalog": "^{{version '@backstage/plugin-catalog'}}", - "@backstage/plugin-register-component": "^{{version '@backstage/plugin-register-component'}}", + "@backstage/plugin-catalog-import": "^{{version '@backstage/plugin-catalog-import'}}", "@backstage/plugin-scaffolder": "^{{version '@backstage/plugin-scaffolder'}}", "@backstage/plugin-techdocs": "^{{version '@backstage/plugin-techdocs'}}", "@backstage/catalog-model": "^{{version '@backstage/catalog-model'}}", diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 2e250a94a1..693e66e0c6 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -12,7 +12,7 @@ import { AppSidebar } from './sidebar'; import { Route, Routes, Navigate } from 'react-router'; import { Router as CatalogRouter } from '@backstage/plugin-catalog'; import { Router as DocsRouter } from '@backstage/plugin-techdocs'; -import { Router as RegisterComponentRouter } from '@backstage/plugin-register-component'; +import { Router as ImportComponentRouter } from '@backstage/plugin-catalog-import'; import { Router as TechRadarRouter } from '@backstage/plugin-tech-radar'; import { SearchPage as SearchRouter } from '@backstage/plugin-search'; @@ -52,8 +52,8 @@ const App = () => ( element={} /> } + path="/catalog-import" + element={} /> Date: Sun, 27 Dec 2020 23:46:15 -0500 Subject: [PATCH 07/39] Add changeset --- .changeset/cuddly-files-argue.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cuddly-files-argue.md diff --git a/.changeset/cuddly-files-argue.md b/.changeset/cuddly-files-argue.md new file mode 100644 index 0000000000..d495a945f4 --- /dev/null +++ b/.changeset/cuddly-files-argue.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Replace `register-component` plugin with new `catalog-import` plugin From 3fc5cb64fda0cce71164f11f850b130cc08bdbe4 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 00:53:47 -0500 Subject: [PATCH 08/39] Fix typo --- docs/plugins/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/index.md b/docs/plugins/index.md index dcef3f995c..32d523b2f5 100644 --- a/docs/plugins/index.md +++ b/docs/plugins/index.md @@ -33,4 +33,4 @@ that someone else will pick up the work. If your plugin isn't supposed to live as a standalone page, but rather needs to be presented as a part of a Service Catalog (e.g. a separate tab or a card on an "Overview" tab), then check out -[the instruction](integrating-plugin-into-service-catalog.md). on how to do it. +[the instruction](integrating-plugin-into-service-catalog.md) on how to do it. From 282ed3ee65d7c031c23cdd355444ade6260b87ba Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 00:58:16 -0500 Subject: [PATCH 09/39] Clarify link --- docs/support/project-structure.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/support/project-structure.md b/docs/support/project-structure.md index bbba127c9a..a30288e85e 100644 --- a/docs/support/project-structure.md +++ b/docs/support/project-structure.md @@ -41,8 +41,8 @@ the code. appreciate contributions in here and encourage them being kept up to date. - [`docs/`](https://github.com/backstage/backstage/tree/master/docs) - This is - where we keep all of our documentation Markdown files. These ends up on - http://backstage.io/docs. Just keep in mind that changes to the + where we keep all of our documentation Markdown files. These end up on + https://backstage.io/docs. Just keep in mind that changes to the [`sidebars.json`](https://github.com/backstage/backstage/blob/master/microsite/sidebars.json) file may be needed as sections are added/removed. From 69d4991ccae5945cd789f726a98657671b49c43e Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 00:58:28 -0500 Subject: [PATCH 10/39] Fix link --- packages/storybook/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/storybook/README.md b/packages/storybook/README.md index 38708afe4b..3e88b54db0 100644 --- a/packages/storybook/README.md +++ b/packages/storybook/README.md @@ -1,7 +1,7 @@ # storybook -This package provides a storybook build for Backstage. See [http://backstage.io/storybook](http://http://backstage.io/storybook) +This package provides a Storybook build for Backstage. See https://backstage.io/storybook/. ## Why is this not part of `@backstage/core`? -This separate storybook package exists because of dependency conflicts with `@backstage/cli`. It uses nohoist to avoid the conflicts, and since you can only use that in private packages it has to be separated out of `@backstage/core`. +This separate storybook package exists because of dependency conflicts with `@backstage/cli`. It uses `nohoist` to avoid the conflicts, and since you can only use that in private packages it has to be separated out of `@backstage/core`. From 9061a9dfe6471c0bc6b07c17e4ce83544e7e4101 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 01:01:29 -0500 Subject: [PATCH 11/39] Point to plugin docs --- plugins/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index 6651ba079f..48db29a70a 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,14 +1,12 @@ # Plugins -Backstage is a single-page application composed of a set of plugins. +Backstage is a single-page application composed of a set of plugins. This folder holds numerous plugins that are managed by this repository. -Our goal for the plugin ecosystem is that the definition of a plugin is flexible enough to allow you to expose pretty much any kind of infrastructure or software development tool as a plugin in Backstage. By following strong [design guidelines](https://github.com/backstage/backstage/blob/master/docs/dls/design.md) we ensure the overall user experience stays consistent between plugins. +For more information about the plugin ecosystem, see the documention here: -![plugin](../docs/assets/my-plugin_screenshot.png) +> https://backstage.io/docs/plugins/ -## Creating a plugin - -To create a plugin, follow the steps outlined [here](https://github.com/backstage/backstage/blob/master/docs/plugins/create-a-plugin.md). +You can also see the [Plugin Marketplace](https://https://backstage.io/plugins) to see other open source plugins to add to your instance. ## Suggesting a plugin From 6e7c9429ab39bcbb6741ec67207982f115c589a7 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 01:02:46 -0500 Subject: [PATCH 12/39] Clarify wording --- plugins/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/README.md b/plugins/README.md index 48db29a70a..b281f33b37 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -6,7 +6,7 @@ For more information about the plugin ecosystem, see the documention here: > https://backstage.io/docs/plugins/ -You can also see the [Plugin Marketplace](https://https://backstage.io/plugins) to see other open source plugins to add to your instance. +You can also see the [Plugin Marketplace](https://https://backstage.io/plugins) for other open source plugins you can add to your Backstage instance. ## Suggesting a plugin From a7f8cbaed19b448939179e564c8191608f42e7f6 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 01:03:25 -0500 Subject: [PATCH 13/39] Fix test description typos --- .../src/scaffolder/stages/publish/helpers.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts index 3a317adacd..61d8e459bc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.test.ts @@ -66,7 +66,7 @@ describe('pushToRemoteCred', () => { expect(mockIndex.writeTree).toHaveBeenCalled(); }); - it('should create a commit with on head with the right name and commiter', async () => { + it('should create a commit with on head with the right name and committer', async () => { const mockSignature = { mockSignature: 'bloblly' }; Signature.now.mockReturnValue(mockSignature); @@ -94,7 +94,7 @@ describe('pushToRemoteCred', () => { expect(Remote.create).toHaveBeenCalledWith(mockRepo, 'origin', 'mockclone'); }); - it('shoud push to the remote repo', async () => { + it('should push to the remote repo', async () => { await pushToRemoteCred(directory, remote, credentialsProvider); const [remotes, { callbacks }] = mockRemote.push.mock From 3440f9990b6fdf4a8217f27c9bbcfc22488251ca Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 01:11:23 -0500 Subject: [PATCH 14/39] Fix typo --- plugins/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/README.md b/plugins/README.md index b281f33b37..b462dbf14f 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -2,7 +2,7 @@ Backstage is a single-page application composed of a set of plugins. This folder holds numerous plugins that are managed by this repository. -For more information about the plugin ecosystem, see the documention here: +For more information about the plugin ecosystem, see the documentation here: > https://backstage.io/docs/plugins/ From f7969caef7edbc2f833d569ab7c661820bd11eb3 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 01:13:35 -0500 Subject: [PATCH 15/39] Fix link --- plugins/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/README.md b/plugins/README.md index b462dbf14f..385e71b6d2 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -6,7 +6,7 @@ For more information about the plugin ecosystem, see the documentation here: > https://backstage.io/docs/plugins/ -You can also see the [Plugin Marketplace](https://https://backstage.io/plugins) for other open source plugins you can add to your Backstage instance. +You can also see the [Plugin Marketplace](https://backstage.io/plugins) for other open source plugins you can add to your Backstage instance. ## Suggesting a plugin From 9d4f9388143ddff747ea8041beff00324a9c49b3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 29 Dec 2020 19:30:23 +0100 Subject: [PATCH 16/39] workflows,scripts: add separate workflow job and script to build and publish releases --- .github/workflows/create-github-release.yml | 36 ------- .github/workflows/master.yml | 97 +++++++++++++++--- scripts/create-release-tag.js | 103 ++++++++++++++++++++ 3 files changed, 185 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/create-github-release.yml create mode 100755 scripts/create-release-tag.js diff --git a/.github/workflows/create-github-release.yml b/.github/workflows/create-github-release.yml deleted file mode 100644 index 8ef3f10f6a..0000000000 --- a/.github/workflows/create-github-release.yml +++ /dev/null @@ -1,36 +0,0 @@ -# New tags and releases are created by changeset https://github.com/atlassian/changesets -name: Create a new release on GitHub when a new tag is created - -on: - push: - tags: - - 'v*' # Push events to matching v*, i.e. v0.4.0, v1.1.0 - -jobs: - build: - name: Create a new release on GitHub when a new tag is created - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: use node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: '12.x' - - - name: Install node dependencies - run: npm install @octokit/rest - - # GITHUB_REF is of the format refs/tags/vA.B.C - # This step extracts vA.B.C from GITHUB_REF - - name: Get the version - id: get_version - run: echo "::set-output name=TAG_NAME::${GITHUB_REF#refs/tags/}" - - # TODO/Note: This will only create a Draft release, which the maintainer can see and publish. - # If the Draft release looks good, modify the step to go ahead publish the release. (By adding the third CLI argument.) - - name: Create release on GitHub - run: node scripts/create-github-release.js ${{ steps.get_version.outputs.TAG_NAME }} - env: - GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4de1ed060a..e9f5e602a5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -70,21 +70,6 @@ jobs: bash <(curl -s https://codecov.io/bash) -f packages/core/coverage/* -F core bash <(curl -s https://codecov.io/bash) -f packages/core-api/coverage/* -F core-api - # Publishes current version of packages that are not already present in the registry - - name: publish - if: matrix.node-version == '12.x' - run: yarn lerna -- publish from-package --yes - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - # Tags the commit with the version in the core package if the tag doesn't exist - - uses: Klemensas/action-autotag@1.2.3 - if: matrix.node-version == '12.x' - with: - GITHUB_TOKEN: '${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }}' - package_root: 'packages/core' - tag_prefix: 'v' - - name: Discord notification if: ${{ failure() }} uses: Ilshidur/action-discord@0.2.0 @@ -92,3 +77,85 @@ jobs: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} with: args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' + + # A separate release build that is only run for commits that are the result of merging the "Version Packages" PR + # We can't re-use the output from the above step, but we'll have a guaranteed node_modules cache and + # only run the build steps that are necessary for publishing + release: + if: ${{ endsWith(github.event.head_commit.message, 'from backstage/changeset-release/master\n\nVersion Packages') }} + needs: build + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.x] + + env: + CI: 'true' + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-v${{ matrix.node-version }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - name: build type declarations + run: yarn tsc:full + + - name: build packages + run: yarn lerna -- run --ignore example-app build + + # Publishes current version of packages that are not already present in the registry + - name: publish + run: yarn lerna -- publish from-package --yes + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + # Creates the next available tag with format "release---[.]" + - name: Create a release tag + id: create_tag + run: node scripts/create-release-tag.js + env: + GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + + # Convert the newly created tag into a release with changelog information + - name: Create release on GitHub + run: node scripts/create-github-release.js ${{ steps.create_tag.outputs.tag_name }} 1 + env: + GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + + # Notify everyone about this great new release :D + - name: Discord notification + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} + TAG_NAME: ${{ steps.create_tag.outputs.tag_name }} + with: + args: 'A new release has been published! https://github.com/backstage/backstage/releases/tag/{{TAG_NAME}}' diff --git a/scripts/create-release-tag.js b/scripts/create-release-tag.js new file mode 100755 index 0000000000..22b374a65e --- /dev/null +++ b/scripts/create-release-tag.js @@ -0,0 +1,103 @@ +#!/usr/bin/env node +/* eslint-disable import/no-extraneous-dependencies */ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This script creates a release on GitHub for the Backstage repository. + * Given a git tag, it identifies the PR created by changesets which is responsible for creating + * the git tag. It then uses the PR description consisting of changelogs for packages as the + * release description. + * + * Example: + * + * Set GITHUB_TOKEN environment variable. + * + * (Dry Run mode, will create a DRAFT release, but will not publish it.) + * (Draft releases are visible to maintainers and do not notify users.) + * $ node scripts/get-release-description v0.4.1 + * + * This will open the git tree at this tag https://github.com/backstage/backstage/tree/v0.4.1 + * It will identify https://github.com/backstage/backstage/pull/3668 as the responsible changeset PR. + * And will use everything in the PR description under "Releases" section. + * + * (Production or GitHub Actions Mode) + * $ node scripts/get-release-description v0.4.1 true + * + * This will do the same steps as above, and will publish the Release with the description. + */ + +const { Octokit } = require('@octokit/rest'); + +const baseOptions = { + owner: 'backstage', + repo: 'backstage', +}; + +async function main() { + const { GITHUB_SHA, GITHUB_TOKEN } = process.env; + if (!GITHUB_SHA) { + throw new Error('GITHUB_SHA is not set'); + } + if (!GITHUB_TOKEN) { + throw new Error('GITHUB_TOKEN is not set'); + } + + const octokit = new Octokit({ auth: GITHUB_TOKEN }); + + const date = new Date(); + const baseTagName = `release-${date.getUTCFullYear()}-${ + date.getUTCMonth() + 1 + }-${date.getUTCDate()}`; + + console.log('Requesting existing tags'); + + const existingTags = await octokit.repos.listTags({ + ...baseOptions, + per_page: 100, + }); + const existingTagNames = existingTags.data.map(obj => obj.name); + + let tagName = baseTagName; + let index = 0; + while (existingTagNames.includes(tagName)) { + index += 1; + tagName = `${baseTagName}.${index}`; + } + + console.log(`Creating release tag ${tagName}`); + + const annotatedTag = await octokit.git.createTag({ + ...baseOptions, + tag: tagName, + message: tagName, + object: GITHUB_SHA, + type: 'commit', + }); + + await octokit.git.createRef({ + ...baseOptions, + ref: `refs/tags/${tagName}`, + sha: annotatedTag.data.sha, + }); + + console.log(`::set-output name=tag_name::${tagName}`); +} + +main().catch(error => { + console.error(error.stack); + process.exit(1); +}); From 73e75ea0a5d92e829abc8b3f0a2cda150a5cf5d6 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 29 Dec 2020 12:47:58 -0800 Subject: [PATCH 17/39] Add changeset --- .changeset/new-horses-protect.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/new-horses-protect.md diff --git a/.changeset/new-horses-protect.md b/.changeset/new-horses-protect.md new file mode 100644 index 0000000000..706011fb2c --- /dev/null +++ b/.changeset/new-horses-protect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Add processor for ingesting AWS accounts from AWS Organizations From 052c083e8eba1e209295ecced9a0b5a17f622530 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 29 Dec 2020 13:00:29 -0800 Subject: [PATCH 18/39] Fix build failures with lint and tsc --- .../src/ingestion/processors/AwsOrganizationProcessor.test.ts | 2 +- .../src/ingestion/processors/AwsOrganizationProcessor.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts index 4fb189e433..61e2d298af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts @@ -29,7 +29,7 @@ describe('AwsOrganizationProcessor', () => { it('generates component entities for accounts', async () => { listAccounts.mockImplementation(() => { return { - promise: async function () { + async promise() { return { Accounts: [ { diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts index 70a74e8443..60b914af9d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts @@ -19,7 +19,7 @@ import { LocationSpec, } from '@backstage/catalog-model'; import AWS, { Organizations } from 'aws-sdk'; -import { Account } from 'aws-sdk/clients/organizations'; +import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; @@ -71,7 +71,7 @@ export class AwsOrganizationProcessor implements CatalogProcessor { let NextToken = undefined; while (isInitialAttempt || NextToken) { isInitialAttempt = false; - const orgAccounts = await this.organizations + const orgAccounts: ListAccountsResponse = await this.organizations .listAccounts({ NextToken }) .promise(); if (orgAccounts.Accounts) { From 5eba5efcdf022a729af3c2f7c45ff0e53f661673 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 29 Dec 2020 14:44:58 -0800 Subject: [PATCH 19/39] Address some review comments --- ...OrganizationCloudAccountProcessor.test.ts} | 8 ++--- ...> AwsOrganizationCloudAccountProcessor.ts} | 34 ++++++------------- .../src/ingestion/processors/index.ts | 2 +- 3 files changed, 16 insertions(+), 28 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/{AwsOrganizationProcessor.test.ts => AwsOrganizationCloudAccountProcessor.test.ts} (88%) rename plugins/catalog-backend/src/ingestion/processors/{AwsOrganizationProcessor.ts => AwsOrganizationCloudAccountProcessor.ts} (83%) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts similarity index 88% rename from plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts index 61e2d298af..cea9310f2c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; +import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; -describe('AwsOrganizationProcessor', () => { +describe('AwsOrganizationCloudAccountProcessor', () => { describe('readLocation', () => { - const processor = new AwsOrganizationProcessor(); - const location = { type: 'aws-organization', target: 'b' }; + const processor = new AwsOrganizationCloudAccountProcessor(); + const location = { type: 'aws-cloud-accounts', target: 'b' }; const emit = jest.fn(); const listAccounts = jest.fn(); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts similarity index 83% rename from plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts rename to plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 60b914af9d..77e5e222dc 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -15,7 +15,6 @@ */ import { ComponentEntityV1alpha1, - Entity, LocationSpec, } from '@backstage/catalog-model'; import AWS, { Organizations } from 'aws-sdk'; @@ -25,9 +24,14 @@ import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; const AWS_ORGANIZATION_REGION = 'us-east-1'; -const LOCATION_TYPE = 'aws-organization'; +const LOCATION_TYPE = 'aws-cloud-accounts'; -export class AwsOrganizationProcessor implements CatalogProcessor { +/** + * A processor for ingesting AWS Accounts from AWS Organizations. + * + * If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object. + */ +export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { organizations: Organizations; constructor() { this.organizations = new AWS.Organizations({ @@ -35,18 +39,6 @@ export class AwsOrganizationProcessor implements CatalogProcessor { }); // Only available in us-east-1 } - async handleError(): Promise { - return undefined; - } - - async postProcessEntity(entity: Entity): Promise { - return entity; - } - - async preProcessEntity(entity: Entity): Promise { - return entity; - } - normalizeName(name: string): string { return name .trim() @@ -68,16 +60,16 @@ export class AwsOrganizationProcessor implements CatalogProcessor { async getAwsAccounts(): Promise { let awsAccounts: Account[] = []; let isInitialAttempt = true; - let NextToken = undefined; - while (isInitialAttempt || NextToken) { + let nextToken = undefined; + while (isInitialAttempt || nextToken) { isInitialAttempt = false; const orgAccounts: ListAccountsResponse = await this.organizations - .listAccounts({ NextToken }) + .listAccounts({ NextToken: nextToken }) .promise(); if (orgAccounts.Accounts) { awsAccounts = awsAccounts.concat(orgAccounts.Accounts); - NextToken = orgAccounts.NextToken; } + nextToken = orgAccounts.NextToken; } return awsAccounts; @@ -124,8 +116,4 @@ export class AwsOrganizationProcessor implements CatalogProcessor { return true; } - - async validateEntityKind(): Promise { - return false; - } } diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 7818c083ee..5574b05bc0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -17,7 +17,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; -export { AwsOrganizationProcessor } from './AwsOrganizationProcessor'; +export { AwsOrganizationProcessor } from './AwsOrganizationCloudAccountProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; From 6cfa4075b7f4509675f69ef094620adec9951513 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 29 Dec 2020 14:52:39 -0800 Subject: [PATCH 20/39] Fix error, add location target support --- .../AwsOrganizationCloudAccountProcessor.ts | 14 ++++++++++++++ .../src/ingestion/processors/index.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 77e5e222dc..826ce39a6c 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -26,6 +26,8 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; const AWS_ORGANIZATION_REGION = 'us-east-1'; const LOCATION_TYPE = 'aws-cloud-accounts'; +const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id'; + /** * A processor for ingesting AWS Accounts from AWS Organizations. * @@ -110,6 +112,18 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { (await this.getAwsAccounts()) .map(account => this.mapAccountToComponent(account)) + .filter(entity => { + if (location.target !== '') { + if (entity.metadata.annotations) { + return ( + entity.metadata.annotations[ORGANIZATION_ANNOTATION] === + location.type + ); + } + return false; + } + return true; + }) .forEach((entity: ComponentEntityV1alpha1) => { emit(results.entity(location, entity)); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 5574b05bc0..a7b00d7065 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -17,7 +17,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; -export { AwsOrganizationProcessor } from './AwsOrganizationCloudAccountProcessor'; +export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; From 205638b5acd6f93095cb22414e9f9577c7d96f1e Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 29 Dec 2020 15:24:04 -0800 Subject: [PATCH 21/39] Fix test --- ...sOrganizationCloudAccountProcessor.test.ts | 52 ++++++++++++++++++- .../AwsOrganizationCloudAccountProcessor.ts | 12 +++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts index cea9310f2c..eb5e634d66 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -19,7 +19,7 @@ import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAcco describe('AwsOrganizationCloudAccountProcessor', () => { describe('readLocation', () => { const processor = new AwsOrganizationCloudAccountProcessor(); - const location = { type: 'aws-cloud-accounts', target: 'b' }; + const location = { type: 'aws-cloud-accounts', target: '' }; const emit = jest.fn(); const listAccounts = jest.fn(); @@ -68,5 +68,55 @@ describe('AwsOrganizationCloudAccountProcessor', () => { }, }); }); + + it('filters out accounts not in specified location target', async () => { + const location = { type: 'aws-cloud-accounts', target: 'o-1vl18kc5a3' }; + listAccounts.mockImplementation(() => { + return { + async promise() { + return { + Accounts: [ + { + Arn: + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + Name: 'testaccount', + }, + { + Arn: + 'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395', + Name: 'testaccount2', + }, + ], + NextToken: undefined, + }; + }, + }; + }); + await processor.readLocation(location, false, emit); + expect(emit).toBeCalledTimes(1); + expect(emit).toBeCalledWith({ + type: 'entity', + location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + 'amazonaws.com/account-id': '957140518395', + 'amazonaws.com/organization-id': 'o-1vl18kc5a3', + }, + name: 'testaccount', + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }, + }); + }); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts index 826ce39a6c..f019f5207b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -26,7 +26,9 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types'; const AWS_ORGANIZATION_REGION = 'us-east-1'; const LOCATION_TYPE = 'aws-cloud-accounts'; -const ORGANIZATION_ANNOTATION = 'amazonaws.com/organization-id'; +const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id'; +const ARN_ANNOTATION: string = 'amazonaws.com/arn'; +const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id'; /** * A processor for ingesting AWS Accounts from AWS Organizations. @@ -86,9 +88,9 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { kind: 'Component', metadata: { annotations: { - 'amazonaws.com/arn': account.Arn || '', - 'amazonaws.com/account-id': accountId, - 'amazonaws.com/organization-id': organizationId, + [ACCOUNTID_ANNOTATION]: accountId, + [ARN_ANNOTATION]: account.Arn || '', + [ORGANIZATION_ANNOTATION]: organizationId, }, name: this.normalizeName(account.Name || ''), namespace: 'default', @@ -117,7 +119,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { if (entity.metadata.annotations) { return ( entity.metadata.annotations[ORGANIZATION_ANNOTATION] === - location.type + location.target ); } return false; From 239b29dae444c4826d0cbb1e56054362e9178fb0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Dec 2020 00:27:55 +0100 Subject: [PATCH 22/39] Update create-release-tag.js --- scripts/create-release-tag.js | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/scripts/create-release-tag.js b/scripts/create-release-tag.js index 22b374a65e..5c57c9b278 100755 --- a/scripts/create-release-tag.js +++ b/scripts/create-release-tag.js @@ -16,30 +16,6 @@ * limitations under the License. */ -/** - * This script creates a release on GitHub for the Backstage repository. - * Given a git tag, it identifies the PR created by changesets which is responsible for creating - * the git tag. It then uses the PR description consisting of changelogs for packages as the - * release description. - * - * Example: - * - * Set GITHUB_TOKEN environment variable. - * - * (Dry Run mode, will create a DRAFT release, but will not publish it.) - * (Draft releases are visible to maintainers and do not notify users.) - * $ node scripts/get-release-description v0.4.1 - * - * This will open the git tree at this tag https://github.com/backstage/backstage/tree/v0.4.1 - * It will identify https://github.com/backstage/backstage/pull/3668 as the responsible changeset PR. - * And will use everything in the PR description under "Releases" section. - * - * (Production or GitHub Actions Mode) - * $ node scripts/get-release-description v0.4.1 true - * - * This will do the same steps as above, and will publish the Release with the description. - */ - const { Octokit } = require('@octokit/rest'); const baseOptions = { From 3f7bfe7b1153e2477587669bb8234b98e19b6bd7 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Wed, 30 Dec 2020 13:11:35 +1300 Subject: [PATCH 23/39] remove url check to support on-prem ADO --- .../backend-common/src/reading/AzureUrlReader.ts | 8 +------- packages/integration/src/azure/core.test.ts | 12 ++++++++++++ packages/integration/src/azure/core.ts | 1 - 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index ca934ee42c..ea716a1d4a 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -47,13 +47,7 @@ export class AzureUrlReader implements UrlReader { constructor( private readonly options: AzureIntegrationConfig, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, - ) { - if (options.host !== 'dev.azure.com') { - throw Error( - `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, - ); - } - } + ) {} async read(url: string): Promise { const builtUrl = getAzureFileFetchUrl(url); diff --git a/packages/integration/src/azure/core.test.ts b/packages/integration/src/azure/core.test.ts index 17041a9eb2..438e776eef 100644 --- a/packages/integration/src/azure/core.test.ts +++ b/packages/integration/src/azure/core.test.ts @@ -54,6 +54,18 @@ describe('azure core', () => { result: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', }, + { + url: + 'https://api.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + result: + 'https://api.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }, + { + url: + 'https://api.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + result: + 'https://api.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }, ])('should handle happy path %#', async ({ url, result }) => { expect(getAzureFileFetchUrl(url)).toBe(result); }); diff --git a/packages/integration/src/azure/core.ts b/packages/integration/src/azure/core.ts index 44591f2509..7900774c79 100644 --- a/packages/integration/src/azure/core.ts +++ b/packages/integration/src/azure/core.ts @@ -43,7 +43,6 @@ export function getAzureFileFetchUrl(url: string): string { const ref = parsedUrl.searchParams.get('version')?.substr(2); if ( - parsedUrl.hostname !== 'dev.azure.com' || empty !== '' || userOrOrg === '' || project === '' || From c15c038919c9cdc6f0f391e248972cd16fd356ec Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Wed, 30 Dec 2020 13:21:08 +1300 Subject: [PATCH 24/39] rempove old changeset file --- .changeset/wicked-impalas-tan.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/wicked-impalas-tan.md diff --git a/.changeset/wicked-impalas-tan.md b/.changeset/wicked-impalas-tan.md deleted file mode 100644 index 8cfe919f5c..0000000000 --- a/.changeset/wicked-impalas-tan.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/plugin-tech-radar': minor ---- - -Added tech radar blip history backend support and normalized the data structure From 23c52781af79a46cde99b5f4b0c5f2f845e8679a Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 23:59:34 -0500 Subject: [PATCH 25/39] Remove typos in vocab --- .github/styles/vocab.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index e2b7f36067..7f92035c5a 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -62,7 +62,6 @@ Docusaurus Dominik dtuite dzolotusky -eg Ek env Env @@ -140,7 +139,7 @@ nonces npm nvm oauth -Oauth +OAuth oidc Okta Oldsberg @@ -159,7 +158,6 @@ prebaked preconfigured prepack Preprarer -Prerequisities productional Protobuf proxying @@ -193,7 +191,6 @@ semlas semver Serverless Sinon -smartsymobls Snyk sourcemaps sparklines @@ -220,7 +217,6 @@ Templater templaters Templaters Thauer -theres toc tolerations Tolerations From 64b8d49fcdd17d772f85782223a9e52efb00cde5 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Tue, 29 Dec 2020 23:59:41 -0500 Subject: [PATCH 26/39] Fix typo --- docs/features/techdocs/creating-and-publishing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 38f452dcf3..187fe97aed 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -41,7 +41,7 @@ setup for free. ### Manually add documentation setup to already existing repository -Prerequisities: +Prerequisites: - An existing component [registered in backstage](../software-catalog/index.md#adding-components-to-the-catalog) From 457fb1df8737bf234fab0ebef5a283ce7d80a69e Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 30 Dec 2020 00:00:32 -0500 Subject: [PATCH 27/39] Fix ref to exempli gratia --- CODE_OF_CONDUCT.md | 2 +- docs/features/software-catalog/system-model.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 55269dd2a5..6990f72c3f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -18,7 +18,7 @@ Harassment includes, but is not limited to: - Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation - Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment - Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle -- Physical contact and simulated physical contact (eg, textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop +- Physical contact and simulated physical contact (e.g., textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop - Threats of violence, both physical and psychological - Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm - Deliberate intimidation diff --git a/docs/features/software-catalog/system-model.md b/docs/features/software-catalog/system-model.md index 53f49d5df9..d797e87499 100644 --- a/docs/features/software-catalog/system-model.md +++ b/docs/features/software-catalog/system-model.md @@ -44,8 +44,8 @@ Backstage model and the primary way to discover existing functionality in the ecosystem. APIs are implemented by components and form boundaries between components. They -might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a data schema (eg -Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by +might be defined using an RPC IDL (e.g., Protobuf, GraphQL, ...), a data schema +(e.g., Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by components need to be in a known machine-readable format so we can build further tooling and analysis on top. From 11e34f12cb377475ae883723e105097830d51a43 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 30 Dec 2020 00:01:58 -0500 Subject: [PATCH 28/39] Remove completed TODO --- docs/features/techdocs/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 50cadc7b27..f4e89d8f6f 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -45,8 +45,6 @@ about TechDocs and the philosophy in its [v2]: https://github.com/backstage/backstage/milestone/22 [v3]: https://github.com/backstage/backstage/milestone/17 - - ## Use Cases #### TechDocs V.0 From 056336fb1287f379a7b5f375aef10302d94ae3db Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 30 Dec 2020 01:17:05 -0500 Subject: [PATCH 29/39] Add code fence --- docs/plugins/integrating-plugin-into-service-catalog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/integrating-plugin-into-service-catalog.md b/docs/plugins/integrating-plugin-into-service-catalog.md index 51d2331140..bc93cef63b 100644 --- a/docs/plugins/integrating-plugin-into-service-catalog.md +++ b/docs/plugins/integrating-plugin-into-service-catalog.md @@ -20,7 +20,7 @@ should have a separate package in a folder, which represents your plugin. Example: -``` +```sh $ yarn create-plugin > ? Enter an ID for the plugin [required] my-plugin > ? Enter the owner(s) of the plugin. If specified, this will be added to CODEOWNERS for the plugin path. [optional] From 33bfa457966383ec3d13fa92978d972a2ce97c2f Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 30 Dec 2020 01:17:20 -0500 Subject: [PATCH 30/39] Clean up testing instructions --- docs/plugins/testing.md | 75 +++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/docs/plugins/testing.md b/docs/plugins/testing.md index b564fd732c..30c3bf0ecc 100644 --- a/docs/plugins/testing.md +++ b/docs/plugins/testing.md @@ -16,15 +16,15 @@ frameworks and libraries like [Mocha](https://mochajs.org/), Running all tests: - yarn test-react + yarn test Running an individual test (e.g. `MyComponent.test.js`): - yarn test-react MyComponent + yarn test MyComponent To run both `MyComponent.test.js` and `MyControl.test.js` suite of tests: - yarn test-react MyCo + yarn test MyCo Note: if `console.logs` are not appearing, run only the individual test you are working on. @@ -52,12 +52,12 @@ render React components. TODO. -# Writing Unit Tests +## Writing Unit Tests The following principles are good guides for determining if you are writing high quality frontend unit tests. -## Bad Unit Test Principle +### Bad Unit Test Principle > No unit test is better than a bad one. @@ -69,7 +69,7 @@ Writing a poor unit test: - Adds to future work by requiring updates to the unit test for irrelevant code changes. -## Input/Output Principle +### Input/Output Principle > A unit test verifies an output matches an expected input. @@ -77,7 +77,7 @@ For backend, this would be that when you provide configuration X, then the object responds with Y. For frontend, this would be that when you provide properties X to a component, then the visual functionality responds with Y. -## Blackbox Principle +### Blackbox Principle > A good unit test does not tell the object how it should do its job but should > only compare inputs to outputs. @@ -86,7 +86,7 @@ Consider a unit test for a form. A good unit test would not test the order of the form fields. Instead, it would verify that the inputs to the form fields lead to a certain backend call when submit is clicked. -## Scalability Principle +### Scalability Principle > Unit test quality is directly proportionate to how much code can change > without having to touch the unit test. @@ -97,7 +97,7 @@ to the code, you have to update the unit test. A good unit test suite allows a lot of flexibility in _how_ the code is written so that future refactoring can occur without having to touch the original unit tests. -## Increasing Complexity Principle +### Increasing Complexity Principle > The ordering of unit tests in a suite should proceed from least specific to > most specific. @@ -116,7 +116,7 @@ throwing an error saying that output was incorrect will lead the next developer into thinking they may have broken the entire functionality of the object rather than simply letting them know they had an invalid input. -## Broken Functionality Principle +### Broken Functionality Principle > Generally, a unit test should not test exactly how the output appears, it > should test that the functionality has an expected _general_ response to an @@ -131,7 +131,7 @@ test a slightly different color on the button the unit test will break. A better unit test would verify that the button's CSS classname is assigned properly on hover or test for something completely different. -## Example: Loading Indicator +### Example: Loading Indicator A classic unit test on frontends is verifying a loading indicator displays when a backend request is being made. @@ -192,11 +192,14 @@ returns a result or displays an error or console message, like so: **`StringUtil ellipsis`** - export function ellipsis(text, maxLength, midCharIx = 0, ellipsis = '...') { - // Do something blackbox. We should not care about the internals, only inputs and outputs. - ... - return someFinalValue; - } +```js +export function ellipsis(text, maxLength, midCharIx = 0, ellipsis = '...') { + // Do something blackbox. We should not care about the internals, + // only inputs and outputs. + ... + return someFinalValue; +} +``` There are four things to test for in a utility function: @@ -207,30 +210,36 @@ There are four things to test for in a utility function: > Handle Invalid Input (handle thrown errors): - it('Throws an error on improper arguments', () => { - expect(() => { - ellipsis(); - }).toThrowError('Expected \'text\' to be defined'); - }); +```js +it('Throws an error on improper arguments', () => { + expect(() => { + ellipsis(); + }).toThrowError("Expected 'text' to be defined"); +}); +``` > Verify default input arguments: - it('Works with defaults', () => { - expect(ellipsis('Hello world', 3)).toBe('Hel...'); - expect(ellipsis('', 3)).toBe(''); - expect(ellipsis('H', 3)).toBe('H'); - expect(ellipsis('Hello', 5)).toBe('Hello'); - }); +```js +it('Works with defaults', () => { + expect(ellipsis('Hello world', 3)).toBe('Hel...'); + expect(ellipsis('', 3)).toBe(''); + expect(ellipsis('H', 3)).toBe('H'); + expect(ellipsis('Hello', 5)).toBe('Hello'); +}); +``` > Verify output for expected input arguments: This is especially true for edge cases! - it('Works with midCharIx', () => { - expect(ellipsis('Hello world', 3, 6)).toBe('...o w...'); - expect(ellipsis('', 3, 6)).toBe(''); - expect(ellipsis('Backstage is amazing', 4, 10)).toBe('...e is...'); - }); +```js +it('Works with midCharIx', () => { + expect(ellipsis('Hello world', 3, 6)).toBe('...o w...'); + expect(ellipsis('', 3, 6)).toBe(''); + expect(ellipsis('Backstage is amazing', 4, 10)).toBe('...e is...'); +}); +``` ## Non-React Classes @@ -372,4 +381,4 @@ IDE. In most cases, we have found that using `console.log` works well. Note: if your console.logs are not being displayed, focus your specific unit -test from the command line by running them like so `yarn test-react MyTest`. +test from the command line by running them like so `yarn test MyTest`. From 533968130f0127a8413519baa4d6ef005cefb525 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 30 Dec 2020 01:20:34 -0500 Subject: [PATCH 31/39] Reformat without shell --- docs/plugins/integrating-plugin-into-service-catalog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/plugins/integrating-plugin-into-service-catalog.md b/docs/plugins/integrating-plugin-into-service-catalog.md index bc93cef63b..51d2331140 100644 --- a/docs/plugins/integrating-plugin-into-service-catalog.md +++ b/docs/plugins/integrating-plugin-into-service-catalog.md @@ -20,7 +20,7 @@ should have a separate package in a folder, which represents your plugin. Example: -```sh +``` $ yarn create-plugin > ? Enter an ID for the plugin [required] my-plugin > ? Enter the owner(s) of the plugin. If specified, this will be added to CODEOWNERS for the plugin path. [optional] From 532672460a1a3df7f6dc5f9be1d4f04f0d3c5d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 30 Dec 2020 09:58:54 +0100 Subject: [PATCH 32/39] catalog-backend: unbreak sqlite in create-app repos --- .../migrations/20200702153613_entities.js | 12 ++++-------- .../migrations/20200807120600_entitySearch.js | 10 ++++------ .../20201005122705_add_entity_full_name.js | 5 ++--- .../migrations/20201006130744_entity_data_column.js | 5 +---- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js index c97331796d..9acd7564db 100644 --- a/plugins/catalog-backend/migrations/20200702153613_entities.js +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -20,16 +20,14 @@ * @param {import('knex')} knex */ exports.up = async function up(knex) { - // Drop constraints (Postgres) - try { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); await knex.schema.alterTable('entities', table => { table.dropPrimary('entities_pkey'); }); - } catch (e) { - // SQLite does not support FK and PK, carry on } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); @@ -131,16 +129,14 @@ exports.up = async function up(knex) { * @param {import('knex')} knex */ exports.down = async function down(knex) { - // Drop constraints (Postgres) - try { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); await knex.schema.alterTable('entities', table => { table.dropPrimary('entities_pkey'); }); - } catch (e) { - // SQLite does not support FK and PK, carry on } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js index 6e02975f92..b3a9673641 100644 --- a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -20,12 +20,11 @@ * @param {import('knex')} knex */ exports.up = async function up(knex) { - try { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.text('value').nullable().alter(); }); - } catch (e) { - // Sqlite does not support alter column. } }; @@ -33,11 +32,10 @@ exports.up = async function up(knex) { * @param {import('knex')} knex */ exports.down = async function down(knex) { - try { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.string('value').nullable().alter(); }); - } catch (e) { - // Sqlite does not support alter column. } }; diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 26cc98e74e..4c1ea76de4 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -30,12 +30,11 @@ exports.up = async function up(knex) { ), }); - try { + // SQLite does not support alter column + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities', table => { table.text('full_name').notNullable().alter(); }); - } catch (e) { - // SQLite does not support alter column, ignore } await knex.schema.alterTable('entities', table => { diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js index fdabf72fce..a326a08a8b 100644 --- a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -40,10 +40,7 @@ exports.up = async function up(knex) { table.dropColumn('spec'); }); - // SQLite does not support ALTER COLUMN. Note that we do not use the try/ - // catch method as in other migrations, because if the transaction is - // partially failed, it will further mess up the already messed-up - // statement below this. + // SQLite does not support ALTER COLUMN. if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities', table => { table.text('data').notNullable().alter(); From 9d998b4a2a9c4208554f016f73cb065fb4d8ffd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 30 Dec 2020 10:52:58 +0100 Subject: [PATCH 33/39] catalog-backend: stop it with the varchar(255) --- .../20201230103504_update_log_varchar.js | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js diff --git a/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js new file mode 100644 index 0000000000..71f9500cf2 --- /dev/null +++ b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + // We actually just want to widen columns, but can't do that while a + // view is dependent on them - so we just reconstruct it exactly as it was + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.text('message').alter(); + table.text('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.string('message').alter(); + table.string('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; From cabf2cbbae1f1e160d922918da5d747fb4ce64a5 Mon Sep 17 00:00:00 2001 From: Niall McCullagh Date: Wed, 30 Dec 2020 10:27:46 +0000 Subject: [PATCH 34/39] docs(techradar-plugin) Update loader resp example Updates the tech-radar loader response example to reflect changes in the schema for supporting event history tracking. --- plugins/tech-radar/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/tech-radar/README.md b/plugins/tech-radar/README.md index cbe4dd22eb..21f90ec432 100644 --- a/plugins/tech-radar/README.md +++ b/plugins/tech-radar/README.md @@ -83,13 +83,20 @@ const getHardCodedData = () => rings: [{ id: 'use', name: 'USE', color: '#93c47d' }], entries: [ { - moved: 0, - ring: 'use', url: '#', key: 'github-actions', id: 'github-actions', title: 'GitHub Actions', quadrant: 'infrastructure', + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + description: + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat', + }, + ], }, ], }); From 751f58ff7cf14d5a4bd737ca35b75b31e23ad11f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Dec 2020 12:14:41 +0100 Subject: [PATCH 35/39] backend-common: replace generated cert params with a simple bool switch, and fix certificate generation --- packages/backend-common/config.d.ts | 55 ++++++++++------ .../backend-common/src/service/lib/config.ts | 26 ++++++-- .../src/service/lib/hostFactory.ts | 64 +++++++++++++++++-- 3 files changed, 115 insertions(+), 30 deletions(-) diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 3e21235c63..b7241bcc03 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -32,26 +32,41 @@ export interface Config { port?: string | number; }; - /** HTTPS configuration for the backend. If omitted the backend will serve HTTP */ - https?: { - /** Certificate configuration or parameters for generating a self-signed certificate */ - certificate?: - | { - /** Algorithm to use to generate a self-signed certificate */ - algorithm: string; - keySize?: number; - days?: number; - } - | { - /** PEM encoded certificate. Use $file to load in a file */ - cert: string; - /** - * PEM encoded certificate key. Use $file to load in a file. - * @visibility secret - */ - key: string; - }; - }; + /** + * HTTPS configuration for the backend. If omitted the backend will serve HTTP. + * + * Setting this to `true` will cause self-signed certificates to be generated, which + * can be useful for local development or other non-production scenarios. + */ + https?: + | true + | { + /** + * Certificate configuration or parameters for generating a self-signed certificate + * + * Setting parameters for self-signed certificates is deprecated and will be removed in + * the future, set `backend.https = true` instead. + */ + certificate?: + | { + /** Algorithm to use to generate a self-signed certificate */ + algorithm?: string; + keySize?: number; + days?: number; + attributes: { + commonName: string; + }; + } + | { + /** PEM encoded certificate. Use $file to load in a file */ + cert: string; + /** + * PEM encoded certificate key. Use $file to load in a file. + * @visibility secret + */ + key: string; + }; + }; /** Database connection configuration, select database type using the `client` field */ database: diff --git a/packages/backend-common/src/service/lib/config.ts b/packages/backend-common/src/service/lib/config.ts index ea1a2b0887..6abea97454 100644 --- a/packages/backend-common/src/service/lib/config.ts +++ b/packages/backend-common/src/service/lib/config.ts @@ -47,14 +47,14 @@ export type CertificateReferenceOptions = { }; export type CertificateSigningOptions = { - algorithm: string; + algorithm?: string; size?: number; days?: number; - attributes?: CertificateAttributes; + attributes: CertificateAttributes; }; export type CertificateAttributes = { - commonName?: string; + commonName: string; }; /** @@ -193,8 +193,26 @@ export function readCspOptions( * ``` */ export function readHttpsSettings(config: Config): HttpsSettings | undefined { - const cc = config.getOptionalConfig('https'); + const https = config.get('https'); + if (https === true) { + const baseUrl = config.getString('baseUrl'); + let commonName; + try { + commonName = new URL(baseUrl).hostname; + } catch (error) { + throw new Error(`Invalid backend.baseUrl "${baseUrl}"`); + } + return { + certificate: { + attributes: { + commonName, + }, + }, + }; + } + + const cc = config.getOptionalConfig('https'); if (!cc) { return undefined; } diff --git a/packages/backend-common/src/service/lib/hostFactory.ts b/packages/backend-common/src/service/lib/hostFactory.ts index 8fab2fd4ab..f190b847b6 100644 --- a/packages/backend-common/src/service/lib/hostFactory.ts +++ b/packages/backend-common/src/service/lib/hostFactory.ts @@ -59,26 +59,78 @@ export function createHttpsServer( const signingOptions: any = httpsSettings?.certificate; - if (signingOptions?.algorithm !== undefined) { + if (signingOptions?.attributes) { logger?.info('Generating self-signed certificate with attributes'); + if (signingOptions?.algorithm) { + logger?.warn( + 'Certificate generation configuration with parameters in backend.https.certificate is deprecated, set backend.https = true instead', + ); + } const certificateAttributes: Array = Object.entries( signingOptions.attributes, ).map(([name, value]) => ({ name, value })); - // TODO: Create a type def for selfsigned. const signatures = require('selfsigned').generate(certificateAttributes, { - algorithm: signingOptions?.algorithm, + algorithm: signingOptions?.algorithm || 'sha256', keySize: signingOptions?.size || 2048, days: signingOptions?.days || 30, + extensions: [ + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + nonRepudiation: true, + keyEncipherment: true, + dataEncipherment: true, + }, + { + name: 'extKeyUsage', + serverAuth: true, + clientAuth: true, + codeSigning: true, + timeStamping: true, + }, + { + name: 'subjectAltName', + altNames: [ + { + type: 2, // DNS + value: 'localhost', + }, + { + type: 2, + value: 'localhost.localdomain', + }, + { + type: 2, + value: '[::1]', + }, + { + type: 7, // IP + ip: '127.0.0.1', + }, + { + type: 7, + ip: 'fe80::1', + }, + ...(signingOptions.attributes.commonName + ? [ + { + type: 2, // DNS + value: signingOptions.attributes.commonName, + }, + ] + : []), + ], + }, + ], }); - logger?.info('Bootstrapping self-signed certificate'); - credentials.key = signatures.private; credentials.cert = signatures.cert; } else { - logger?.info('Bootstrapping cert from config'); + logger?.info('Loading certificate from config'); credentials.key = signingOptions?.key; credentials.cert = signingOptions?.cert; From 9db9bc878f0665e26286a3b2a7ba40ef9acf493e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Dec 2020 12:46:58 +0100 Subject: [PATCH 36/39] backend-common: cache generated HTTPS certificates --- .../src/service/lib/ServiceBuilderImpl.ts | 10 +- .../src/service/lib/hostFactory.ts | 217 +++++++++++------- 2 files changed, 142 insertions(+), 85 deletions(-) diff --git a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts index 46bf23beef..9c5ac20fa3 100644 --- a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts +++ b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts @@ -145,7 +145,7 @@ export class ServiceBuilderImpl implements ServiceBuilder { return this; } - start(): Promise { + async start(): Promise { const app = express(); const { port, @@ -168,16 +168,16 @@ export class ServiceBuilderImpl implements ServiceBuilder { app.use(notFoundHandler()); app.use(errorHandler()); + const server: http.Server = httpsSettings + ? await createHttpsServer(app, httpsSettings, logger) + : createHttpServer(app, logger); + return new Promise((resolve, reject) => { app.on('error', e => { logger.error(`Failed to start up on port ${port}, ${e}`); reject(e); }); - const server: http.Server = httpsSettings - ? createHttpsServer(app, httpsSettings, logger) - : createHttpServer(app, logger); - const stoppableServer = stoppable( server.listen(port, host, () => { logger.info(`Listening on ${host}:${port}`); diff --git a/packages/backend-common/src/service/lib/hostFactory.ts b/packages/backend-common/src/service/lib/hostFactory.ts index f190b847b6..656f160c31 100644 --- a/packages/backend-common/src/service/lib/hostFactory.ts +++ b/packages/backend-common/src/service/lib/hostFactory.ts @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; import express from 'express'; import * as http from 'http'; import * as https from 'https'; import { Logger } from 'winston'; -import { HttpsSettings } from './config'; +import { CertificateSigningOptions, HttpsSettings } from './config'; + +const ALMOST_MONTH_IN_MS = 25 * 24 * 60 * 60 * 1000; /** * Creates a Http server instance based on an Express application. @@ -45,100 +50,152 @@ export function createHttpServer( * @returns A Https server instance * */ -export function createHttpsServer( +export async function createHttpsServer( app: express.Express, httpsSettings: HttpsSettings, logger?: Logger, -): http.Server { +): Promise { logger?.info('Initializing https server'); - const credentials: { key: string; cert: string } = { - key: '', - cert: '', - }; + let credentials: { key: string | Buffer; cert: string | Buffer }; const signingOptions: any = httpsSettings?.certificate; + // TODO(Rugvip): remove support for generated certificate params and make this a more straightforward check if (signingOptions?.attributes) { - logger?.info('Generating self-signed certificate with attributes'); - if (signingOptions?.algorithm) { - logger?.warn( - 'Certificate generation configuration with parameters in backend.https.certificate is deprecated, set backend.https = true instead', - ); - } - - const certificateAttributes: Array = Object.entries( - signingOptions.attributes, - ).map(([name, value]) => ({ name, value })); - - const signatures = require('selfsigned').generate(certificateAttributes, { - algorithm: signingOptions?.algorithm || 'sha256', - keySize: signingOptions?.size || 2048, - days: signingOptions?.days || 30, - extensions: [ - { - name: 'keyUsage', - keyCertSign: true, - digitalSignature: true, - nonRepudiation: true, - keyEncipherment: true, - dataEncipherment: true, - }, - { - name: 'extKeyUsage', - serverAuth: true, - clientAuth: true, - codeSigning: true, - timeStamping: true, - }, - { - name: 'subjectAltName', - altNames: [ - { - type: 2, // DNS - value: 'localhost', - }, - { - type: 2, - value: 'localhost.localdomain', - }, - { - type: 2, - value: '[::1]', - }, - { - type: 7, // IP - ip: '127.0.0.1', - }, - { - type: 7, - ip: 'fe80::1', - }, - ...(signingOptions.attributes.commonName - ? [ - { - type: 2, // DNS - value: signingOptions.attributes.commonName, - }, - ] - : []), - ], - }, - ], - }); - - credentials.key = signatures.private; - credentials.cert = signatures.cert; + credentials = await getGeneratedCertificate(signingOptions, logger); } else { logger?.info('Loading certificate from config'); - credentials.key = signingOptions?.key; - credentials.cert = signingOptions?.cert; + credentials = { + key: signingOptions?.key, + cert: signingOptions?.cert, + }; } - if (credentials.key === '' || credentials.cert === '') { - throw new Error('Invalid credentials'); + if (!credentials.key || !credentials.cert) { + throw new Error('Invalid HTTPS credentials'); } return https.createServer(credentials, app) as http.Server; } + +async function getGeneratedCertificate( + options: CertificateSigningOptions, + logger?: Logger, +) { + if (options?.algorithm) { + logger?.warn( + 'Certificate generation configuration with parameters in backend.https.certificate is deprecated, set backend.https = true instead', + ); + } + + const hasModules = await fs.pathExists('node_modules'); + let certPath; + if (hasModules) { + certPath = resolvePath( + 'node_modules/.cache/backstage-backend/dev-cert.pem', + ); + await fs.ensureDir(dirname(certPath)); + } else { + certPath = resolvePath('.dev-cert.pem'); + } + + let cert = undefined; + if (await fs.pathExists(certPath)) { + const stat = await fs.stat(certPath); + const ageMs = Date.now() - stat.ctimeMs; + if (stat.isFile() && ageMs < ALMOST_MONTH_IN_MS) { + cert = await fs.readFile(certPath); + } + } + + if (cert) { + logger?.info('Using existing self-signed certificate'); + return { + key: cert, + cert: cert, + }; + } + + logger?.info('Generating new self-signed certificate'); + const newCert = await createCertificate(options); + await fs.writeFile(certPath, newCert.cert + newCert.key, 'utf8'); + return newCert; +} + +async function createCertificate(options: CertificateSigningOptions) { + const attributes: Array = Object.entries( + options.attributes, + ).map(([name, value]) => ({ name, value })); + + const params = { + algorithm: options?.algorithm || 'sha256', + keySize: options?.size || 2048, + days: options?.days || 30, + extensions: [ + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + nonRepudiation: true, + keyEncipherment: true, + dataEncipherment: true, + }, + { + name: 'extKeyUsage', + serverAuth: true, + clientAuth: true, + codeSigning: true, + timeStamping: true, + }, + { + name: 'subjectAltName', + altNames: [ + { + type: 2, // DNS + value: 'localhost', + }, + { + type: 2, + value: 'localhost.localdomain', + }, + { + type: 2, + value: '[::1]', + }, + { + type: 7, // IP + ip: '127.0.0.1', + }, + { + type: 7, + ip: 'fe80::1', + }, + ...(options.attributes.commonName + ? [ + { + type: 2, // DNS + value: options.attributes.commonName, + }, + ] + : []), + ], + }, + ], + }; + + return new Promise<{ key: string; cert: string }>((resolve, reject) => + require('selfsigned').generate( + attributes, + params, + (err: Error, bundle: { private: string; cert: string }) => { + if (err) { + reject(err); + } else { + resolve({ key: bundle.private, cert: bundle.cert }); + } + }, + ), + ); +} From 5ecd50f8a0fe86926606c9c9256473079b8b95f8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Dec 2020 12:51:14 +0100 Subject: [PATCH 37/39] changeset: add changeset for backend https certificate generation --- .changeset/cyan-lizards-confess.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cyan-lizards-confess.md diff --git a/.changeset/cyan-lizards-confess.md b/.changeset/cyan-lizards-confess.md new file mode 100644 index 0000000000..17ff8f0cdf --- /dev/null +++ b/.changeset/cyan-lizards-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Fix HTTPS certificate generation and add new config switch, enabling it simply by setting `backend.https = true`. Also introduces caching of generated certificates in order to avoid having to add a browser override every time the backend is restarted. From 036a84373f2848d1cec5e94e955a3806491c44a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 30 Dec 2020 12:12:34 +0100 Subject: [PATCH 38/39] add forgotten changeset --- .changeset/odd-eyes-beg.md | 6 ++++++ .github/styles/vocab.txt | 1 + 2 files changed, 7 insertions(+) create mode 100644 .changeset/odd-eyes-beg.md diff --git a/.changeset/odd-eyes-beg.md b/.changeset/odd-eyes-beg.md new file mode 100644 index 0000000000..25a7699516 --- /dev/null +++ b/.changeset/odd-eyes-beg.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Provide support for on-prem azure devops diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index e2b7f36067..57827b48c9 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -49,6 +49,7 @@ dataflow deadnaming destructured dev +devops devs dhenneke discoverability From bc909178d7b1c4c39275b72cfd9bd25e832eceed Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 30 Dec 2020 13:57:55 +0100 Subject: [PATCH 39/39] Create tough-jars-share.md --- .changeset/tough-jars-share.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tough-jars-share.md diff --git a/.changeset/tough-jars-share.md b/.changeset/tough-jars-share.md new file mode 100644 index 0000000000..f1b551f31e --- /dev/null +++ b/.changeset/tough-jars-share.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': patch +--- + +Updated example data in `README`.