diff --git a/.changeset/cuddly-files-argue.md b/.changeset/cuddly-files-argue.md new file mode 100644 index 0000000000..d495a945f4 --- /dev/null +++ b/.changeset/cuddly-files-argue.md @@ -0,0 +1,5 @@ +--- +'@backstage/create-app': patch +--- + +Replace `register-component` plugin with new `catalog-import` plugin diff --git a/.changeset/cyan-lizards-confess.md b/.changeset/cyan-lizards-confess.md new file mode 100644 index 0000000000..17ff8f0cdf --- /dev/null +++ b/.changeset/cyan-lizards-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Fix HTTPS certificate generation and add new config switch, enabling it simply by setting `backend.https = true`. Also introduces caching of generated certificates in order to avoid having to add a browser override every time the backend is restarted. diff --git a/.changeset/new-horses-protect.md b/.changeset/new-horses-protect.md new file mode 100644 index 0000000000..706011fb2c --- /dev/null +++ b/.changeset/new-horses-protect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Add processor for ingesting AWS accounts from AWS Organizations diff --git a/.changeset/odd-eyes-beg.md b/.changeset/odd-eyes-beg.md new file mode 100644 index 0000000000..25a7699516 --- /dev/null +++ b/.changeset/odd-eyes-beg.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +--- + +Provide support for on-prem azure devops diff --git a/.changeset/tough-jars-share.md b/.changeset/tough-jars-share.md new file mode 100644 index 0000000000..f1b551f31e --- /dev/null +++ b/.changeset/tough-jars-share.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-radar': patch +--- + +Updated example data in `README`. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index e2b7f36067..dcc41f44ab 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -49,6 +49,7 @@ dataflow deadnaming destructured dev +devops devs dhenneke discoverability @@ -62,7 +63,6 @@ Docusaurus Dominik dtuite dzolotusky -eg Ek env Env @@ -140,7 +140,7 @@ nonces npm nvm oauth -Oauth +OAuth oidc Okta Oldsberg @@ -159,7 +159,6 @@ prebaked preconfigured prepack Preprarer -Prerequisities productional Protobuf proxying @@ -193,7 +192,6 @@ semlas semver Serverless Sinon -smartsymobls Snyk sourcemaps sparklines @@ -220,7 +218,6 @@ Templater templaters Templaters Thauer -theres toc tolerations Tolerations diff --git a/.github/workflows/create-github-release.yml b/.github/workflows/create-github-release.yml deleted file mode 100644 index 8ef3f10f6a..0000000000 --- a/.github/workflows/create-github-release.yml +++ /dev/null @@ -1,36 +0,0 @@ -# New tags and releases are created by changeset https://github.com/atlassian/changesets -name: Create a new release on GitHub when a new tag is created - -on: - push: - tags: - - 'v*' # Push events to matching v*, i.e. v0.4.0, v1.1.0 - -jobs: - build: - name: Create a new release on GitHub when a new tag is created - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: use node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: '12.x' - - - name: Install node dependencies - run: npm install @octokit/rest - - # GITHUB_REF is of the format refs/tags/vA.B.C - # This step extracts vA.B.C from GITHUB_REF - - name: Get the version - id: get_version - run: echo "::set-output name=TAG_NAME::${GITHUB_REF#refs/tags/}" - - # TODO/Note: This will only create a Draft release, which the maintainer can see and publish. - # If the Draft release looks good, modify the step to go ahead publish the release. (By adding the third CLI argument.) - - name: Create release on GitHub - run: node scripts/create-github-release.js ${{ steps.get_version.outputs.TAG_NAME }} - env: - GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 4de1ed060a..e9f5e602a5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -70,21 +70,6 @@ jobs: bash <(curl -s https://codecov.io/bash) -f packages/core/coverage/* -F core bash <(curl -s https://codecov.io/bash) -f packages/core-api/coverage/* -F core-api - # Publishes current version of packages that are not already present in the registry - - name: publish - if: matrix.node-version == '12.x' - run: yarn lerna -- publish from-package --yes - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - # Tags the commit with the version in the core package if the tag doesn't exist - - uses: Klemensas/action-autotag@1.2.3 - if: matrix.node-version == '12.x' - with: - GITHUB_TOKEN: '${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }}' - package_root: 'packages/core' - tag_prefix: 'v' - - name: Discord notification if: ${{ failure() }} uses: Ilshidur/action-discord@0.2.0 @@ -92,3 +77,85 @@ jobs: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} with: args: 'Master build failed https://github.com/{{GITHUB_REPOSITORY}}/actions/runs/{{GITHUB_RUN_ID}}' + + # A separate release build that is only run for commits that are the result of merging the "Version Packages" PR + # We can't re-use the output from the above step, but we'll have a guaranteed node_modules cache and + # only run the build steps that are necessary for publishing + release: + if: ${{ endsWith(github.event.head_commit.message, 'from backstage/changeset-release/master\n\nVersion Packages') }} + needs: build + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.x] + + env: + CI: 'true' + NODE_OPTIONS: --max-old-space-size=4096 + + steps: + - uses: actions/checkout@v2 + + # Beginning of yarn setup, keep in sync between all workflows, see ci.yml + - name: use node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + registry-url: https://registry.npmjs.org/ # Needed for auth + - name: cache all node_modules + id: cache-modules + uses: actions/cache@v2 + with: + path: '**/node_modules' + key: ${{ runner.os }}-v${{ matrix.node-version }}-node_modules-${{ hashFiles('yarn.lock', '**/package.json') }} + - name: find location of global yarn cache + id: yarn-cache + if: steps.cache-modules.outputs.cache-hit != 'true' + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache global yarn cache + uses: actions/cache@v2 + if: steps.cache-modules.outputs.cache-hit != 'true' + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - name: yarn install + run: yarn install --frozen-lockfile + # End of yarn setup + + - name: build type declarations + run: yarn tsc:full + + - name: build packages + run: yarn lerna -- run --ignore example-app build + + # Publishes current version of packages that are not already present in the registry + - name: publish + run: yarn lerna -- publish from-package --yes + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + # Creates the next available tag with format "release---[.]" + - name: Create a release tag + id: create_tag + run: node scripts/create-release-tag.js + env: + GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + + # Convert the newly created tag into a release with changelog information + - name: Create release on GitHub + run: node scripts/create-github-release.js ${{ steps.create_tag.outputs.tag_name }} 1 + env: + GITHUB_TOKEN: ${{ secrets.GH_SERVICE_ACCOUNT_TOKEN }} + + # Notify everyone about this great new release :D + - name: Discord notification + uses: Ilshidur/action-discord@0.2.0 + env: + DISCORD_WEBHOOK: ${{ secrets.DISCORD_RELEASE_WEBHOOK }} + TAG_NAME: ${{ steps.create_tag.outputs.tag_name }} + with: + args: 'A new release has been published! https://github.com/backstage/backstage/releases/tag/{{TAG_NAME}}' diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 55269dd2a5..6990f72c3f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -18,7 +18,7 @@ Harassment includes, but is not limited to: - Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, neuro(a)typicality, physical appearance, body size, race, age, regional discrimination, political or religious affiliation - Unwelcome comments regarding a person’s lifestyle choices and practices, including those related to food, health, parenting, drugs, and employment - Deliberate misgendering. This includes deadnaming or persistently using a pronoun that does not correctly reflect a person's gender identity. You must address people by the name they give you when not addressing them by their username or handle -- Physical contact and simulated physical contact (eg, textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop +- Physical contact and simulated physical contact (e.g., textual descriptions like “_hug_” or “_backrub_”) without consent or after a request to stop - Threats of violence, both physical and psychological - Incitement of violence towards any individual, including encouraging a person to commit suicide or to engage in self-harm - Deliberate intimidation diff --git a/docs/features/software-catalog/system-model.md b/docs/features/software-catalog/system-model.md index 53f49d5df9..d797e87499 100644 --- a/docs/features/software-catalog/system-model.md +++ b/docs/features/software-catalog/system-model.md @@ -44,8 +44,8 @@ Backstage model and the primary way to discover existing functionality in the ecosystem. APIs are implemented by components and form boundaries between components. They -might be defined using an RPC IDL (eg Protobuf, GraphQL, ...), a data schema (eg -Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by +might be defined using an RPC IDL (e.g., Protobuf, GraphQL, ...), a data schema +(e.g., Avro, TFRecord, ...), or as code interfaces. In any case, APIs exposed by components need to be in a known machine-readable format so we can build further tooling and analysis on top. diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 50cadc7b27..f4e89d8f6f 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -45,8 +45,6 @@ about TechDocs and the philosophy in its [v2]: https://github.com/backstage/backstage/milestone/22 [v3]: https://github.com/backstage/backstage/milestone/17 - - ## Use Cases #### TechDocs V.0 diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 38f452dcf3..187fe97aed 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -41,7 +41,7 @@ setup for free. ### Manually add documentation setup to already existing repository -Prerequisities: +Prerequisites: - An existing component [registered in backstage](../software-catalog/index.md#adding-components-to-the-catalog) diff --git a/docs/plugins/index.md b/docs/plugins/index.md index dcef3f995c..32d523b2f5 100644 --- a/docs/plugins/index.md +++ b/docs/plugins/index.md @@ -33,4 +33,4 @@ that someone else will pick up the work. If your plugin isn't supposed to live as a standalone page, but rather needs to be presented as a part of a Service Catalog (e.g. a separate tab or a card on an "Overview" tab), then check out -[the instruction](integrating-plugin-into-service-catalog.md). on how to do it. +[the instruction](integrating-plugin-into-service-catalog.md) on how to do it. diff --git a/docs/plugins/testing.md b/docs/plugins/testing.md index b564fd732c..30c3bf0ecc 100644 --- a/docs/plugins/testing.md +++ b/docs/plugins/testing.md @@ -16,15 +16,15 @@ frameworks and libraries like [Mocha](https://mochajs.org/), Running all tests: - yarn test-react + yarn test Running an individual test (e.g. `MyComponent.test.js`): - yarn test-react MyComponent + yarn test MyComponent To run both `MyComponent.test.js` and `MyControl.test.js` suite of tests: - yarn test-react MyCo + yarn test MyCo Note: if `console.logs` are not appearing, run only the individual test you are working on. @@ -52,12 +52,12 @@ render React components. TODO. -# Writing Unit Tests +## Writing Unit Tests The following principles are good guides for determining if you are writing high quality frontend unit tests. -## Bad Unit Test Principle +### Bad Unit Test Principle > No unit test is better than a bad one. @@ -69,7 +69,7 @@ Writing a poor unit test: - Adds to future work by requiring updates to the unit test for irrelevant code changes. -## Input/Output Principle +### Input/Output Principle > A unit test verifies an output matches an expected input. @@ -77,7 +77,7 @@ For backend, this would be that when you provide configuration X, then the object responds with Y. For frontend, this would be that when you provide properties X to a component, then the visual functionality responds with Y. -## Blackbox Principle +### Blackbox Principle > A good unit test does not tell the object how it should do its job but should > only compare inputs to outputs. @@ -86,7 +86,7 @@ Consider a unit test for a form. A good unit test would not test the order of the form fields. Instead, it would verify that the inputs to the form fields lead to a certain backend call when submit is clicked. -## Scalability Principle +### Scalability Principle > Unit test quality is directly proportionate to how much code can change > without having to touch the unit test. @@ -97,7 +97,7 @@ to the code, you have to update the unit test. A good unit test suite allows a lot of flexibility in _how_ the code is written so that future refactoring can occur without having to touch the original unit tests. -## Increasing Complexity Principle +### Increasing Complexity Principle > The ordering of unit tests in a suite should proceed from least specific to > most specific. @@ -116,7 +116,7 @@ throwing an error saying that output was incorrect will lead the next developer into thinking they may have broken the entire functionality of the object rather than simply letting them know they had an invalid input. -## Broken Functionality Principle +### Broken Functionality Principle > Generally, a unit test should not test exactly how the output appears, it > should test that the functionality has an expected _general_ response to an @@ -131,7 +131,7 @@ test a slightly different color on the button the unit test will break. A better unit test would verify that the button's CSS classname is assigned properly on hover or test for something completely different. -## Example: Loading Indicator +### Example: Loading Indicator A classic unit test on frontends is verifying a loading indicator displays when a backend request is being made. @@ -192,11 +192,14 @@ returns a result or displays an error or console message, like so: **`StringUtil ellipsis`** - export function ellipsis(text, maxLength, midCharIx = 0, ellipsis = '...') { - // Do something blackbox. We should not care about the internals, only inputs and outputs. - ... - return someFinalValue; - } +```js +export function ellipsis(text, maxLength, midCharIx = 0, ellipsis = '...') { + // Do something blackbox. We should not care about the internals, + // only inputs and outputs. + ... + return someFinalValue; +} +``` There are four things to test for in a utility function: @@ -207,30 +210,36 @@ There are four things to test for in a utility function: > Handle Invalid Input (handle thrown errors): - it('Throws an error on improper arguments', () => { - expect(() => { - ellipsis(); - }).toThrowError('Expected \'text\' to be defined'); - }); +```js +it('Throws an error on improper arguments', () => { + expect(() => { + ellipsis(); + }).toThrowError("Expected 'text' to be defined"); +}); +``` > Verify default input arguments: - it('Works with defaults', () => { - expect(ellipsis('Hello world', 3)).toBe('Hel...'); - expect(ellipsis('', 3)).toBe(''); - expect(ellipsis('H', 3)).toBe('H'); - expect(ellipsis('Hello', 5)).toBe('Hello'); - }); +```js +it('Works with defaults', () => { + expect(ellipsis('Hello world', 3)).toBe('Hel...'); + expect(ellipsis('', 3)).toBe(''); + expect(ellipsis('H', 3)).toBe('H'); + expect(ellipsis('Hello', 5)).toBe('Hello'); +}); +``` > Verify output for expected input arguments: This is especially true for edge cases! - it('Works with midCharIx', () => { - expect(ellipsis('Hello world', 3, 6)).toBe('...o w...'); - expect(ellipsis('', 3, 6)).toBe(''); - expect(ellipsis('Backstage is amazing', 4, 10)).toBe('...e is...'); - }); +```js +it('Works with midCharIx', () => { + expect(ellipsis('Hello world', 3, 6)).toBe('...o w...'); + expect(ellipsis('', 3, 6)).toBe(''); + expect(ellipsis('Backstage is amazing', 4, 10)).toBe('...e is...'); +}); +``` ## Non-React Classes @@ -372,4 +381,4 @@ IDE. In most cases, we have found that using `console.log` works well. Note: if your console.logs are not being displayed, focus your specific unit -test from the command line by running them like so `yarn test-react MyTest`. +test from the command line by running them like so `yarn test MyTest`. diff --git a/docs/support/project-structure.md b/docs/support/project-structure.md index bbba127c9a..a30288e85e 100644 --- a/docs/support/project-structure.md +++ b/docs/support/project-structure.md @@ -41,8 +41,8 @@ the code. appreciate contributions in here and encourage them being kept up to date. - [`docs/`](https://github.com/backstage/backstage/tree/master/docs) - This is - where we keep all of our documentation Markdown files. These ends up on - http://backstage.io/docs. Just keep in mind that changes to the + where we keep all of our documentation Markdown files. These end up on + https://backstage.io/docs. Just keep in mind that changes to the [`sidebars.json`](https://github.com/backstage/backstage/blob/master/microsite/sidebars.json) file may be needed as sections are added/removed. diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 3e21235c63..b7241bcc03 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -32,26 +32,41 @@ export interface Config { port?: string | number; }; - /** HTTPS configuration for the backend. If omitted the backend will serve HTTP */ - https?: { - /** Certificate configuration or parameters for generating a self-signed certificate */ - certificate?: - | { - /** Algorithm to use to generate a self-signed certificate */ - algorithm: string; - keySize?: number; - days?: number; - } - | { - /** PEM encoded certificate. Use $file to load in a file */ - cert: string; - /** - * PEM encoded certificate key. Use $file to load in a file. - * @visibility secret - */ - key: string; - }; - }; + /** + * HTTPS configuration for the backend. If omitted the backend will serve HTTP. + * + * Setting this to `true` will cause self-signed certificates to be generated, which + * can be useful for local development or other non-production scenarios. + */ + https?: + | true + | { + /** + * Certificate configuration or parameters for generating a self-signed certificate + * + * Setting parameters for self-signed certificates is deprecated and will be removed in + * the future, set `backend.https = true` instead. + */ + certificate?: + | { + /** Algorithm to use to generate a self-signed certificate */ + algorithm?: string; + keySize?: number; + days?: number; + attributes: { + commonName: string; + }; + } + | { + /** PEM encoded certificate. Use $file to load in a file */ + cert: string; + /** + * PEM encoded certificate key. Use $file to load in a file. + * @visibility secret + */ + key: string; + }; + }; /** Database connection configuration, select database type using the `client` field */ database: diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index ca934ee42c..ea716a1d4a 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -47,13 +47,7 @@ export class AzureUrlReader implements UrlReader { constructor( private readonly options: AzureIntegrationConfig, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, - ) { - if (options.host !== 'dev.azure.com') { - throw Error( - `Azure integration currently only supports 'dev.azure.com', tried to use host '${options.host}'`, - ); - } - } + ) {} async read(url: string): Promise { const builtUrl = getAzureFileFetchUrl(url); diff --git a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts index 46bf23beef..9c5ac20fa3 100644 --- a/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts +++ b/packages/backend-common/src/service/lib/ServiceBuilderImpl.ts @@ -145,7 +145,7 @@ export class ServiceBuilderImpl implements ServiceBuilder { return this; } - start(): Promise { + async start(): Promise { const app = express(); const { port, @@ -168,16 +168,16 @@ export class ServiceBuilderImpl implements ServiceBuilder { app.use(notFoundHandler()); app.use(errorHandler()); + const server: http.Server = httpsSettings + ? await createHttpsServer(app, httpsSettings, logger) + : createHttpServer(app, logger); + return new Promise((resolve, reject) => { app.on('error', e => { logger.error(`Failed to start up on port ${port}, ${e}`); reject(e); }); - const server: http.Server = httpsSettings - ? createHttpsServer(app, httpsSettings, logger) - : createHttpServer(app, logger); - const stoppableServer = stoppable( server.listen(port, host, () => { logger.info(`Listening on ${host}:${port}`); diff --git a/packages/backend-common/src/service/lib/config.ts b/packages/backend-common/src/service/lib/config.ts index ea1a2b0887..6abea97454 100644 --- a/packages/backend-common/src/service/lib/config.ts +++ b/packages/backend-common/src/service/lib/config.ts @@ -47,14 +47,14 @@ export type CertificateReferenceOptions = { }; export type CertificateSigningOptions = { - algorithm: string; + algorithm?: string; size?: number; days?: number; - attributes?: CertificateAttributes; + attributes: CertificateAttributes; }; export type CertificateAttributes = { - commonName?: string; + commonName: string; }; /** @@ -193,8 +193,26 @@ export function readCspOptions( * ``` */ export function readHttpsSettings(config: Config): HttpsSettings | undefined { - const cc = config.getOptionalConfig('https'); + const https = config.get('https'); + if (https === true) { + const baseUrl = config.getString('baseUrl'); + let commonName; + try { + commonName = new URL(baseUrl).hostname; + } catch (error) { + throw new Error(`Invalid backend.baseUrl "${baseUrl}"`); + } + return { + certificate: { + attributes: { + commonName, + }, + }, + }; + } + + const cc = config.getOptionalConfig('https'); if (!cc) { return undefined; } diff --git a/packages/backend-common/src/service/lib/hostFactory.ts b/packages/backend-common/src/service/lib/hostFactory.ts index 8fab2fd4ab..656f160c31 100644 --- a/packages/backend-common/src/service/lib/hostFactory.ts +++ b/packages/backend-common/src/service/lib/hostFactory.ts @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, dirname } from 'path'; import express from 'express'; import * as http from 'http'; import * as https from 'https'; import { Logger } from 'winston'; -import { HttpsSettings } from './config'; +import { CertificateSigningOptions, HttpsSettings } from './config'; + +const ALMOST_MONTH_IN_MS = 25 * 24 * 60 * 60 * 1000; /** * Creates a Http server instance based on an Express application. @@ -45,48 +50,152 @@ export function createHttpServer( * @returns A Https server instance * */ -export function createHttpsServer( +export async function createHttpsServer( app: express.Express, httpsSettings: HttpsSettings, logger?: Logger, -): http.Server { +): Promise { logger?.info('Initializing https server'); - const credentials: { key: string; cert: string } = { - key: '', - cert: '', - }; + let credentials: { key: string | Buffer; cert: string | Buffer }; const signingOptions: any = httpsSettings?.certificate; - if (signingOptions?.algorithm !== undefined) { - logger?.info('Generating self-signed certificate with attributes'); - - const certificateAttributes: Array = Object.entries( - signingOptions.attributes, - ).map(([name, value]) => ({ name, value })); - - // TODO: Create a type def for selfsigned. - const signatures = require('selfsigned').generate(certificateAttributes, { - algorithm: signingOptions?.algorithm, - keySize: signingOptions?.size || 2048, - days: signingOptions?.days || 30, - }); - - logger?.info('Bootstrapping self-signed certificate'); - - credentials.key = signatures.private; - credentials.cert = signatures.cert; + // TODO(Rugvip): remove support for generated certificate params and make this a more straightforward check + if (signingOptions?.attributes) { + credentials = await getGeneratedCertificate(signingOptions, logger); } else { - logger?.info('Bootstrapping cert from config'); + logger?.info('Loading certificate from config'); - credentials.key = signingOptions?.key; - credentials.cert = signingOptions?.cert; + credentials = { + key: signingOptions?.key, + cert: signingOptions?.cert, + }; } - if (credentials.key === '' || credentials.cert === '') { - throw new Error('Invalid credentials'); + if (!credentials.key || !credentials.cert) { + throw new Error('Invalid HTTPS credentials'); } return https.createServer(credentials, app) as http.Server; } + +async function getGeneratedCertificate( + options: CertificateSigningOptions, + logger?: Logger, +) { + if (options?.algorithm) { + logger?.warn( + 'Certificate generation configuration with parameters in backend.https.certificate is deprecated, set backend.https = true instead', + ); + } + + const hasModules = await fs.pathExists('node_modules'); + let certPath; + if (hasModules) { + certPath = resolvePath( + 'node_modules/.cache/backstage-backend/dev-cert.pem', + ); + await fs.ensureDir(dirname(certPath)); + } else { + certPath = resolvePath('.dev-cert.pem'); + } + + let cert = undefined; + if (await fs.pathExists(certPath)) { + const stat = await fs.stat(certPath); + const ageMs = Date.now() - stat.ctimeMs; + if (stat.isFile() && ageMs < ALMOST_MONTH_IN_MS) { + cert = await fs.readFile(certPath); + } + } + + if (cert) { + logger?.info('Using existing self-signed certificate'); + return { + key: cert, + cert: cert, + }; + } + + logger?.info('Generating new self-signed certificate'); + const newCert = await createCertificate(options); + await fs.writeFile(certPath, newCert.cert + newCert.key, 'utf8'); + return newCert; +} + +async function createCertificate(options: CertificateSigningOptions) { + const attributes: Array = Object.entries( + options.attributes, + ).map(([name, value]) => ({ name, value })); + + const params = { + algorithm: options?.algorithm || 'sha256', + keySize: options?.size || 2048, + days: options?.days || 30, + extensions: [ + { + name: 'keyUsage', + keyCertSign: true, + digitalSignature: true, + nonRepudiation: true, + keyEncipherment: true, + dataEncipherment: true, + }, + { + name: 'extKeyUsage', + serverAuth: true, + clientAuth: true, + codeSigning: true, + timeStamping: true, + }, + { + name: 'subjectAltName', + altNames: [ + { + type: 2, // DNS + value: 'localhost', + }, + { + type: 2, + value: 'localhost.localdomain', + }, + { + type: 2, + value: '[::1]', + }, + { + type: 7, // IP + ip: '127.0.0.1', + }, + { + type: 7, + ip: 'fe80::1', + }, + ...(options.attributes.commonName + ? [ + { + type: 2, // DNS + value: options.attributes.commonName, + }, + ] + : []), + ], + }, + ], + }; + + return new Promise<{ key: string; cert: string }>((resolve, reject) => + require('selfsigned').generate( + attributes, + params, + (err: Error, bundle: { private: string; cert: string }) => { + if (err) { + reject(err); + } else { + resolve({ key: bundle.private, cert: bundle.cert }); + } + }, + ), + ); +} diff --git a/packages/create-app/README.md b/packages/create-app/README.md index 7533110372..dafc01e365 100644 --- a/packages/create-app/README.md +++ b/packages/create-app/README.md @@ -1,9 +1,10 @@ # @backstage/create-app -This package provides a CLI for creating apps. +This package provides a CLI for creating a copy of the Backstage app. + You can use the flag `--skip-install` to skip the install. -## Installation +## Usage With `npx`: @@ -11,6 +12,12 @@ With `npx`: $ npx @backstage/create-app ``` +With a local clone of this repo, from the main `create-app/` folder, run: + +```sh +$ yarn backstage-create-app +``` + ## Documentation - [Backstage Readme](https://github.com/backstage/backstage/blob/master/README.md) diff --git a/packages/create-app/package.json b/packages/create-app/package.json index ac29dc1431..b9743e2bc0 100644 --- a/packages/create-app/package.json +++ b/packages/create-app/package.json @@ -47,12 +47,12 @@ "@backstage/plugin-auth-backend": "^0.2.8", "@backstage/plugin-catalog": "^0.2.9", "@backstage/plugin-catalog-backend": "^0.5.1", + "@backstage/plugin-catalog-import": "^0.3.2", "@backstage/plugin-circleci": "^0.2.5", "@backstage/plugin-explore": "^0.2.2", "@backstage/plugin-github-actions": "^0.2.6", "@backstage/plugin-lighthouse": "^0.2.6", "@backstage/plugin-proxy-backend": "^0.2.3", - "@backstage/plugin-register-component": "^0.2.6", "@backstage/plugin-rollbar-backend": "^0.1.5", "@backstage/plugin-scaffolder": "^0.3.5", "@backstage/plugin-search": "^0.2.4", diff --git a/packages/create-app/src/lib/versions.ts b/packages/create-app/src/lib/versions.ts index df8b9c01ea..5cbf6c77ee 100644 --- a/packages/create-app/src/lib/versions.ts +++ b/packages/create-app/src/lib/versions.ts @@ -39,12 +39,12 @@ import { version as pluginAppBackend } from '@backstage/plugin-app-backend/packa import { version as pluginAuthBackend } from '@backstage/plugin-auth-backend/package.json'; import { version as pluginCatalog } from '@backstage/plugin-catalog/package.json'; import { version as pluginCatalogBackend } from '@backstage/plugin-catalog-backend/package.json'; +import { version as pluginCatalogImport } from '@backstage/plugin-catalog-import/package.json'; import { version as pluginCircleci } from '@backstage/plugin-circleci/package.json'; import { version as pluginExplore } from '@backstage/plugin-explore/package.json'; import { version as pluginGithubActions } from '@backstage/plugin-github-actions/package.json'; import { version as pluginLighthouse } from '@backstage/plugin-lighthouse/package.json'; import { version as pluginProxyBackend } from '@backstage/plugin-proxy-backend/package.json'; -import { version as pluginRegisterComponent } from '@backstage/plugin-register-component/package.json'; import { version as pluginRollbarBackend } from '@backstage/plugin-rollbar-backend/package.json'; import { version as pluginScaffolder } from '@backstage/plugin-scaffolder/package.json'; import { version as pluginScaffolderBackend } from '@backstage/plugin-scaffolder-backend/package.json'; @@ -67,12 +67,12 @@ export const packageVersions = { '@backstage/plugin-auth-backend': pluginAuthBackend, '@backstage/plugin-catalog': pluginCatalog, '@backstage/plugin-catalog-backend': pluginCatalogBackend, + '@backstage/plugin-catalog-import': pluginCatalogImport, '@backstage/plugin-circleci': pluginCircleci, '@backstage/plugin-explore': pluginExplore, '@backstage/plugin-github-actions': pluginGithubActions, '@backstage/plugin-lighthouse': pluginLighthouse, '@backstage/plugin-proxy-backend': pluginProxyBackend, - '@backstage/plugin-register-component': pluginRegisterComponent, '@backstage/plugin-rollbar-backend': pluginRollbarBackend, '@backstage/plugin-scaffolder': pluginScaffolder, '@backstage/plugin-scaffolder-backend': pluginScaffolderBackend, diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs index 919ca678a0..48ab600c97 100644 --- a/packages/create-app/templates/default-app/packages/app/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs @@ -10,7 +10,7 @@ "@backstage/core": "^{{version '@backstage/core'}}", "@backstage/plugin-api-docs": "^{{version '@backstage/plugin-api-docs'}}", "@backstage/plugin-catalog": "^{{version '@backstage/plugin-catalog'}}", - "@backstage/plugin-register-component": "^{{version '@backstage/plugin-register-component'}}", + "@backstage/plugin-catalog-import": "^{{version '@backstage/plugin-catalog-import'}}", "@backstage/plugin-scaffolder": "^{{version '@backstage/plugin-scaffolder'}}", "@backstage/plugin-techdocs": "^{{version '@backstage/plugin-techdocs'}}", "@backstage/catalog-model": "^{{version '@backstage/catalog-model'}}", diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 2e250a94a1..693e66e0c6 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -12,7 +12,7 @@ import { AppSidebar } from './sidebar'; import { Route, Routes, Navigate } from 'react-router'; import { Router as CatalogRouter } from '@backstage/plugin-catalog'; import { Router as DocsRouter } from '@backstage/plugin-techdocs'; -import { Router as RegisterComponentRouter } from '@backstage/plugin-register-component'; +import { Router as ImportComponentRouter } from '@backstage/plugin-catalog-import'; import { Router as TechRadarRouter } from '@backstage/plugin-tech-radar'; import { SearchPage as SearchRouter } from '@backstage/plugin-search'; @@ -52,8 +52,8 @@ const App = () => ( element={} /> } + path="/catalog-import" + element={} /> { result: 'https://dev.azure.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', }, + { + url: + 'https://api.com/org-name/project-name/_git/repo-name?path=my-template.yaml', + result: + 'https://api.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml', + }, + { + url: + 'https://api.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + result: + 'https://api.com/org-name/project-name/_apis/git/repositories/repo-name/items?path=my-template.yaml&version=master', + }, ])('should handle happy path %#', async ({ url, result }) => { expect(getAzureFileFetchUrl(url)).toBe(result); }); diff --git a/packages/integration/src/azure/core.ts b/packages/integration/src/azure/core.ts index 44591f2509..7900774c79 100644 --- a/packages/integration/src/azure/core.ts +++ b/packages/integration/src/azure/core.ts @@ -43,7 +43,6 @@ export function getAzureFileFetchUrl(url: string): string { const ref = parsedUrl.searchParams.get('version')?.substr(2); if ( - parsedUrl.hostname !== 'dev.azure.com' || empty !== '' || userOrOrg === '' || project === '' || diff --git a/packages/storybook/README.md b/packages/storybook/README.md index 38708afe4b..3e88b54db0 100644 --- a/packages/storybook/README.md +++ b/packages/storybook/README.md @@ -1,7 +1,7 @@ # storybook -This package provides a storybook build for Backstage. See [http://backstage.io/storybook](http://http://backstage.io/storybook) +This package provides a Storybook build for Backstage. See https://backstage.io/storybook/. ## Why is this not part of `@backstage/core`? -This separate storybook package exists because of dependency conflicts with `@backstage/cli`. It uses nohoist to avoid the conflicts, and since you can only use that in private packages it has to be separated out of `@backstage/core`. +This separate storybook package exists because of dependency conflicts with `@backstage/cli`. It uses `nohoist` to avoid the conflicts, and since you can only use that in private packages it has to be separated out of `@backstage/core`. diff --git a/plugins/README.md b/plugins/README.md index 6651ba079f..385e71b6d2 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -1,14 +1,12 @@ # Plugins -Backstage is a single-page application composed of a set of plugins. +Backstage is a single-page application composed of a set of plugins. This folder holds numerous plugins that are managed by this repository. -Our goal for the plugin ecosystem is that the definition of a plugin is flexible enough to allow you to expose pretty much any kind of infrastructure or software development tool as a plugin in Backstage. By following strong [design guidelines](https://github.com/backstage/backstage/blob/master/docs/dls/design.md) we ensure the overall user experience stays consistent between plugins. +For more information about the plugin ecosystem, see the documentation here: -![plugin](../docs/assets/my-plugin_screenshot.png) +> https://backstage.io/docs/plugins/ -## Creating a plugin - -To create a plugin, follow the steps outlined [here](https://github.com/backstage/backstage/blob/master/docs/plugins/create-a-plugin.md). +You can also see the [Plugin Marketplace](https://backstage.io/plugins) for other open source plugins you can add to your Backstage instance. ## Suggesting a plugin diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js index c97331796d..9acd7564db 100644 --- a/plugins/catalog-backend/migrations/20200702153613_entities.js +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -20,16 +20,14 @@ * @param {import('knex')} knex */ exports.up = async function up(knex) { - // Drop constraints (Postgres) - try { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); await knex.schema.alterTable('entities', table => { table.dropPrimary('entities_pkey'); }); - } catch (e) { - // SQLite does not support FK and PK, carry on } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); @@ -131,16 +129,14 @@ exports.up = async function up(knex) { * @param {import('knex')} knex */ exports.down = async function down(knex) { - // Drop constraints (Postgres) - try { + // SQLite does not support FK and PK + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); await knex.schema.alterTable('entities', table => { table.dropPrimary('entities_pkey'); }); - } catch (e) { - // SQLite does not support FK and PK, carry on } await knex.schema.alterTable('entities', table => { table.dropUnique([], 'entities_unique_name'); diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js index 6e02975f92..b3a9673641 100644 --- a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -20,12 +20,11 @@ * @param {import('knex')} knex */ exports.up = async function up(knex) { - try { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.text('value').nullable().alter(); }); - } catch (e) { - // Sqlite does not support alter column. } }; @@ -33,11 +32,10 @@ exports.up = async function up(knex) { * @param {import('knex')} knex */ exports.down = async function down(knex) { - try { + // Sqlite does not support alter column. + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities_search', table => { table.string('value').nullable().alter(); }); - } catch (e) { - // Sqlite does not support alter column. } }; diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 26cc98e74e..4c1ea76de4 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -30,12 +30,11 @@ exports.up = async function up(knex) { ), }); - try { + // SQLite does not support alter column + if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities', table => { table.text('full_name').notNullable().alter(); }); - } catch (e) { - // SQLite does not support alter column, ignore } await knex.schema.alterTable('entities', table => { diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js index fdabf72fce..a326a08a8b 100644 --- a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -40,10 +40,7 @@ exports.up = async function up(knex) { table.dropColumn('spec'); }); - // SQLite does not support ALTER COLUMN. Note that we do not use the try/ - // catch method as in other migrations, because if the transaction is - // partially failed, it will further mess up the already messed-up - // statement below this. + // SQLite does not support ALTER COLUMN. if (knex.client.config.client !== 'sqlite3') { await knex.schema.alterTable('entities', table => { table.text('data').notNullable().alter(); diff --git a/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js new file mode 100644 index 0000000000..71f9500cf2 --- /dev/null +++ b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex')} knex + */ +exports.up = async function up(knex) { + if (knex.client.config.client !== 'sqlite3') { + // We actually just want to widen columns, but can't do that while a + // view is dependent on them - so we just reconstruct it exactly as it was + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.text('message').alter(); + table.text('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; + +/** + * @param {import('knex')} knex + */ +exports.down = async function down(knex) { + if (knex.client.config.client !== 'sqlite3') { + await knex.schema + .raw('DROP VIEW location_update_log_latest;') + .alterTable('location_update_log', table => { + table.string('message').alter(); + table.string('entity_name').nullable().alter(); + }).raw(` + CREATE VIEW location_update_log_latest AS + SELECT t1.* FROM location_update_log t1 + JOIN + ( + SELECT location_id, MAX(id) AS MAXID + FROM location_update_log + GROUP BY location_id + ) t2 + ON t1.location_id = t2.location_id + AND t1.id = t2.MAXID + GROUP BY t1.location_id, t1.id + ORDER BY created_at DESC; + `); + } +}; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 5ebe1c3e41..1a43dd6d62 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -36,6 +36,7 @@ "@octokit/graphql": "^4.5.6", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", + "aws-sdk": "^2.817.0", "codeowners-utils": "^1.0.2", "core-js": "^3.6.5", "cross-fetch": "^3.0.6", diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts new file mode 100644 index 0000000000..eb5e634d66 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; + +describe('AwsOrganizationCloudAccountProcessor', () => { + describe('readLocation', () => { + const processor = new AwsOrganizationCloudAccountProcessor(); + const location = { type: 'aws-cloud-accounts', target: '' }; + const emit = jest.fn(); + const listAccounts = jest.fn(); + + processor.organizations.listAccounts = listAccounts; + afterEach(() => jest.resetAllMocks()); + + it('generates component entities for accounts', async () => { + listAccounts.mockImplementation(() => { + return { + async promise() { + return { + Accounts: [ + { + Arn: + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + Name: 'testaccount', + }, + ], + NextToken: undefined, + }; + }, + }; + }); + await processor.readLocation(location, false, emit); + expect(emit).toBeCalledWith({ + type: 'entity', + location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + 'amazonaws.com/account-id': '957140518395', + 'amazonaws.com/organization-id': 'o-1vl18kc5a3', + }, + name: 'testaccount', + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }, + }); + }); + + it('filters out accounts not in specified location target', async () => { + const location = { type: 'aws-cloud-accounts', target: 'o-1vl18kc5a3' }; + listAccounts.mockImplementation(() => { + return { + async promise() { + return { + Accounts: [ + { + Arn: + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + Name: 'testaccount', + }, + { + Arn: + 'arn:aws:organizations::192594491037:account/o-zzzzzzzzz/957140518395', + Name: 'testaccount2', + }, + ], + NextToken: undefined, + }; + }, + }; + }); + await processor.readLocation(location, false, emit); + expect(emit).toBeCalledTimes(1); + expect(emit).toBeCalledWith({ + type: 'entity', + location, + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + 'amazonaws.com/arn': + 'arn:aws:organizations::192594491037:account/o-1vl18kc5a3/957140518395', + 'amazonaws.com/account-id': '957140518395', + 'amazonaws.com/organization-id': 'o-1vl18kc5a3', + }, + name: 'testaccount', + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }, + }); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts new file mode 100644 index 0000000000..f019f5207b --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AwsOrganizationCloudAccountProcessor.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + ComponentEntityV1alpha1, + LocationSpec, +} from '@backstage/catalog-model'; +import AWS, { Organizations } from 'aws-sdk'; +import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; + +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +const AWS_ORGANIZATION_REGION = 'us-east-1'; +const LOCATION_TYPE = 'aws-cloud-accounts'; + +const ACCOUNTID_ANNOTATION: string = 'amazonaws.com/account-id'; +const ARN_ANNOTATION: string = 'amazonaws.com/arn'; +const ORGANIZATION_ANNOTATION: string = 'amazonaws.com/organization-id'; + +/** + * A processor for ingesting AWS Accounts from AWS Organizations. + * + * If custom authentication is needed, it can be achieved by configuring the global AWS.credentials object. + */ +export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { + organizations: Organizations; + constructor() { + this.organizations = new AWS.Organizations({ + region: AWS_ORGANIZATION_REGION, + }); // Only available in us-east-1 + } + + normalizeName(name: string): string { + return name + .trim() + .toLocaleLowerCase() + .replace(/[^a-zA-Z0-9\-]/g, '-'); + } + + extractInformationFromArn( + arn: string, + ): { accountId: string; organizationId: string } { + const parts = arn.split('/'); + + return { + accountId: parts[parts.length - 1], + organizationId: parts[parts.length - 2], + }; + } + + async getAwsAccounts(): Promise { + let awsAccounts: Account[] = []; + let isInitialAttempt = true; + let nextToken = undefined; + while (isInitialAttempt || nextToken) { + isInitialAttempt = false; + const orgAccounts: ListAccountsResponse = await this.organizations + .listAccounts({ NextToken: nextToken }) + .promise(); + if (orgAccounts.Accounts) { + awsAccounts = awsAccounts.concat(orgAccounts.Accounts); + } + nextToken = orgAccounts.NextToken; + } + + return awsAccounts; + } + + mapAccountToComponent(account: Account): ComponentEntityV1alpha1 { + const { accountId, organizationId } = this.extractInformationFromArn( + account.Arn as string, + ); + return { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + annotations: { + [ACCOUNTID_ANNOTATION]: accountId, + [ARN_ANNOTATION]: account.Arn || '', + [ORGANIZATION_ANNOTATION]: organizationId, + }, + name: this.normalizeName(account.Name || ''), + namespace: 'default', + }, + spec: { + type: 'cloud-account', + lifecycle: 'unknown', + owner: 'unknown', + }, + }; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== LOCATION_TYPE) { + return false; + } + + (await this.getAwsAccounts()) + .map(account => this.mapAccountToComponent(account)) + .filter(entity => { + if (location.target !== '') { + if (entity.metadata.annotations) { + return ( + entity.metadata.annotations[ORGANIZATION_ANNOTATION] === + location.target + ); + } + return false; + } + return true; + }) + .forEach((entity: ComponentEntityV1alpha1) => { + emit(results.entity(location, entity)); + }); + + return true; + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 2b477f18f9..a7b00d7065 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -17,6 +17,7 @@ import * as results from './results'; export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; +export { AwsOrganizationCloudAccountProcessor } from './AwsOrganizationCloudAccountProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; diff --git a/plugins/tech-radar/README.md b/plugins/tech-radar/README.md index cbe4dd22eb..21f90ec432 100644 --- a/plugins/tech-radar/README.md +++ b/plugins/tech-radar/README.md @@ -83,13 +83,20 @@ const getHardCodedData = () => rings: [{ id: 'use', name: 'USE', color: '#93c47d' }], entries: [ { - moved: 0, - ring: 'use', url: '#', key: 'github-actions', id: 'github-actions', title: 'GitHub Actions', quadrant: 'infrastructure', + timeline: [ + { + moved: 0, + ringId: 'use', + date: new Date('2020-08-06'), + description: + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat', + }, + ], }, ], }); diff --git a/scripts/create-release-tag.js b/scripts/create-release-tag.js new file mode 100755 index 0000000000..5c57c9b278 --- /dev/null +++ b/scripts/create-release-tag.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node +/* eslint-disable import/no-extraneous-dependencies */ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { Octokit } = require('@octokit/rest'); + +const baseOptions = { + owner: 'backstage', + repo: 'backstage', +}; + +async function main() { + const { GITHUB_SHA, GITHUB_TOKEN } = process.env; + if (!GITHUB_SHA) { + throw new Error('GITHUB_SHA is not set'); + } + if (!GITHUB_TOKEN) { + throw new Error('GITHUB_TOKEN is not set'); + } + + const octokit = new Octokit({ auth: GITHUB_TOKEN }); + + const date = new Date(); + const baseTagName = `release-${date.getUTCFullYear()}-${ + date.getUTCMonth() + 1 + }-${date.getUTCDate()}`; + + console.log('Requesting existing tags'); + + const existingTags = await octokit.repos.listTags({ + ...baseOptions, + per_page: 100, + }); + const existingTagNames = existingTags.data.map(obj => obj.name); + + let tagName = baseTagName; + let index = 0; + while (existingTagNames.includes(tagName)) { + index += 1; + tagName = `${baseTagName}.${index}`; + } + + console.log(`Creating release tag ${tagName}`); + + const annotatedTag = await octokit.git.createTag({ + ...baseOptions, + tag: tagName, + message: tagName, + object: GITHUB_SHA, + type: 'commit', + }); + + await octokit.git.createRef({ + ...baseOptions, + ref: `refs/tags/${tagName}`, + sha: annotatedTag.data.sha, + }); + + console.log(`::set-output name=tag_name::${tagName}`); +} + +main().catch(error => { + console.error(error.stack); + process.exit(1); +}); diff --git a/yarn.lock b/yarn.lock index 957a7cebe4..ca7b6f065f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7698,6 +7698,21 @@ autoprefixer@^9.7.2: postcss "^7.0.32" postcss-value-parser "^4.1.0" +aws-sdk@^2.817.0: + version "2.817.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.817.0.tgz#3a97b690b0ec494cf8ee927affb3973cf26abcc8" + integrity sha512-DZIdWpkcqbqsCz0MEskHsyFaqc6Tk9XIFqXAg1AKHbOgC8nU45bz+Y2osX77pU01JkS/G7OhGtGmlKDrOPvFwg== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -8564,7 +8579,7 @@ buffer-xor@^1.0.3: resolved "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" integrity sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk= -buffer@^4.3.0: +buffer@4.9.2, buffer@^4.3.0: version "4.9.2" resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== @@ -11980,6 +11995,11 @@ eventemitter3@^4.0.0: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb" integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg== +events@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + events@3.1.0, events@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/events/-/events-3.1.0.tgz#84279af1b34cb75aa88bf5ff291f6d0bd9b31a59" @@ -14238,6 +14258,11 @@ identity-obj-proxy@3.0.0: dependencies: harmony-reflect "^1.4.6" +ieee754@1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ieee754@^1.1.13, ieee754@^1.1.4, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -15657,6 +15682,11 @@ jest@^26.0.1: import-local "^3.0.2" jest-cli "^26.6.3" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + jose@^1.27.1: version "1.27.1" resolved "https://registry.npmjs.org/jose/-/jose-1.27.1.tgz#a1de2ecb5b3ae1ae28f0d9d0cc536349ada27ec8" @@ -21916,6 +21946,11 @@ sanitize-html@^1.27.0: srcset "^2.0.1" xtend "^4.0.1" +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + sax@>=0.6.0, sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -24422,6 +24457,14 @@ url-parse@^1.4.3, url-parse@^1.4.7: querystringify "^2.1.1" requires-port "^1.0.0" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0, url@~0.11.0: version "0.11.0" resolved "https://registry.npmjs.org/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -24520,6 +24563,11 @@ utils-merge@1.0.1, utils-merge@1.x.x: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2, uuid@^3.4.0: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -25251,6 +25299,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + xml2js@0.4.x: version "0.4.23" resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" @@ -25264,6 +25320,11 @@ xmlbuilder@^11.0.0, xmlbuilder@~11.0.0: resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"