From 257a3b52ed64d82230b270a2c04a9543fc1f42ec Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:17:57 +1200 Subject: [PATCH 001/114] Add Azure ingestion processor --- .../src/ingestion/LocationReaders.ts | 2 + .../AzureApiReaderProcessor.test.ts | 84 +++++++++++ .../processors/AzureApiReaderProcessor.ts | 134 ++++++++++++++++++ .../RegisterComponentForm.tsx | 2 +- .../RegisterComponentPage.tsx | 13 +- 5 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 05f93815b8..29892bb2dc 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -31,6 +31,7 @@ import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor' import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor'; +import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor'; import { UrlReaderProcessor } from './processors/UrlReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import { StaticLocationProcessor } from './processors/StaticLocationProcessor'; @@ -79,6 +80,7 @@ export class LocationReaders implements LocationReader { new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), new BitbucketApiReaderProcessor(), + new AzureApiReaderProcessor(), new UrlReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts new file mode 100644 index 0000000000..35101b2607 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; + +describe('BitbucketApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new AzureApiReaderProcessor(); + const tests = [ + { + target: + 'https://dev.azure.com/org-name/project-name/_git/repo-name?path=my-template.yaml&version=GBmaster', + url: new URL( + 'https://dev.azure.com/org-name/project-name/_apis/sourceProviders/TfsGit/filecontents?repository=repo-name&commitOrBranch=master&path=my-template.yaml&api-version=6.0-preview.1', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Azure Devops URL or Invalid file path', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else if (test.url) { + expect(processor.buildRawUrl(test.target).toString()).toEqual( + test.url.toString(), + ); + } else { + throw new Error( + 'This should not have happened. Either err or url should have matched.', + ); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Authorization: 'Basic OjAxMjM0NTY3ODk=', + }, + }, + }, + { + token: '', + expect: { + headers: {}, + }, + }, + ]; + + for (const test of tests) { + process.env.AZURE_PRIVATE_TOKEN = test.token; + const processor = new AzureApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts new file mode 100644 index 0000000000..41b4336b4f --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class AzureApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.AZURE_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = {}; + + if (this.privateToken !== '') { + headers.Authorization = `Basic ${Buffer.from( + `:${this.privateToken}`, + 'utf8', + ).toString('base64')}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'azure/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // Converts + // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents + // to: https://dev.azure.com/{organization}/{project}/_apis/sourceProviders/{providerName}/filecontents?repository={repository}&commitOrBranch={commitOrBranch}&path={path}&api-version=6.0-preview.1 + + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + project, + srcKeyword, + repoName, + ] = url.pathname.split('/'); + + const path = url.searchParams.get('path') || ''; + const ref = url.searchParams.get('version')?.substr(2); + + if ( + url.hostname !== 'dev.azure.com' || + empty !== '' || + userOrOrg === '' || + project === '' || + srcKeyword !== '_git' || + repoName === '' || + path === '' || + ref === '' || + !path.match(/\.yaml$/) + ) { + throw new Error('Wrong Azure Devops URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + userOrOrg, + project, + '_apis', + 'sourceProviders', + 'TfsGit', + 'filecontents', + ].join('/'); + + url.search = [ + `repository=${repoName}`, + `commitOrBranch=${ref}`, + `path=${path}`, + 'api-version=6.0-preview.1', + ].join('&'); + + url.protocol = 'https'; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index c1ea9c454d..a61469e969 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo." + helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 85d63fe2cb..be497e9aa7 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,7 +79,18 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - const data = await catalogApi.addLocation('github', target); + var typeMapping = [ + { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, + { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, + { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, + { url: /.*/, type: 'github' }, + ]; + + var type = typeMapping.filter(function (item) { + return new RegExp(item.url).test(target); + })[0].type; + + const data = await catalogApi.addLocation(type, target); if (!isMounted()) return; From c33fc6c356227a1f5b034350630a879c413e85be Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:41:58 +1200 Subject: [PATCH 002/114] Fix linting errors --- .../RegisterComponentPage/RegisterComponentPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index be497e9aa7..1f0eb4fead 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -79,14 +79,14 @@ const RegisterComponentPage: FC<{}> = () => { setFormState(FormStates.Submitting); const { componentLocation: target } = formData; try { - var typeMapping = [ + const typeMapping = [ { url: /https:\/\/gitlab\.com\/.*/, type: 'gitlab' }, { url: /https:\/\/bitbucket\.org\/.*/, type: 'bitbucket/api' }, { url: /https:\/\/dev\.azure\.com\/.*/, type: 'azure/api' }, { url: /.*/, type: 'github' }, ]; - var type = typeMapping.filter(function (item) { + const type = typeMapping.filter(item => { return new RegExp(item.url).test(target); })[0].type; From 7e454952f7e623fde4ea0437ed98e08c9a6ac0b8 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 00:59:03 +1200 Subject: [PATCH 003/114] fix broken UI test --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index 01402ca19b..fd7a1cc0f8 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,14 +30,14 @@ const setup = (props?: Partial) => { ), }; }; -describe('RegisterComponentForm', () => { +fdescribe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in GitHub to start tracking your component. It must be in a public repo.', + 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); From 54b6a9cc06d17e3365096c3314c4e78e4bb87cca Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 01:02:32 +1200 Subject: [PATCH 004/114] undo test focus --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index fd7a1cc0f8..d4d2cf4789 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -30,7 +30,7 @@ const setup = (props?: Partial) => { ), }; }; -fdescribe('RegisterComponentForm', () => { +describe('RegisterComponentForm', () => { afterEach(() => cleanup()); it('should initially render a disabled button', async () => { From f57f6a7402965d7c95eb0b73af2608f779770e99 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Sun, 23 Aug 2020 14:26:39 +1200 Subject: [PATCH 005/114] modify ends with .yaml check to contains .yaml --- .../processors/AzureApiReaderProcessor.test.ts | 2 +- .../ingestion/processors/YamlProcessor.test.ts | 16 ++++++++++++++++ .../src/ingestion/processors/YamlProcessor.ts | 2 +- .../RegisterComponentPage.tsx | 4 +--- .../register-component/src/util/validate.test.ts | 3 ++- plugins/register-component/src/util/validate.ts | 4 ++-- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts index 35101b2607..0ef111fcb1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.test.ts @@ -16,7 +16,7 @@ import { AzureApiReaderProcessor } from './AzureApiReaderProcessor'; -describe('BitbucketApiReaderProcessor', () => { +describe('AzureApiReaderProcessor', () => { it('should build raw api', () => { const processor = new AzureApiReaderProcessor(); const tests = [ diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts index 587793a6de..9f1ede3f9e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts @@ -51,6 +51,22 @@ describe('YamlProcessor', () => { expect(never).not.toBeCalled(); }); + it('should process url that contains yaml', async () => { + const containsYamlLocationSpec = { + type: 'url', + target: 'http://example.com/component?path=test.yaml&c=1&d=2', + }; + + const buffer = Buffer.from([]); + const emit = jest.fn(); + + expect( + await processor.parseData(buffer, containsYamlLocationSpec, emit), + ).toBe(true); + + expect(emit).toBeCalled(); + }); + it('should process entity with yaml', async () => { const entity = { apiVersion: 'backstage.io/v1alpha1', diff --git a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts index 6a2b5cf419..79ae55fae1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts @@ -26,7 +26,7 @@ export class YamlProcessor implements LocationProcessor { location: LocationSpec, emit: LocationProcessorEmit, ): Promise { - if (!location.target.match(/\.ya?ml$/)) { + if (!location.target.match(/\.ya?ml/)) { return false; } diff --git a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx index 1f0eb4fead..e61f904347 100644 --- a/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx +++ b/plugins/register-component/src/components/RegisterComponentPage/RegisterComponentPage.tsx @@ -86,9 +86,7 @@ const RegisterComponentPage: FC<{}> = () => { { url: /.*/, type: 'github' }, ]; - const type = typeMapping.filter(item => { - return new RegExp(item.url).test(target); - })[0].type; + const type = typeMapping.filter(item => item.url.test(target))[0].type; const data = await catalogApi.addLocation(type, target); diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index d062655bd2..e4fc90699b 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -31,11 +31,12 @@ describe('ComponentIdValidators', () => { }); }); describe('yamlValidator', () => { - const errorMessage = "Must end with '.yaml'."; + const errorMessage = "Must contain '.yaml'."; test.each([ [true, '.yaml'], [true, 'http://example.com/blob/master/service.yaml'], [true, 'https://example.yaml'], + [true, 'https://example.com?path=abc.yaml&c=1'], [errorMessage, '.yml'], [errorMessage, 'http://example.com/blob/master/service'], [errorMessage, undefined], diff --git a/plugins/register-component/src/util/validate.ts b/plugins/register-component/src/util/validate.ts index 78d20995f5..8552872015 100644 --- a/plugins/register-component/src/util/validate.ts +++ b/plugins/register-component/src/util/validate.ts @@ -19,6 +19,6 @@ export const ComponentIdValidators = { (typeof value === 'string' && value.match(/^https:\/\//) !== null) || 'Must start with https://.', yamlValidator: (value: any) => - (typeof value === 'string' && value.match(/.yaml$/) !== null) || - "Must end with '.yaml'.", + (typeof value === 'string' && value.match(/.yaml/) !== null) || + "Must contain '.yaml'.", }; From ba8bef38a637a3544b0aaae56a839fefcb393e56 Mon Sep 17 00:00:00 2001 From: Omer Farooq Date: Tue, 25 Aug 2020 18:30:12 +1200 Subject: [PATCH 006/114] update casing --- .../RegisterComponentForm/RegisterComponentForm.test.tsx | 2 +- .../components/RegisterComponentForm/RegisterComponentForm.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index d4d2cf4789..e1226bbf32 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', + 'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', ), ).toBeInTheDocument(); diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index a61469e969..de3c54610d 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in Github, Gitlab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." + helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." inputRef={register({ required: true, validate: ComponentIdValidators, From 118e4861be8d791119614ea8acaa7fbc51aed366 Mon Sep 17 00:00:00 2001 From: Tim Jacomb Date: Tue, 25 Aug 2020 11:27:56 +0100 Subject: [PATCH 007/114] Use cookie cutter installed on host --- plugins/scaffolder-backend/package.json | 1 + .../stages/templater/cookiecutter.ts | 43 ++++++++++++------- .../scaffolder/stages/templater/helpers.ts | 43 +++++++++++++++++++ yarn.lock | 5 +++ 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 7112d0f49d..275da3700b 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -27,6 +27,7 @@ "@octokit/rest": "^18.0.0", "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", + "command-exists-promise": "^2.0.2", "compression": "^1.7.4", "cors": "^2.8.5", "dockerode": "^3.2.0", diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index fff349cf82..21b6fc3dde 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -15,11 +15,13 @@ */ import fs from 'fs-extra'; import { JsonValue } from '@backstage/config'; -import { runDockerContainer } from './helpers'; +import { runDockerContainer, runCommand } from './helpers'; import { TemplaterBase, TemplaterRunOptions } from '.'; import path from 'path'; import { TemplaterRunResult } from './types'; +const commandExists = require('command-exists-promise'); + export class CookieCutter implements TemplaterBase { private async fetchTemplateCookieCutter( directory: string, @@ -51,21 +53,30 @@ export class CookieCutter implements TemplaterBase { const templateDir = options.directory; const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`); - await runDockerContainer({ - imageName: 'spotify/backstage-cookiecutter', - args: [ - 'cookiecutter', - '--no-input', - '-o', - '/result', - '/template', - '--verbose', - ], - templateDir, - resultDir, - logStream: options.logStream, - dockerClient: options.dockerClient, - }); + const cookieCutterInstalled = await commandExists('cookiecutter'); + if (cookieCutterInstalled) { + await runCommand({ + command: 'cookiecutter', + args: ['--no-input', '-o', resultDir, templateDir, '--verbose'], + logStream: options.logStream, + }); + } else { + await runDockerContainer({ + imageName: 'spotify/backstage-cookiecutter', + args: [ + 'cookiecutter', + '--no-input', + '-o', + '/result', + '/template', + '--verbose', + ], + templateDir, + resultDir, + logStream: options.logStream, + dockerClient: options.dockerClient, + }); + } return { resultDir: path.resolve(resultDir, options.values.component_id as string), diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index a3dea7ea5a..e71b63bfb0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -18,6 +18,7 @@ import Docker from 'dockerode'; import fs from 'fs'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { InputError } from '@backstage/backend-common'; +import { spawn } from 'child_process'; export type RunDockerContainerOptions = { imageName: string; @@ -29,6 +30,12 @@ export type RunDockerContainerOptions = { createOptions?: Docker.ContainerCreateOptions; }; +export type RunCommandOptions = { + command: string; + args: string[]; + logStream?: Writable; +}; + /** * Gets the templater key to use for templating from the entity * @param entity Template entity @@ -43,6 +50,42 @@ export const getTemplaterKey = (entity: TemplateEntityV1alpha1): string => { return templater; }; +/** + * + * @param options the options object + * @param options.command the command to run + * @param options.args the arguments to pass the command + * @param options.logStream the log streamer to capture log messages + */ +export const runCommand = async ({ + command, + args, + logStream = new PassThrough(), +}: RunCommandOptions) => { + await new Promise((resolve, reject) => { + const process = spawn(command, args); + + process.stdout.on('data', stream => { + logStream.write(stream); + }); + + process.stderr.on('data', stream => { + logStream.write(stream); + }); + + process.on('error', error => { + return reject(error); + }); + + process.on('close', code => { + if (code !== 0) { + return reject(`Command ${command} failed, exit code: ${code}`); + } + return resolve(); + }); + }); +}; + /** * * @param options the options object diff --git a/yarn.lock b/yarn.lock index 857b8b2cc0..6f725fe353 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8104,6 +8104,11 @@ comma-separated-tokens@^1.0.0: resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz#632b80b6117867a158f1080ad498b2fbe7e3f5ea" integrity sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw== +command-exists-promise@^2.0.2: + version "2.0.2" + resolved "https://registry.npmjs.org/command-exists-promise/-/command-exists-promise-2.0.2.tgz#7beecc4b218299f3c61fa69a4047aa0b36a64a99" + integrity sha512-T6PB6vdFrwnHXg/I0kivM3DqaCGZLjjYSOe0a5WgFKcz1sOnmOeIjnhQPXVXX3QjVbLyTJ85lJkX6lUpukTzaA== + commander@^2.11.0, commander@^2.19.0, commander@^2.20.0, commander@^2.20.3, commander@^2.8.1, commander@~2.20.3: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" From 3ca9863f039e9f86c2c567fcf47e50ba49061bd1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 25 Aug 2020 15:24:02 +0200 Subject: [PATCH 008/114] cli: introduce jest transformModules options for providing a list of modules to transform --- package.json | 5 ++ packages/cli/config/jest.js | 87 +++++++++++-------- .../templates/default-app/package.json.hbs | 5 ++ 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index e4aa18f540..e6ee961701 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,10 @@ "*.{json,md}": [ "prettier --write" ] + }, + "jest": { + "transformModules": [ + "@kyma-project/asyncapi-react" + ] } } diff --git a/packages/cli/config/jest.js b/packages/cli/config/jest.js index cc7d45c139..f69e18190d 100644 --- a/packages/cli/config/jest.js +++ b/packages/cli/config/jest.js @@ -25,43 +25,6 @@ async function getConfig() { return require(path.resolve('jest.config.ts')); } - const options = { - rootDir: path.resolve('src'), - coverageDirectory: path.resolve('coverage'), - collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], - moduleNameMapper: { - '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), - }, - - // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed - // TODO: jest is working on module support, it's possible that we can remove this in the future - transform: { - '\\.esm\\.js$': require.resolve('jest-esm-transformer'), - '\\.(js|jsx|ts|tsx)$': [ - require.resolve('ts-jest'), - { isolatedModules: true }, - ], - '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve( - './jestFileTransform.js', - ), - }, - - // A bit more opinionated - testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], - - // Default behaviour is to not apply transforms for node_modules, but we still want - // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. - // The @kyma-project/asyncapi-react library needs to be transformed. - transformIgnorePatterns: [ - '/node_modules/(?!@kyma-project/asyncapi-react/)(?!.*\\.(?:esm\\.js|bmp|gif|jpg|jpeg|png|frag|xml|svg)$)', - ], - }; - - // Use src/setupTests.ts as the default location for configuring test env - if (fs.existsSync('src/setupTests.ts')) { - options.setupFilesAfterEnv = ['/setupTests.ts']; - } - // We read all "jest" config fields in package.json files all the way to the filesystem root. // All configs are merged together to create the final config, with longer paths taking precedence. // The merging of the configs is shallow, meaning e.g. all transforms are replaced if new ones are defined. @@ -92,6 +55,56 @@ async function getConfig() { currentPath = newPath; } + // We add an additional Jest config parameter only known by the Backstage CLI + // called `transformModules`. It's a list of modules that we want to apply + // our configured jest transformations for. + // This is useful when packages are published in untranspiled ESM or TS form. + const transformModules = pkgJsonConfigs + .flatMap(conf => { + const modules = conf.transformModules || []; + delete conf.transformModules; + return modules; + }) + .map(name => `${name}/`) + .join('|'); + const transformModulePattern = transformModules && `(?!${transformModules})`; + + const options = { + rootDir: path.resolve('src'), + coverageDirectory: path.resolve('coverage'), + collectCoverageFrom: ['**/*.{js,jsx,ts,tsx}', '!**/*.d.ts'], + moduleNameMapper: { + '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'), + }, + + // We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed + // TODO: jest is working on module support, it's possible that we can remove this in the future + transform: { + '\\.esm\\.js$': require.resolve('jest-esm-transformer'), + '\\.(js|jsx|ts|tsx)$': [ + require.resolve('ts-jest'), + { isolatedModules: true }, + ], + '\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve( + './jestFileTransform.js', + ), + }, + + // A bit more opinionated + testMatch: ['**/?(*.)test.{js,jsx,mjs,ts,tsx}'], + + // Default behaviour is to not apply transforms for node_modules, but we still want + // to apply the esm-transformer to .esm.js files, since that's what we use in backstage packages. + transformIgnorePatterns: [ + `/node_modules/${transformModulePattern}(?:(?!\\.esm).)*\\.(?:js|json)$`, + ], + }; + + // Use src/setupTests.ts as the default location for configuring test env + if (fs.existsSync('src/setupTests.ts')) { + options.setupFilesAfterEnv = ['/setupTests.ts']; + } + return Object.assign(options, ...pkgJsonConfigs); } diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index b59ca63fa3..af4384318a 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -42,5 +42,10 @@ "*.{json,md}": [ "prettier --write" ] + }, + "jest": { + "transformModules": [ + "@kyma-project/asyncapi-react" + ] } } From b8858e122026c761bbaa95a128f439557e5b14b3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 15 Aug 2020 17:25:38 +0200 Subject: [PATCH 009/114] cli: add __non_webpack_require__ to backend eslint globals --- packages/cli/config/eslint.backend.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/cli/config/eslint.backend.js b/packages/cli/config/eslint.backend.js index 8389f5c508..9b9efc7649 100644 --- a/packages/cli/config/eslint.backend.js +++ b/packages/cli/config/eslint.backend.js @@ -28,6 +28,9 @@ module.exports = { env: { jest: true, }, + globals: { + __non_webpack_require__: 'readonly', + }, parserOptions: { ecmaVersion: 2018, sourceType: 'module', From 9ae483312ee97f14f757fca1a91928ccd4ef2b7d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 15 Aug 2020 17:53:34 +0200 Subject: [PATCH 010/114] plugins: add initial app-backend --- plugins/app-backend/.eslintrc.js | 3 + plugins/app-backend/README.md | 3 + plugins/app-backend/package.json | 41 ++++++++ plugins/app-backend/scripts/mock-data.sh | 2 + plugins/app-backend/src/index.ts | 17 ++++ plugins/app-backend/src/run.ts | 33 +++++++ .../service/__fixtures__/app-dir/.gitignore | 1 + .../__fixtures__/app-dir/dist/index.html | 1 + .../__fixtures__/app-dir/dist/other.html | 1 + .../__fixtures__/app-dir/dist/static/main.txt | 1 + .../app-backend/src/service/router.test.ts | 93 +++++++++++++++++++ plugins/app-backend/src/service/router.ts | 57 ++++++++++++ .../src/service/standaloneServer.ts | 46 +++++++++ plugins/app-backend/src/setupTests.ts | 17 ++++ 14 files changed, 316 insertions(+) create mode 100644 plugins/app-backend/.eslintrc.js create mode 100644 plugins/app-backend/README.md create mode 100644 plugins/app-backend/package.json create mode 100755 plugins/app-backend/scripts/mock-data.sh create mode 100644 plugins/app-backend/src/index.ts create mode 100644 plugins/app-backend/src/run.ts create mode 100644 plugins/app-backend/src/service/__fixtures__/app-dir/.gitignore create mode 100644 plugins/app-backend/src/service/__fixtures__/app-dir/dist/index.html create mode 100644 plugins/app-backend/src/service/__fixtures__/app-dir/dist/other.html create mode 100644 plugins/app-backend/src/service/__fixtures__/app-dir/dist/static/main.txt create mode 100644 plugins/app-backend/src/service/router.test.ts create mode 100644 plugins/app-backend/src/service/router.ts create mode 100644 plugins/app-backend/src/service/standaloneServer.ts create mode 100644 plugins/app-backend/src/setupTests.ts diff --git a/plugins/app-backend/.eslintrc.js b/plugins/app-backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/app-backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/app-backend/README.md b/plugins/app-backend/README.md new file mode 100644 index 0000000000..28480b81a1 --- /dev/null +++ b/plugins/app-backend/README.md @@ -0,0 +1,3 @@ +# App backend plugin + +This backend plugin can be installed to serve static content of a Backstage app. diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json new file mode 100644 index 0000000000..a26667ef7c --- /dev/null +++ b/plugins/app-backend/package.json @@ -0,0 +1,41 @@ +{ + "name": "@backstage/plugin-app-backend", + "version": "0.1.1-alpha.18", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean", + "mock-data": "./scripts/mock-data.sh" + }, + "dependencies": { + "@backstage/backend-common": "^0.1.1-alpha.18", + "@types/express": "^4.17.6", + "express": "^4.17.1", + "express-promise-router": "^3.0.3", + "winston": "^3.2.1", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.18", + "@types/supertest": "^2.0.8", + "msw": "^0.19.5", + "supertest": "^4.0.2" + }, + "files": [ + "dist", + "static" + ] +} diff --git a/plugins/app-backend/scripts/mock-data.sh b/plugins/app-backend/scripts/mock-data.sh new file mode 100755 index 0000000000..ff30921715 --- /dev/null +++ b/plugins/app-backend/scripts/mock-data.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +echo "use this script to load your service with some mock data if needed!" diff --git a/plugins/app-backend/src/index.ts b/plugins/app-backend/src/index.ts new file mode 100644 index 0000000000..7612c392a2 --- /dev/null +++ b/plugins/app-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; diff --git a/plugins/app-backend/src/run.ts b/plugins/app-backend/src/run.ts new file mode 100644 index 0000000000..b96989e4b8 --- /dev/null +++ b/plugins/app-backend/src/run.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getRootLogger } from '@backstage/backend-common'; +import yn from 'yn'; +import { startStandaloneServer } from './service/standaloneServer'; + +const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000; +const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); +const logger = getRootLogger(); + +startStandaloneServer({ port, enableCors, logger }).catch(err => { + logger.error(err); + process.exit(1); +}); + +process.on('SIGINT', () => { + logger.info('CTRL+C pressed; exiting.'); + process.exit(0); +}); diff --git a/plugins/app-backend/src/service/__fixtures__/app-dir/.gitignore b/plugins/app-backend/src/service/__fixtures__/app-dir/.gitignore new file mode 100644 index 0000000000..cbdb9611d1 --- /dev/null +++ b/plugins/app-backend/src/service/__fixtures__/app-dir/.gitignore @@ -0,0 +1 @@ +!dist diff --git a/plugins/app-backend/src/service/__fixtures__/app-dir/dist/index.html b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/index.html new file mode 100644 index 0000000000..2085b25414 --- /dev/null +++ b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/index.html @@ -0,0 +1 @@ +this is index.html diff --git a/plugins/app-backend/src/service/__fixtures__/app-dir/dist/other.html b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/other.html new file mode 100644 index 0000000000..a4ea9022ac --- /dev/null +++ b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/other.html @@ -0,0 +1 @@ +this is other.html diff --git a/plugins/app-backend/src/service/__fixtures__/app-dir/dist/static/main.txt b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/static/main.txt new file mode 100644 index 0000000000..ec835599f6 --- /dev/null +++ b/plugins/app-backend/src/service/__fixtures__/app-dir/dist/static/main.txt @@ -0,0 +1 @@ +this is main.txt diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts new file mode 100644 index 0000000000..8910129394 --- /dev/null +++ b/plugins/app-backend/src/service/router.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import Router from 'express-promise-router'; +import request from 'supertest'; + +import { createRouter } from './router'; + +global.__non_webpack_require__ = { + resolve: () => resolvePath(__dirname, '__fixtures__/app-dir/package.json'), +}; + +describe('createRouter', () => { + let app: express.Express; + + beforeAll(async () => { + const router = await createRouter({ + logger: getVoidLogger(), + appPackageName: 'example-app', + }); + app = express().use(router); + }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('returns index.html', async () => { + const response = await request(app).get('/index.html'); + + expect(response.status).toBe(200); + expect(response.text).toBe('this is index.html\n'); + }); + + it('returns other.html', async () => { + const response = await request(app).get('/other.html'); + + expect(response.status).toBe(200); + expect(response.text).toBe('this is other.html\n'); + }); + + it('returns index.html if missing', async () => { + const response = await request(app).get('/missing.html'); + + expect(response.status).toBe(200); + expect(response.text).toBe('this is index.html\n'); + }); +}); + +describe('createRouter with static fallback handler', () => { + it('uses static fallback handler', async () => { + const staticFallbackHandler = Router(); + + staticFallbackHandler.get('/test.txt', (_req, res) => { + res.end('this is test.txt'); + }); + + const router = await createRouter({ + logger: getVoidLogger(), + appPackageName: 'example-app', + staticFallbackHandler, + }); + + const app = express().use(router); + + const response1 = await request(app).get('/static/main.txt'); + expect(response1.status).toBe(200); + expect(response1.text).toBe('this is main.txt\n'); + + const response2 = await request(app).get('/static/test.txt'); + expect(response2.status).toBe(200); + expect(response2.text).toBe('this is test.txt'); + + const response3 = await request(app).get('/static/missing.txt'); + expect(response3.status).toBe(404); + }); +}); diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts new file mode 100644 index 0000000000..efb6d4718e --- /dev/null +++ b/plugins/app-backend/src/service/router.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath, dirname } from 'path'; +import { notFoundHandler } from '@backstage/backend-common'; +import express from 'express'; +import Router from 'express-promise-router'; +import { Logger } from 'winston'; + +export interface RouterOptions { + logger: Logger; + appPackageName?: string; + staticFallbackHandler?: express.Handler; +} + +export async function createRouter( + options: RouterOptions, +): Promise { + const appDistDir = resolvePath( + dirname( + __non_webpack_require__.resolve(`${options.appPackageName}/package.json`), + ), + 'dist', + ); + options.logger.info(`Serving static app content from ${appDistDir}`); + + const router = Router(); + + // Use a separate router for static content so that a fallback can be provided by backend + const staticRouter = Router(); + staticRouter.use(express.static(resolvePath(appDistDir, 'static'))); + if (options.staticFallbackHandler) { + staticRouter.use(options.staticFallbackHandler); + } + staticRouter.use(notFoundHandler()); + + router.use('/static', staticRouter); + router.use(express.static(appDistDir)); + router.get('/*', (_req, res) => { + res.sendFile(resolvePath(appDistDir, 'index.html')); + }); + + return router; +} diff --git a/plugins/app-backend/src/service/standaloneServer.ts b/plugins/app-backend/src/service/standaloneServer.ts new file mode 100644 index 0000000000..8abf3b81f2 --- /dev/null +++ b/plugins/app-backend/src/service/standaloneServer.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceBuilder } from '@backstage/backend-common'; +import { Server } from 'http'; +import { Logger } from 'winston'; +import { createRouter } from './router'; + +export interface ServerOptions { + port: number; + enableCors: boolean; + logger: Logger; +} + +export async function startStandaloneServer( + options: ServerOptions, +): Promise { + const logger = options.logger.child({ service: 'app-backend' }); + logger.debug('Starting application server...'); + const router = await createRouter({ + logger, + appPackageName: 'example-app', + }); + + const service = createServiceBuilder(module).addRouter('', router); + + return await service.start().catch(err => { + logger.error(err); + process.exit(1); + }); +} + +module.hot?.accept(); diff --git a/plugins/app-backend/src/setupTests.ts b/plugins/app-backend/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/plugins/app-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; From c13de0b115907fff19753f2c439de136b9c16f46 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 16 Aug 2020 09:10:09 +0200 Subject: [PATCH 011/114] config-loader: rename readEnv to readEnvConfig and export from package --- packages/config-loader/src/index.ts | 1 + packages/config-loader/src/lib/env.test.ts | 42 +++++++++++----------- packages/config-loader/src/lib/env.ts | 2 +- packages/config-loader/src/lib/index.ts | 2 +- packages/config-loader/src/loader.ts | 4 +-- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packages/config-loader/src/index.ts b/packages/config-loader/src/index.ts index 6738611a17..02db134fe5 100644 --- a/packages/config-loader/src/index.ts +++ b/packages/config-loader/src/index.ts @@ -14,5 +14,6 @@ * limitations under the License. */ +export { readEnvConfig } from './lib'; export { loadConfig } from './loader'; export type { LoadConfigOptions } from './loader'; diff --git a/packages/config-loader/src/lib/env.test.ts b/packages/config-loader/src/lib/env.test.ts index 93bd962596..6b1e49365a 100644 --- a/packages/config-loader/src/lib/env.test.ts +++ b/packages/config-loader/src/lib/env.test.ts @@ -14,16 +14,16 @@ * limitations under the License. */ -import { readEnv } from './env'; +import { readEnvConfig } from './env'; -describe('readEnv', () => { +describe('readEnvConfig', () => { it('should return empty config for empty env', () => { - expect(readEnv({})).toEqual([]); + expect(readEnvConfig({})).toEqual([]); }); it('should return empty config for no matching keys', () => { expect( - readEnv({ + readEnvConfig({ NODE_ENV: 'production', NOPE_ENV: 'development', APP_CONFIG: 'foo', @@ -34,7 +34,7 @@ describe('readEnv', () => { it('should create config from env', () => { expect( - readEnv({ + readEnvConfig({ NODE_ENV: 'production', APP_CONFIG_foo: '"bar"', APP_CONFIG_numbers_a: '1', @@ -57,22 +57,22 @@ describe('readEnv', () => { }); it('should accept string values', () => { - expect(readEnv({ APP_CONFIG_foo: '"abc"', APP_CONFIG_bar: 'xyz' })).toEqual( - [ - { - data: { - foo: 'abc', - bar: 'xyz', - }, - context: 'env', + expect( + readEnvConfig({ APP_CONFIG_foo: '"abc"', APP_CONFIG_bar: 'xyz' }), + ).toEqual([ + { + data: { + foo: 'abc', + bar: 'xyz', }, - ], - ); + context: 'env', + }, + ]); }); it('should accept complex objects', () => { expect( - readEnv({ + readEnvConfig({ APP_CONFIG_foo: '{ "a": 123, "b": "123", "c": [] }', APP_CONFIG_bar: '[123, "abc", {}]', }), @@ -95,7 +95,7 @@ describe('readEnv', () => { ['APP_CONFIG_fo o'], ['APP_CONFIG_foo_(foo)_foo'], ])('should reject invalid key %p', key => { - expect(() => readEnv({ [key]: '0' })).toThrow( + expect(() => readEnvConfig({ [key]: '0' })).toThrow( `Invalid env config key '${key.replace('APP_CONFIG_', '')}'`, ); }); @@ -103,7 +103,7 @@ describe('readEnv', () => { it.each([['hello'], ['"hello'], ['{'], ['}']])( 'should fallback to string when invalid json value %p', value => { - expect(readEnv({ APP_CONFIG_foo: value })).toEqual([ + expect(readEnvConfig({ APP_CONFIG_foo: value })).toEqual([ { data: { foo: value, @@ -116,7 +116,7 @@ describe('readEnv', () => { it('should not allow null as a value', () => { expect(() => - readEnv({ + readEnvConfig({ APP_CONFIG_foo: 'null', }), ).toThrow( @@ -126,7 +126,7 @@ describe('readEnv', () => { it('should not allow duplicate values', () => { expect(() => - readEnv({ + readEnvConfig({ APP_CONFIG_foo_bar: '1', APP_CONFIG_foo_bar_baz: '2', }), @@ -137,7 +137,7 @@ describe('readEnv', () => { it('should not allow mixing of objects and other values', () => { expect(() => - readEnv({ + readEnvConfig({ APP_CONFIG_nested_foo: '1', APP_CONFIG_nested: '2', }), diff --git a/packages/config-loader/src/lib/env.ts b/packages/config-loader/src/lib/env.ts index 53c6d6c63b..84d39263e3 100644 --- a/packages/config-loader/src/lib/env.ts +++ b/packages/config-loader/src/lib/env.ts @@ -39,7 +39,7 @@ const CONFIG_KEY_PART_PATTERN = /^[a-z][a-z0-9]*(?:[-_][a-z][a-z0-9]*)*$/i; * * APP_CONFIG_app_title='"My Title"' */ -export function readEnv(env: { +export function readEnvConfig(env: { [name: string]: string | undefined; }): AppConfig[] { let data: JsonObject | undefined = undefined; diff --git a/packages/config-loader/src/lib/index.ts b/packages/config-loader/src/lib/index.ts index 226932784b..40252b9cec 100644 --- a/packages/config-loader/src/lib/index.ts +++ b/packages/config-loader/src/lib/index.ts @@ -16,5 +16,5 @@ export { resolveStaticConfig } from './resolver'; export { readConfigFile } from './reader'; -export { readEnv } from './env'; +export { readEnvConfig } from './env'; export { readSecret } from './secrets'; diff --git a/packages/config-loader/src/loader.ts b/packages/config-loader/src/loader.ts index 679ab71e45..cf5edd3ce3 100644 --- a/packages/config-loader/src/loader.ts +++ b/packages/config-loader/src/loader.ts @@ -20,7 +20,7 @@ import { AppConfig, JsonObject } from '@backstage/config'; import { resolveStaticConfig, readConfigFile, - readEnv, + readEnvConfig, readSecret, } from './lib'; @@ -102,7 +102,7 @@ export async function loadConfig( ); } - configs.push(...readEnv(process.env)); + configs.push(...readEnvConfig(process.env)); return configs; } From 35e1af6efaf31811d140f4c4eb01896d94a9063d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 17 Aug 2020 11:51:13 +0200 Subject: [PATCH 012/114] plugins/app-backend: add env config injection --- plugins/app-backend/package.json | 2 + plugins/app-backend/src/lib/config.test.ts | 136 ++++++++++++++++++ plugins/app-backend/src/lib/config.ts | 74 ++++++++++ .../app-backend/src/service/router.test.ts | 2 + plugins/app-backend/src/service/router.ts | 7 + 5 files changed, 221 insertions(+) create mode 100644 plugins/app-backend/src/lib/config.test.ts create mode 100644 plugins/app-backend/src/lib/config.ts diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index a26667ef7c..29902e9806 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -22,9 +22,11 @@ }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.18", + "@backstage/config-loader": "^0.1.1-alpha.18", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", + "fs-extra": "^9.0.0", "winston": "^3.2.1", "yn": "^4.0.0" }, diff --git a/plugins/app-backend/src/lib/config.test.ts b/plugins/app-backend/src/lib/config.test.ts new file mode 100644 index 0000000000..16c5600b1d --- /dev/null +++ b/plugins/app-backend/src/lib/config.test.ts @@ -0,0 +1,136 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'path'; +import { getVoidLogger } from '@backstage/backend-common'; +import { injectEnvConfig } from './config'; + +jest.mock('fs-extra'); + +const fsMock = fs as jest.Mocked; +const readFileMock = (fsMock.readFile as unknown) as jest.MockedFunction< + (name: string) => Promise +>; + +const MOCK_DIR = 'mock-dir'; + +const baseOptions = { + env: {}, + staticDir: MOCK_DIR, + logger: getVoidLogger(), +}; + +describe('injectEnvConfig', () => { + beforeEach(() => { + fsMock.readdir.mockResolvedValue(['main.js']); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should not inject without config', async () => { + await injectEnvConfig(baseOptions); + expect(fsMock.readdir).toHaveBeenCalledTimes(0); + expect(fsMock.readFile).toHaveBeenCalledTimes(0); + expect(fsMock.writeFile).toHaveBeenCalledTimes(0); + }); + + it('should find the correct file to inject', async () => { + fsMock.readdir.mockResolvedValue([ + 'before.js', + 'not-js.txt', + 'main.js', + 'after.js', + ]); + readFileMock.mockImplementation(async (file: string) => { + if (file.endsWith('main.js')) { + return '"__APP_INJECTED_RUNTIME_CONFIG__"'; + } + return 'NO_PLACEHOLDER_HERE'; + }); + + await injectEnvConfig({ ...baseOptions, env: { APP_CONFIG_x: '0' } }); + expect(fsMock.readFile).toHaveBeenCalledTimes(2); + expect(fsMock.readFile).toHaveBeenNthCalledWith( + 1, + resolvePath(MOCK_DIR, 'before.js'), + 'utf8', + ); + expect(fsMock.readFile).toHaveBeenNthCalledWith( + 2, + resolvePath(MOCK_DIR, 'main.js'), + 'utf8', + ); + + expect(fsMock.writeFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledWith( + resolvePath(MOCK_DIR, 'main.js'), + '/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/', + 'utf8', + ); + + // eslint-disable-next-line no-eval + expect(JSON.parse(eval(fsMock.writeFile.mock.calls[0][1]))).toEqual({ + x: 0, + }); + }); + + it('should re-inject config', async () => { + fsMock.readdir.mockResolvedValue(['main.js']); + readFileMock.mockResolvedValue( + 'JSON.parse("__APP_INJECTED_RUNTIME_CONFIG__")', + ); + + await injectEnvConfig({ + ...baseOptions, + env: { + APP_CONFIG_x: '0', + }, + }); + + expect(fsMock.writeFile).toHaveBeenCalledTimes(1); + expect(fsMock.writeFile).toHaveBeenCalledWith( + resolvePath(MOCK_DIR, 'main.js'), + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":0}"/*__INJECTED_END__*/)', + 'utf8', + ); + + // eslint-disable-next-line no-eval + expect(eval(fsMock.writeFile.mock.calls[0][1])).toEqual({ x: 0 }); + + readFileMock.mockResolvedValue(fsMock.writeFile.mock.calls[0][1]); + + await injectEnvConfig({ + ...baseOptions, + env: { + APP_CONFIG_x: '1', + APP_CONFIG_y: '2', + }, + }); + + expect(fsMock.writeFile).toHaveBeenCalledTimes(2); + expect(fsMock.writeFile).toHaveBeenLastCalledWith( + resolvePath(MOCK_DIR, 'main.js'), + 'JSON.parse(/*__APP_INJECTED_CONFIG_MARKER__*/"{\\"x\\":1,\\"y\\":2}"/*__INJECTED_END__*/)', + 'utf8', + ); + + // eslint-disable-next-line no-eval + expect(eval(fsMock.writeFile.mock.calls[1][1])).toEqual({ x: 1, y: 2 }); + }); +}); diff --git a/plugins/app-backend/src/lib/config.ts b/plugins/app-backend/src/lib/config.ts new file mode 100644 index 0000000000..0ed56ef722 --- /dev/null +++ b/plugins/app-backend/src/lib/config.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'path'; +import { readEnvConfig } from '@backstage/config-loader'; +import { Logger } from 'winston'; + +type Options = { + // Environment to read config from + env: { [name: string]: string | undefined }; + // Directory of the static JS files to search for file to inject + staticDir: string; + logger: Logger; +}; + +/** + * Injects config from APP_CONFIG_ env vars, replacing existing + * injected config if it has already been injected. + */ +export async function injectEnvConfig(options: Options) { + const { env, staticDir, logger } = options; + + const envConfig = readEnvConfig(env); + if (envConfig.length === 0) { + return; + } + + const files = await fs.readdir(staticDir); + const jsFiles = files.filter(file => file.endsWith('.js')); + + const [{ data }] = envConfig; + const escapedData = JSON.stringify(data).replace(/("|'|\\)/g, '\\$1'); + const injected = `/*__APP_INJECTED_CONFIG_MARKER__*/"${escapedData}"/*__INJECTED_END__*/`; + + for (const jsFile of jsFiles) { + const path = resolvePath(staticDir, jsFile); + + const content = await fs.readFile(path, 'utf8'); + if (content.includes('__APP_INJECTED_RUNTIME_CONFIG__')) { + logger.info(`Injecting env config into ${jsFile}`); + + const newContent = content.replace( + '"__APP_INJECTED_RUNTIME_CONFIG__"', + injected, + ); + await fs.writeFile(path, newContent, 'utf8'); + return; + } else if (content.includes('__APP_INJECTED_CONFIG_MARKER__')) { + logger.info(`Replacing injected env config in ${jsFile}`); + + const newContent = content.replace( + /\/\*__APP_INJECTED_CONFIG_MARKER__\*\/.*\/\*__INJECTED_END__\*\//, + injected, + ); + await fs.writeFile(path, newContent, 'utf8'); + return; + } + } + logger.info('Env config not injected'); +} diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index 8910129394..4e0c27c47a 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -22,6 +22,8 @@ import request from 'supertest'; import { createRouter } from './router'; +jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() })); + global.__non_webpack_require__ = { resolve: () => resolvePath(__dirname, '__fixtures__/app-dir/package.json'), }; diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index efb6d4718e..ecc5b4fa4b 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -19,6 +19,7 @@ import { notFoundHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; +import { injectEnvConfig } from '../lib/config'; export interface RouterOptions { logger: Logger; @@ -37,6 +38,12 @@ export async function createRouter( ); options.logger.info(`Serving static app content from ${appDistDir}`); + await injectEnvConfig({ + env: process.env, + logger: options.logger, + staticDir: resolvePath(appDistDir, 'static'), + }); + const router = Router(); // Use a separate router for static content so that a fallback can be provided by backend From 5cb3cf7b9064e0c9ba23a540f6ed90189c3fde29 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 17 Aug 2020 12:57:16 +0200 Subject: [PATCH 013/114] backend: install app-backend --- packages/backend/package.json | 2 ++ packages/backend/src/index.ts | 5 ++++- packages/backend/src/plugins/app.ts | 25 +++++++++++++++++++++++++ plugins/app-backend/package.json | 8 ++++---- 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 packages/backend/src/plugins/app.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 9c3c5a5f58..7393ede81e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -18,9 +18,11 @@ "migrate:create": "knex migrate:make -x ts" }, "dependencies": { + "example-app": "^0.1.1-alpha.20", "@backstage/backend-common": "^0.1.1-alpha.20", "@backstage/catalog-model": "^0.1.1-alpha.20", "@backstage/config": "^0.1.1-alpha.20", + "@backstage/plugin-app-backend": "^0.1.1-alpha.20", "@backstage/plugin-auth-backend": "^0.1.1-alpha.20", "@backstage/plugin-catalog-backend": "^0.1.1-alpha.20", "@backstage/plugin-graphql-backend": "^0.1.1-alpha.20", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 50f59795fc..21c8794b28 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -40,6 +40,7 @@ import sentry from './plugins/sentry'; import proxy from './plugins/proxy'; import techdocs from './plugins/techdocs'; import graphql from './plugins/graphql'; +import app from './plugins/app'; import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { @@ -71,6 +72,7 @@ async function main() { const sentryEnv = useHotMemoize(module, () => createEnv('sentry')); const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const graphqlEnv = useHotMemoize(module, () => createEnv('graphql')); + const appEnv = useHotMemoize(module, () => createEnv('app')); const service = createServiceBuilder(module) .loadConfig(configReader) @@ -83,7 +85,8 @@ async function main() { .addRouter('/identity', await identity(identityEnv)) .addRouter('/techdocs', await techdocs(techdocsEnv)) .addRouter('/proxy', await proxy(proxyEnv)) - .addRouter('/graphql', await graphql(graphqlEnv)); + .addRouter('/graphql', await graphql(graphqlEnv)) + .addRouter('', await app(appEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/backend/src/plugins/app.ts b/packages/backend/src/plugins/app.ts new file mode 100644 index 0000000000..c9f7c0622a --- /dev/null +++ b/packages/backend/src/plugins/app.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from '@backstage/plugin-app-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ logger }: PluginEnvironment) { + return await createRouter({ + logger, + appPackageName: 'example-app', + }); +} diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index 29902e9806..61977200d5 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-app-backend", - "version": "0.1.1-alpha.18", + "version": "0.1.1-alpha.20", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -21,8 +21,8 @@ "mock-data": "./scripts/mock-data.sh" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.18", - "@backstage/config-loader": "^0.1.1-alpha.18", + "@backstage/backend-common": "^0.1.1-alpha.20", + "@backstage/config-loader": "^0.1.1-alpha.20", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", @@ -31,7 +31,7 @@ "yn": "^4.0.0" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.18", + "@backstage/cli": "^0.1.1-alpha.20", "@types/supertest": "^2.0.8", "msw": "^0.19.5", "supertest": "^4.0.2" From 5cc4c55e3899b54dfa263f56c06df9e6d882ed02 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 17 Aug 2020 13:20:25 +0200 Subject: [PATCH 014/114] plugins/app-backend: add installation instructions --- plugins/app-backend/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/plugins/app-backend/README.md b/plugins/app-backend/README.md index 28480b81a1..5d290093d7 100644 --- a/plugins/app-backend/README.md +++ b/plugins/app-backend/README.md @@ -1,3 +1,32 @@ # App backend plugin This backend plugin can be installed to serve static content of a Backstage app. + +## Installation + +Add both this package and your local frontend app package as dependencies to your backend, for example + +```bash +yarn add @backstage/plugin-app-backend example-app +``` + +By adding the app package as a dependency we ensure that it is build as part of the backend, and that it can be resolved at runtime. + +Now add the plugin router to your app, creating it for example like this: + +```ts +const router = await createRouter({ + logger, + appPackageName: 'example-app', +}); +``` + +And registering it like this: + +```ts +createServiceBuilder(module) + ... + .addRouter('', router); +``` + +Be sure to register the app router last, as it serves content for HTML5-mode navigation, i.e. falling back to serving `index.html` for any route that can't be found. From e405fff068c38788dfebd1e0ef9f41db903574dd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 27 Aug 2020 07:25:16 +0200 Subject: [PATCH 015/114] plugins/app-backend: remove mock-data script --- plugins/app-backend/package.json | 3 +-- plugins/app-backend/scripts/mock-data.sh | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100755 plugins/app-backend/scripts/mock-data.sh diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index 61977200d5..46cd8dcf25 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -17,8 +17,7 @@ "test": "backstage-cli test", "prepack": "backstage-cli prepack", "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean", - "mock-data": "./scripts/mock-data.sh" + "clean": "backstage-cli clean" }, "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.20", diff --git a/plugins/app-backend/scripts/mock-data.sh b/plugins/app-backend/scripts/mock-data.sh deleted file mode 100755 index ff30921715..0000000000 --- a/plugins/app-backend/scripts/mock-data.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -echo "use this script to load your service with some mock data if needed!" From 8c33d30705ca327aa4d9e345d272dea14d2f40e1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 27 Aug 2020 07:28:27 +0200 Subject: [PATCH 016/114] plugins/app-backend: fix typo --- plugins/app-backend/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/app-backend/README.md b/plugins/app-backend/README.md index 5d290093d7..2dc302accb 100644 --- a/plugins/app-backend/README.md +++ b/plugins/app-backend/README.md @@ -10,7 +10,7 @@ Add both this package and your local frontend app package as dependencies to you yarn add @backstage/plugin-app-backend example-app ``` -By adding the app package as a dependency we ensure that it is build as part of the backend, and that it can be resolved at runtime. +By adding the app package as a dependency we ensure that it is built as part of the backend, and that it can be resolved at runtime. Now add the plugin router to your app, creating it for example like this: From 23c34ff0b149601371d09d980ed9275218b347e4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2020 11:05:18 +0000 Subject: [PATCH 017/114] chore(deps): bump whatwg-fetch from 2.0.4 to 3.4.0 Bumps [whatwg-fetch](https://github.com/github/fetch) from 2.0.4 to 3.4.0. - [Release notes](https://github.com/github/fetch/releases) - [Commits](https://github.com/github/fetch/compare/v2.0.4...v3.4.0) Signed-off-by: dependabot-preview[bot] --- plugins/catalog/package.json | 2 +- plugins/graphql/package.json | 2 +- yarn.lock | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index 2c9c09f688..0f82add554 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -54,7 +54,7 @@ "jest-fetch-mock": "^3.0.3", "msw": "^0.20.5", "react-test-renderer": "^16.13.1", - "whatwg-fetch": "^2.0.0" + "whatwg-fetch": "^3.4.0" }, "files": [ "dist" diff --git a/plugins/graphql/package.json b/plugins/graphql/package.json index 3aaabdd3f9..045cd471a1 100644 --- a/plugins/graphql/package.json +++ b/plugins/graphql/package.json @@ -27,7 +27,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "graphql": "^15.3.0", - "whatwg-fetch": "^2.0.0", + "whatwg-fetch": "^3.4.0", "winston": "^3.2.1", "yn": "^4.0.0" }, diff --git a/yarn.lock b/yarn.lock index 0c7305b955..e039de0a47 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22598,15 +22598,15 @@ whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3, whatwg-encoding@^1.0.5: dependencies: iconv-lite "0.4.24" -whatwg-fetch@^2.0.0, whatwg-fetch@^2.0.4: +whatwg-fetch@^2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz#dde6a5df315f9d39991aa17621853d720b85566f" integrity sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng== -whatwg-fetch@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb" - integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q== +whatwg-fetch@^3.0.0, whatwg-fetch@^3.4.0: + version "3.4.0" + resolved "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.4.0.tgz#e11de14f4878f773fbebcde8871b2c0699af8b30" + integrity sha512-rsum2ulz2iuZH08mJkT0Yi6JnKhwdw4oeyMjokgxd+mmqYSd9cPpOQf01TIWgjxG/U4+QR+AwKq6lSbXVxkyoQ== whatwg-mimetype@^2.1.0, whatwg-mimetype@^2.2.0, whatwg-mimetype@^2.3.0: version "2.3.0" From ffa9886051b3bb1aed694531898c8f9227da7ccd Mon Sep 17 00:00:00 2001 From: ebarrios Date: Thu, 27 Aug 2020 18:30:59 +0200 Subject: [PATCH 018/114] Added Widget for listing workflows run to catalog under CI/CD tab --- .../src/components/EntityPage/EntityPage.tsx | 2 ++ .../components/EntityPageCi/EntityPageCi.tsx | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx diff --git a/plugins/catalog/src/components/EntityPage/EntityPage.tsx b/plugins/catalog/src/components/EntityPage/EntityPage.tsx index adbc758067..558f4b6551 100644 --- a/plugins/catalog/src/components/EntityPage/EntityPage.tsx +++ b/plugins/catalog/src/components/EntityPage/EntityPage.tsx @@ -36,6 +36,7 @@ import { catalogApiRef } from '../..'; import { EntityContextMenu } from '../EntityContextMenu/EntityContextMenu'; import { EntityPageDocs } from '../EntityPageDocs/EntityDocsPage'; import { EntityPageApi } from '../EntityPageApi/EntityPageApi'; +import { EntityPageCi } from '../EntityPageCi/EntityPageCi'; import { EntityPageOverview } from '../EntityPageOverview/EntityPageOverview'; import { FavouriteEntity } from '../FavouriteEntity/FavouriteEntity'; import { UnregisterEntityDialog } from '../UnregisterEntityDialog/UnregisterEntityDialog'; @@ -129,6 +130,7 @@ export const EntityPage: FC<{}> = () => { { id: 'ci', label: 'CI/CD', + content: (e: Entity) => , }, { id: 'tests', diff --git a/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx new file mode 100644 index 0000000000..01de6dc9a2 --- /dev/null +++ b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Content } from '@backstage/core'; +import { WidgetList as GithubActionsListWidget } from '@backstage/plugin-github-actions'; +import { Grid } from '@material-ui/core'; +import React, { FC } from 'react'; + +export const EntityPageCi: FC<{ entity: Entity }> = ({ entity }) => { + return ( + + + {entity.metadata?.annotations?.['backstage.io/github-actions-id'] && ( + + + + )} + + + ); +}; From de7c7268b3746e5f6abff90d5e84e1022ac0db85 Mon Sep 17 00:00:00 2001 From: ebarrios Date: Thu, 27 Aug 2020 18:30:59 +0200 Subject: [PATCH 019/114] Added Widget for listing workflows run to catalog under CI/CD tab --- .../src/components/EntityPage/EntityPage.tsx | 2 ++ .../components/EntityPageCi/EntityPageCi.tsx | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx diff --git a/plugins/catalog/src/components/EntityPage/EntityPage.tsx b/plugins/catalog/src/components/EntityPage/EntityPage.tsx index adbc758067..558f4b6551 100644 --- a/plugins/catalog/src/components/EntityPage/EntityPage.tsx +++ b/plugins/catalog/src/components/EntityPage/EntityPage.tsx @@ -36,6 +36,7 @@ import { catalogApiRef } from '../..'; import { EntityContextMenu } from '../EntityContextMenu/EntityContextMenu'; import { EntityPageDocs } from '../EntityPageDocs/EntityDocsPage'; import { EntityPageApi } from '../EntityPageApi/EntityPageApi'; +import { EntityPageCi } from '../EntityPageCi/EntityPageCi'; import { EntityPageOverview } from '../EntityPageOverview/EntityPageOverview'; import { FavouriteEntity } from '../FavouriteEntity/FavouriteEntity'; import { UnregisterEntityDialog } from '../UnregisterEntityDialog/UnregisterEntityDialog'; @@ -129,6 +130,7 @@ export const EntityPage: FC<{}> = () => { { id: 'ci', label: 'CI/CD', + content: (e: Entity) => , }, { id: 'tests', diff --git a/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx new file mode 100644 index 0000000000..01de6dc9a2 --- /dev/null +++ b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity } from '@backstage/catalog-model'; +import { Content } from '@backstage/core'; +import { WidgetList as GithubActionsListWidget } from '@backstage/plugin-github-actions'; +import { Grid } from '@material-ui/core'; +import React, { FC } from 'react'; + +export const EntityPageCi: FC<{ entity: Entity }> = ({ entity }) => { + return ( + + + {entity.metadata?.annotations?.['backstage.io/github-actions-id'] && ( + + + + )} + + + ); +}; From 6d774764cd6e444cfe672af46478ca28781fd92c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 26 Aug 2020 16:18:21 +0200 Subject: [PATCH 020/114] feat(proxy-backend): more proxy config tweaks and docs --- app-config.yaml | 12 ++--- packages/backend/src/index.ts | 2 +- packages/backend/src/plugins/proxy.ts | 11 +++-- plugins/proxy-backend/README.md | 46 ++++++++++++++++++- .../proxy-backend/src/service/router.test.ts | 1 + plugins/proxy-backend/src/service/router.ts | 42 +++++++++++++++-- .../src/service/standaloneServer.ts | 1 + 7 files changed, 95 insertions(+), 20 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 0a85198630..2efe6adc32 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -14,21 +14,15 @@ backend: client: sqlite3 connection: ':memory:' +# See README.md in the proxy-backend plugin for information on the configuration format proxy: - '/circleci/api': - target: 'https://circleci.com/api/v1.1' - changeOrigin: true - pathRewrite: - '^/proxy/circleci/api/': '/' + '/circleci/api': https://circleci.com/api/v1.1 '/jenkins/api': - target: 'http://localhost:8080' - changeOrigin: true + target: http://localhost:8080 headers: Authorization: $secret: env: JENKINS_BASIC_AUTH_HEADER - pathRewrite: - '^/proxy/jenkins/api/': '/' organization: name: Spotify diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 50f59795fc..91b1af06fb 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -82,7 +82,7 @@ async function main() { .addRouter('/auth', await auth(authEnv)) .addRouter('/identity', await identity(identityEnv)) .addRouter('/techdocs', await techdocs(techdocsEnv)) - .addRouter('/proxy', await proxy(proxyEnv)) + .addRouter('/proxy', await proxy(proxyEnv, '/proxy')) .addRouter('/graphql', await graphql(graphqlEnv)); await service.start().catch(err => { diff --git a/packages/backend/src/plugins/proxy.ts b/packages/backend/src/plugins/proxy.ts index 4964de130e..e96acf69d3 100644 --- a/packages/backend/src/plugins/proxy.ts +++ b/packages/backend/src/plugins/proxy.ts @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // @ts-ignore import { createRouter } from '@backstage/plugin-proxy-backend'; import { PluginEnvironment } from '../types'; -export default async function createPlugin({ - logger, - config, -}: PluginEnvironment) { - return await createRouter({ logger, config }); +export default async function createPlugin( + { logger, config }: PluginEnvironment, + pathPrefix: string, +) { + return await createRouter({ logger, config, pathPrefix }); } diff --git a/plugins/proxy-backend/README.md b/plugins/proxy-backend/README.md index d12031f495..baa2709820 100644 --- a/plugins/proxy-backend/README.md +++ b/plugins/proxy-backend/README.md @@ -19,8 +19,8 @@ const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); const service = createServiceBuilder(module) .loadConfig(configReader) - /** several different routers */ - .addRouter('/', await proxy(proxyEnv)); + /** ... other routers ... */ + .addRouter('/proxy', await proxy(proxyEnv, '/proxy')); ``` 2. Start the backend @@ -31,6 +31,48 @@ yarn workspace example-backend start This will launch the full example backend. +## Configuration + +Example: + +```yaml +# in app-config.yaml +proxy: + '/simple-example': http://simple.example.com:8080 + '/larger-example/v1': + target: http://larger.example.com:8080/svc.v1 + headers: + Authorization: + $secret: + env: EXAMPLE_AUTH_HEADER +``` + +Each key under the proxy configuration entry is a route to match, below the prefix that the proxy +plugin is mounted on. For example, if the backend mounts the proxy plugin as `/proxy`, +the above configuration will lead to the proxy acting on backend requests to +`/proxy/simple-example/...` and `/proxy/larger-example/v1/...`. + +The value inside each route is either a simple URL string, or an object on the format accepted by +[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). + +If the value is a string, it is assumed to correspond to: + +```yaml +target: +changeOrigin: true +pathRewrite: + '^/': '/' +``` + +When the target is an object, it is given verbatim to `http-proxy-middleware` except with the +following caveats for convenience: + +- If `changeOrigin` is not specified, it is set to `true`. This is the most commonly useful value. +- If `pathRewrite` is not specified, it is set to a single rewrite that removes the entire route. In the + above example, a rewrite of `'^/proxy/larger-example/v1/': '/'` is added. That means that a request to + `/proxy/larger-example/v1/some/path` will be translated to a request to + `http://larger.example.com:8080/svc.v1/some/path`. + ## Links - [http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware) diff --git a/plugins/proxy-backend/src/service/router.test.ts b/plugins/proxy-backend/src/service/router.test.ts index 085647bc4e..c7d7fff1d4 100644 --- a/plugins/proxy-backend/src/service/router.test.ts +++ b/plugins/proxy-backend/src/service/router.test.ts @@ -26,6 +26,7 @@ describe('createRouter', () => { const router = await createRouter({ config, logger, + pathPrefix: '/proxy', }); expect(router).toBeDefined(); }); diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts index b71299ef7e..5273b2c806 100644 --- a/plugins/proxy-backend/src/service/router.ts +++ b/plugins/proxy-backend/src/service/router.ts @@ -17,21 +17,57 @@ import { Config } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; -import createProxyMiddleware from 'http-proxy-middleware'; +import createProxyMiddleware, { + Config as ProxyConfig, + Proxy, +} from 'http-proxy-middleware'; import { Logger } from 'winston'; export interface RouterOptions { logger: Logger; config: Config; + // The URL path prefix that the router itself is mounted as, commonly "/proxy" + pathPrefix: string; +} + +// Creates a proxy middleware, possibly with defaults added on top of the +// given config. +function buildMiddleware( + pathPrefix: string, + route: string, + config: string | ProxyConfig, +): Proxy { + const fullConfig = + typeof config === 'string' ? { target: config } : { ...config }; + + // Default is to do a path rewrite that strips out the proxy's path prefix + // and the rest of the route. + if (fullConfig.pathRewrite === undefined) { + const routeWithSlash = route.endsWith('/') ? route : `${route}/`; + fullConfig.pathRewrite = { + [`^${pathPrefix}${routeWithSlash}`]: '/', + }; + } + + // Default is to update the Host header to the target + if (fullConfig.changeOrigin === undefined) { + fullConfig.changeOrigin = true; + } + + return createProxyMiddleware(fullConfig); } export async function createRouter( options: RouterOptions, ): Promise { const router = Router(); - const proxyConfig = options.config.get('proxy') ?? {}; + + const proxyConfig = options.config.getOptional('proxy') ?? {}; Object.entries(proxyConfig).forEach(([route, proxyRouteConfig]) => { - router.use(route, createProxyMiddleware(proxyRouteConfig)); + router.use( + route, + buildMiddleware(options.pathPrefix, route, proxyRouteConfig), + ); }); return router; diff --git a/plugins/proxy-backend/src/service/standaloneServer.ts b/plugins/proxy-backend/src/service/standaloneServer.ts index f75c044f0f..68e9ade183 100644 --- a/plugins/proxy-backend/src/service/standaloneServer.ts +++ b/plugins/proxy-backend/src/service/standaloneServer.ts @@ -40,6 +40,7 @@ export async function startStandaloneServer( const router = await createRouter({ config, logger, + pathPrefix: '/proxy', }); const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) From b2f62347e8ebb80cee183931de34cedc04d4b65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 27 Aug 2020 09:26:06 +0200 Subject: [PATCH 021/114] fix tests --- .../default-app/packages/backend/src/index.ts | 2 +- .../default-app/packages/backend/src/plugins/proxy.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/index.ts b/packages/create-app/templates/default-app/packages/backend/src/index.ts index 0e1a85f0a3..d6fba2478a 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/index.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/index.ts @@ -55,7 +55,7 @@ async function main() { .addRouter('/auth', await auth(authEnv)) .addRouter('/identity', await identity(identityEnv)) .addRouter('/techdocs', await techdocs(techdocsEnv)) - .addRouter('/proxy', await proxy(proxyEnv)); + .addRouter('/proxy', await proxy(proxyEnv, '/proxy')); await service.start().catch(err => { console.log(err); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/proxy.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/proxy.ts index 574d0c17a7..b20265c2cd 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/proxy.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/proxy.ts @@ -2,9 +2,9 @@ import { createRouter } from '@backstage/plugin-proxy-backend'; import { PluginEnvironment } from '../types'; -export default async function createPlugin({ - logger, - config, -}: PluginEnvironment) { - return await createRouter({ logger, config }); +export default async function createPlugin( + { logger, config }: PluginEnvironment, + pathPrefix: string, +) { + return await createRouter({ logger, config, pathPrefix }); } From 9a64135bf7d66642da35b574070828e2d7d0c0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 27 Aug 2020 12:06:53 +0200 Subject: [PATCH 022/114] move docs to the microsite --- docs/plugins/call-existing-api.md | 11 ++--- docs/plugins/proxying.md | 71 ++++++++++++++++++++++++++++++- plugins/proxy-backend/README.md | 66 +++++----------------------- 3 files changed, 83 insertions(+), 65 deletions(-) diff --git a/docs/plugins/call-existing-api.md b/docs/plugins/call-existing-api.md index f6c9990b90..9658176224 100644 --- a/docs/plugins/call-existing-api.md +++ b/docs/plugins/call-existing-api.md @@ -71,11 +71,7 @@ Example: ```yaml # In app-config.yaml proxy: - '/frobs': - target: 'http://api.frobsco.com/v1' - changeOrigin: true - pathRewrite: - '^/proxy/frobs/': '/' + '/frobs': http://api.frobsco.com/v1 ``` ```ts @@ -86,9 +82,8 @@ fetch(`${backendUrl}/proxy/frobs/list`) .then(payload => setFrobs(payload as Frob[])); ``` -The proxy is powered by the `http-proxy-middleware` package, and supports all of -its -[configuration options](https://github.com/chimurai/http-proxy-middleware#options). +The proxy is powered by the `http-proxy-middleware` package. See +[Proxying](proxying.md) for a full description of its configuration options. Internally at Spotify, the proxy option has been the overwhelmingly most popular choice for plugin makers. Since we have DNS based service discovery in place and diff --git a/docs/plugins/proxying.md b/docs/plugins/proxying.md index 4cf5b4c025..658df3dda1 100644 --- a/docs/plugins/proxying.md +++ b/docs/plugins/proxying.md @@ -3,4 +3,73 @@ id: proxying title: Proxying --- -## TODO +## Overview + +The Backstage backend comes packaged with a basic HTTP proxy, that can aid in +reaching backend service APIs from frontend plugin code. See +[Call Existing API](call-existing-api.md) for a description of when the proxy +can be the best choice for communicating with an API. + +## Getting Started + +The plugin is already added to a default Backstage project. + +In `packages/backend/src/index.ts`: + +```ts +const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + +const service = createServiceBuilder(module) + .loadConfig(configReader) + /** ... other routers ... */ + .addRouter('/proxy', await proxy(proxyEnv, '/proxy')); +``` + +## Configuration + +Configuration for the proxy plugin lives under a `proxy` root key of your +`app-config.yaml` file. + +Example: + +```yaml +# in app-config.yaml +proxy: + '/simple-example': http://simple.example.com:8080 + '/larger-example/v1': + target: http://larger.example.com:8080/svc.v1 + headers: + Authorization: + $secret: + env: EXAMPLE_AUTH_HEADER +``` + +Each key under the proxy configuration entry is a route to match, below the +prefix that the proxy plugin is mounted on. It must start with a slash. For +example, if the backend mounts the proxy plugin as `/proxy`, the above +configuration will lead to the proxy acting on backend requests to +`/proxy/simple-example/...` and `/proxy/larger-example/v1/...`. + +The value inside each route is either a simple URL string, or an object on the +format accepted by +[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). + +If the value is a string, it is assumed to correspond to: + +```yaml +target: +changeOrigin: true +pathRewrite: + '^/': '/' +``` + +When the target is an object, it is given verbatim to `http-proxy-middleware` +except with the following caveats for convenience: + +- If `changeOrigin` is not specified, it is set to `true`. This is the most + commonly useful value. +- If `pathRewrite` is not specified, it is set to a single rewrite that removes + the entire prefix and route. In the above example, a rewrite of + `'^/proxy/larger-example/v1/': '/'` is added. That means that a request to + `/proxy/larger-example/v1/some/path` will be translated to a request to + `http://larger.example.com:8080/svc.v1/some/path`. diff --git a/plugins/proxy-backend/README.md b/plugins/proxy-backend/README.md index baa2709820..f67449dda3 100644 --- a/plugins/proxy-backend/README.md +++ b/plugins/proxy-backend/README.md @@ -1,6 +1,7 @@ # Proxy backend plugin -This is the backend plugin that enables proxy definitions to be declared in and read from app-config.yaml. +This is the backend plugin that enables proxy definitions to be declared in, +and read from, `app-config.yaml`. Relies on the `http-proxy-middleware` package. @@ -10,69 +11,22 @@ This backend plugin can be started in a standalone mode from directly in this pa with `yarn start`. However, it will have limited functionality and that process is most convenient when developing the plugin itself. -To run it within the backend do: - -1. Register the router in `packages/backend/src/index.ts`: - -```ts -const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); - -const service = createServiceBuilder(module) - .loadConfig(configReader) - /** ... other routers ... */ - .addRouter('/proxy', await proxy(proxyEnv, '/proxy')); -``` - -2. Start the backend +The proxy is already installed in the Backstage backend per default, so you can also +start up the full example backend to experiment with the proxy. ```bash yarn workspace example-backend start ``` -This will launch the full example backend. - ## Configuration -Example: - -```yaml -# in app-config.yaml -proxy: - '/simple-example': http://simple.example.com:8080 - '/larger-example/v1': - target: http://larger.example.com:8080/svc.v1 - headers: - Authorization: - $secret: - env: EXAMPLE_AUTH_HEADER -``` - -Each key under the proxy configuration entry is a route to match, below the prefix that the proxy -plugin is mounted on. For example, if the backend mounts the proxy plugin as `/proxy`, -the above configuration will lead to the proxy acting on backend requests to -`/proxy/simple-example/...` and `/proxy/larger-example/v1/...`. - -The value inside each route is either a simple URL string, or an object on the format accepted by -[http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware). - -If the value is a string, it is assumed to correspond to: - -```yaml -target: -changeOrigin: true -pathRewrite: - '^/': '/' -``` - -When the target is an object, it is given verbatim to `http-proxy-middleware` except with the -following caveats for convenience: - -- If `changeOrigin` is not specified, it is set to `true`. This is the most commonly useful value. -- If `pathRewrite` is not specified, it is set to a single rewrite that removes the entire route. In the - above example, a rewrite of `'^/proxy/larger-example/v1/': '/'` is added. That means that a request to - `/proxy/larger-example/v1/some/path` will be translated to a request to - `http://larger.example.com:8080/svc.v1/some/path`. +See [the proxy docs](https://backstage.io/docs/plugins/proxying). ## Links +- [Call Existing API](https://backstage.io/docs/plugins/call-existing-api) helps the + decision process of what method of communication to use from a frontend plugin to + your API +- [The proxy plugin documentation](https://backstage.io/docs/plugins/proxying) describes + configuration options and more - [http-proxy-middleware](https://www.npmjs.com/package/http-proxy-middleware) From 86aff7a6cfd87bd8c8ef5b46d7fac57007a13fc0 Mon Sep 17 00:00:00 2001 From: ebarrios Date: Fri, 28 Aug 2020 14:39:36 +0200 Subject: [PATCH 023/114] Added github as CI/CD in catalog plugin --- plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx | 2 +- plugins/github-actions/src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx index 01de6dc9a2..d39617da25 100644 --- a/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx +++ b/plugins/catalog/src/components/EntityPageCi/EntityPageCi.tsx @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { Content } from '@backstage/core'; -import { WidgetList as GithubActionsListWidget } from '@backstage/plugin-github-actions'; +import { RecentWorkflowRunsCard as GithubActionsListWidget } from '@backstage/plugin-github-actions'; import { Grid } from '@material-ui/core'; import React, { FC } from 'react'; diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 4a69c363cd..4e383fdd3d 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -16,4 +16,4 @@ export { plugin } from './plugin'; export * from './api'; -export { Widget } from './components/Widget'; +export { Widget, RecentWorkflowRunsCard } from './components/Widget'; From 63339d2692b550035fa26b7e680a03d66987acc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 28 Aug 2020 14:50:41 +0200 Subject: [PATCH 024/114] chore(register-component): just a few little tweaks --- .../RegisterComponentForm.test.tsx | 6 +++--- .../RegisterComponentForm/RegisterComponentForm.tsx | 10 +++++----- plugins/register-component/src/util/validate.ts | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx index e1226bbf32..bb7faabc9d 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.test.tsx @@ -14,10 +14,10 @@ * limitations under the License. */ +import { cleanup, fireEvent, render } from '@testing-library/react'; import React from 'react'; -import { render, fireEvent, cleanup } from '@testing-library/react'; -import RegisterComponentForm, { Props } from './RegisterComponentForm'; import { act } from 'react-dom/test-utils'; +import RegisterComponentForm, { Props } from './RegisterComponentForm'; const setup = (props?: Partial) => { return { @@ -37,7 +37,7 @@ describe('RegisterComponentForm', () => { const { rendered } = setup(); expect( await rendered.findByText( - 'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config.', + 'Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component.', ), ).toBeInTheDocument(); diff --git a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx index de3c54610d..1b84b79234 100644 --- a/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx +++ b/plugins/register-component/src/components/RegisterComponentForm/RegisterComponentForm.tsx @@ -14,17 +14,17 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import { BackstageTheme } from '@backstage/theme'; import { Button, FormControl, FormHelperText, - TextField, LinearProgress, + TextField, } from '@material-ui/core'; -import { useForm } from 'react-hook-form'; import { makeStyles } from '@material-ui/core/styles'; -import { BackstageTheme } from '@backstage/theme'; +import React, { FC } from 'react'; +import { useForm } from 'react-hook-form'; import { ComponentIdValidators } from '../../util/validate'; const useStyles = makeStyles(theme => ({ @@ -71,7 +71,7 @@ const RegisterComponentForm: FC = ({ onSubmit, submitting }) => { name="componentLocation" required margin="normal" - helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component. For private repo provide authentication information via config." + helperText="Enter the full path to the component.yaml file in GitHub, GitLab, Bitbucket or Azure to start tracking your component." inputRef={register({ required: true, validate: ComponentIdValidators, diff --git a/plugins/register-component/src/util/validate.ts b/plugins/register-component/src/util/validate.ts index 8552872015..9afb5ebd97 100644 --- a/plugins/register-component/src/util/validate.ts +++ b/plugins/register-component/src/util/validate.ts @@ -19,6 +19,6 @@ export const ComponentIdValidators = { (typeof value === 'string' && value.match(/^https:\/\//) !== null) || 'Must start with https://.', yamlValidator: (value: any) => - (typeof value === 'string' && value.match(/.yaml/) !== null) || + (typeof value === 'string' && value.match(/\.yaml/) !== null) || "Must contain '.yaml'.", }; From 92d6f9bacffe019bc13024960c0f6ea846d490a0 Mon Sep 17 00:00:00 2001 From: Ashok Manji <1902568+ashokm@users.noreply.github.com> Date: Fri, 28 Aug 2020 15:31:29 +0200 Subject: [PATCH 025/114] Fix typo on Backstage Helm Chart README --- install/kubernetes/backstage/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/kubernetes/backstage/README.md b/install/kubernetes/backstage/README.md index e8d809c228..fc3d20733b 100644 --- a/install/kubernetes/backstage/README.md +++ b/install/kubernetes/backstage/README.md @@ -23,7 +23,7 @@ | app.resources | Kubernetes Pod resource requests/limits | `{}` | | app.nodeSelector | Node selectors for scheduling app/frontend pods | `{}` | | app.tolerations | Tolerations for scheduling app/frontend pods | `{}` | -| app.affinity | Affinity setttings for scheduling app/frontend pods | `{}` | +| app.affinity | Affinity settings for scheduling app/frontend pods | `{}` | ## Backend Values @@ -48,4 +48,4 @@ | backend.resources | Kubernetes Pod resource requests/limits | `{}` | | backend.nodeSelector | Node selectors for scheduling backend pods | `{}` | | backend.tolerations | Tolerations for scheduling backend pods | `{}` | -| backend.affinity | Affinity setttings for scheduling backend pods | `{}` | +| backend.affinity | Affinity settings for scheduling backend pods | `{}` | From 314eeb31d77897603791c2331a4719cac8bd4a38 Mon Sep 17 00:00:00 2001 From: Jesse Alan Johnson Date: Fri, 28 Aug 2020 11:57:04 -0400 Subject: [PATCH 026/114] documentation update (#2147) --- docs/getting-started/index.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index bfd3c70913..abe7776371 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -3,9 +3,22 @@ id: index title: Running Backstage Locally --- +First make sure you are using NodeJS with an Active LTS Release, currently v12. +This is made easy with a version manager such as nvm which allows for version switching. + +```bash +# Checking your version +node --version +> v14.7.0 + +# Adding a second node version +nvm install 12 +> Downloading and installing node v12.18.3... +> Now using node v12.18.3 (npm v6.14.6) +``` + To get up and running with a local Backstage to evaluate it, let's clone it off -of GitHub and run an initial build. First make sure that you have at least node -version 12 installed locally. +of GitHub and run an initial build. ```bash # Start from your local development folder From 57c3e2ad5baf30f6b2bc732ad79492e997c7af9f Mon Sep 17 00:00:00 2001 From: Rod Machen Date: Fri, 28 Aug 2020 14:07:25 -0500 Subject: [PATCH 027/114] update software template docs --- .../software-templates/adding-templates.md | 6 +++--- .../extending/create-your-own-publisher.md | 6 +++--- .../extending/create-your-own-templater.md | 12 +++++------ .../software-templates/extending/index.md | 20 +++++++++---------- docs/features/software-templates/index.md | 12 +++++------ .../src/scaffolder/stages/templater/types.ts | 2 +- .../src/techdocs/stages/generate/types.ts | 2 +- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 67e9c23605..67467bd98e 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -58,7 +58,7 @@ Currently the catalog supports loading definitions from GitHub + Local Files. To load from other places, not only will there need to be another preparer, but the support to load the location will also need to be added to the Catalog. -For loading from a file the following command should work when the backend is +For loading from a file, the following command should work when the backend is running: ```sh @@ -69,7 +69,7 @@ curl \ --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" ``` -If loading from a git location, you can run the following +If loading from a Git location, you can run the following ```sh curl \ @@ -83,7 +83,7 @@ This should then have added the catalog, and also should now be listed under the create page at http://localhost:3000/create. Alternatively, if you want to get setup with some mock templates that are -already provided for you, you can run the following to load those templates: +already provided, run the following to load those templates: ``` yarn lerna run mock-data diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md index 51869dc3ad..391f6f0658 100644 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ b/docs/features/software-templates/extending/create-your-own-publisher.md @@ -7,8 +7,8 @@ Publishers are responsible for pushing and storing the templated skeleton after the values have been templated by the `Templater`. See [Create your own templater](./create-your-own-templater.md) for more info. -They recieve a directory or location where the templater has sucessfully run on, -and is now ready to store somewhere. They also get given some other options +They receive a directory or location where the templater has sucessfully run +and is now ready to store somewhere. They also are given some other options which are sent from the frontend, such as the `storePath` which is a string of where the frontend thinks we should save this templated folder. @@ -17,7 +17,7 @@ Currently we provide the following `publishers`: - `github` This publisher is passed through to the `createRouter` function of the -`@spotify/plugin-scaffolder-backend`. Currently only one publisher is supported, +`@spotify/plugin-scaffolder-backend`. Currently, only one publisher is supported, but PR's are always welcome. An full example backend can be found diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md index a39f87924b..9dee66af06 100644 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ b/docs/features/software-templates/extending/create-your-own-templater.md @@ -8,14 +8,14 @@ returned by the preparers, and then executing the templating command on top of the file and returning the completed template path. This may or may not be the same directory as the input directory. -They also recieve additional values from the frontend, which can be used to +They also receive additional values from the frontend, which can be used to interpolate into the skeleton files. Currently we provide the following templaters: - `cookiecutter` -This templater is added the `TemplaterBuilder` and then passed into the +This templater is added to the `TemplaterBuilder` and then passed into the `createRouter` function of the `@spotify/plugin-scaffolder-backend` An full example backend can be found @@ -48,7 +48,7 @@ This `TemplaterKey` is used to select the correct templater from the `spec.templater` in the [Template Entity](../../software-catalog/descriptor-format.md#kind-template). -If you wish to add a new templater you'll need to register it with the +If you wish to add a new templater, you'll need to register it with the `TemplaterBuilder`. ### Creating your own Templater to add to the `TemplaterBuilder` @@ -83,10 +83,10 @@ follows: - `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to be able to run docker containers. -_note_ currently the templaters that we provide are basically docker action +_note_ Currently the templaters that we provide are basically Docker action containers that are run on top of the skeleton folder. This keeps dependencies to a minimal for running backstage scaffolder, but you don't /have/ to use -docker. You could create your own templater that spins up an EC2 instance and +Docker. You could create your own templater that spins up an EC2 instance and downloads the folder and does everything using an AMI if you want. It's entirely up to you! @@ -138,7 +138,7 @@ spec: description: Description of the component ``` -You see that the `spec.templater` is set as `handlebars`, you'll need to +You see that the `spec.templater` is set as `handlebars`, so you'll need to register this with the `TemplaterBuilder` like so: ```ts diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md index 21d8befda1..02d27a5725 100644 --- a/docs/features/software-templates/extending/index.md +++ b/docs/features/software-templates/extending/index.md @@ -5,11 +5,11 @@ title: Extending the Scaffolder Welcome. Take a seat. You're at the Scaffolder Documentation. -So - You wanna create stuff inside your company from some prebaked templates? +So, you want to create stuff inside your company from some prebaked templates? You're at the right place. -This guide is gonna take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through whats going on in the backend to be +This guide is going to take you through how the Scaffolder in Backstage works. +We'll dive into some jargon and run through what's going on in the backend to be able to create these templates. There's also more guides that you might find useful at the bottom of this document. At it's core, theres 3 simple stages. @@ -25,9 +25,9 @@ scaffolder that you will need to know: 3. Publish Each of these steps can be configured for your own use case, but we provide some -sensible defaults too. +sensible defaults, too. -Lets dive a little deeper into these phases. +Let's dive a little deeper into these phases. ### Glossary and Jargon @@ -38,8 +38,8 @@ the router to pick the correct `Preparer` to run for the `Template` entity. **Templater** - The templater is responsible for actually running the chosen templater on top of the previously returned temporary directory from the -**Preprarer**. We advise making these docker containers as it can keep all -dependencies, for example Cookiecutter, self contained and not a dependency on +**Preprarer**. We advise making these Docker containers as it can keep all +dependencies--for example Cookiecutter--self contained and not a dependency on the host machine. **Publisher** - The publisher is responsible for taking the finished directory, @@ -50,11 +50,11 @@ passed through to the scaffolder backend. ### How it works -The main of the heavy lifting is done in the +Most of the heavy lifting is done in the [router.ts](https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) file in the `scaffolder-backend` plugin. -There are 2 routes defined in the router. `POST /v1/jobs` and +There are two routes defined in the router: `POST /v1/jobs` and `GET /v1/job/:jobId` To create a scaffolding job, a JSON object containing the @@ -78,7 +78,7 @@ additional templating values must be posted as the post body. The values should represent something that is valid with the `schema` part of the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) -Once that has been posted, a job will be setup with different stages. And the +Once that has been posted, a job will be setup with different stages, and the job processor will complete each stage before moving onto the next stage, whilst collecting logs and mutating the running job. diff --git a/docs/features/software-templates/index.md b/docs/features/software-templates/index.md index 503fbb7a3d..1c67676b3e 100644 --- a/docs/features/software-templates/index.md +++ b/docs/features/software-templates/index.md @@ -4,8 +4,8 @@ title: Software Templates --- The Software Templates part of Backstage is a tool that can help you create -Components inside Backstage. It by default has the ability to load skeletons of -code, template in some variables and then publish the template to some location +Components inside Backstage. By default, it has the ability to load skeletons of +code, template in some variables, and then publish the template to some location like GitHub.