From e9773a0bdb801dd58663448688e2a875c8b4aae0 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 01:11:20 +0700 Subject: [PATCH 01/41] catalog(github): add github v3 reader processor --- .../src/ingestion/LocationReaders.ts | 2 + .../GithubV3ReaderProcessor.test.ts | 21 ++++ .../processors/GithubV3ReaderProcessor.ts | 119 ++++++++++++++++++ yarn.lock | 7 -- 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0fe87440fb..7d11659bf4 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GithubV3ReaderProcessor } from './processors/GithubV3ReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -57,6 +58,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GithubV3ReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts new file mode 100644 index 0000000000..1b39da2753 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor'; + +describe('GithubV3ReaderProcessor', () => { + // it('should build raw url') +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts new file mode 100644 index 0000000000..2316c36629 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +const privateToken = process.env.GITHUB_PRIVATE_TOKEN; + +export class GithubV3ReaderProcessor implements LocationProcessor { + getRequestOptions(): RequestInit { + const requestOptions: RequestInit = { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }; + + if (privateToken) { + requestOptions.headers.Authorization = `token ${privateToken}`; + } + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/v3') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const buffer = await response.buffer(); + + emit(result.data(location, buffer)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/yarn.lock b/yarn.lock index 8f5d9e8cec..0641f5ed77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3960,13 +3960,6 @@ "@types/node" "*" rollup "^0.63.4" -"@types/sanitize-html@^1.23.3": - version "1.23.3" - resolved "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.23.3.tgz#26527783aba3bf195ad8a3c3e51bd3713526fc0d" - integrity sha512-Isg8N0ifKdDq6/kaNlIcWfapDXxxquMSk2XC5THsOICRyOIhQGds95XH75/PL/g9mExi4bL8otIqJM/Wo96WxA== - dependencies: - htmlparser2 "^4.1.0" - "@types/serve-static@*": version "1.13.3" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1" From c295d8ce0687ba4dbac0853002dc7de2cb787685 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 12:14:28 +0700 Subject: [PATCH 02/41] catalog(github): merge github v3 to github --- .../src/ingestion/LocationReaders.ts | 1 - .../processors/GithubReaderProcessor.test.ts | 65 ++++++++++ .../processors/GithubReaderProcessor.ts | 44 +++++-- .../GithubV3ReaderProcessor.test.ts | 21 ---- .../processors/GithubV3ReaderProcessor.ts | 119 ------------------ 5 files changed, 100 insertions(+), 150 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts delete mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 7d11659bf4..bbdc7d8933 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -58,7 +58,6 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), - new GithubV3ReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts new file mode 100644 index 0000000000..5d64caf5b9 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubReaderProcessor } from './GithubReaderProcessor'; + +describe('GithubReaderProcessor', () => { + it('should build raw api', () => { + const processor = new GithubReaderProcessor(); + + const tests = [ + { + target: 'https://github.com/a/b/blob/master/path/to/c.yaml', + url: new URL( + 'https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master', + ), + err: undefined, + }, + { + target: 'https://api.com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL', + }, + { + target: 'com/a/b/blob/master/path/to/c.yaml', + url: null, + err: + 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', + }, + { + target: + 'https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/playback-order-component.yaml', + url: new URL( + 'https://api.github.com/repos/spotify/backstage/contents/packages/catalog-model/examples/playback-order-component.yaml?ref=master', + ), + err: undefined, + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target)).toThrowError(test.err); + } else { + expect(processor.buildRawUrl(test.target)).toEqual(test.url); + } + } + }); + + it('should return request options', () => { + // todo + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index b83c9a16f3..11610234bd 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,11 +15,28 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; +const privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + export class GithubReaderProcessor implements LocationProcessor { + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + if (privateToken) { + headers.Authorization = `token ${privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + async readLocation( location: LocationSpec, optional: boolean, @@ -34,7 +51,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString()); + const response = await fetch(url.toString(), this.getRequestOptions()); if (response.ok) { const data = await response.buffer(); @@ -58,9 +75,9 @@ export class GithubReaderProcessor implements LocationProcessor { } // Converts - // from: https://github.com/a/b/blob/master/c.yaml - // to: https://raw.githubusercontent.com/a/b/master/c.yaml - private buildRawUrl(target: string): URL { + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -69,6 +86,7 @@ export class GithubReaderProcessor implements LocationProcessor { userOrOrg, repoName, blobKeyword, + ref, ...restOfPath ] = url.pathname.split('/'); @@ -80,13 +98,21 @@ export class GithubReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL'); + throw new Error('Wrong GitHub URL or Invalid file path'); } - // Removing the "blob" part - url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); - url.hostname = 'raw.githubusercontent.com'; + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; url.protocol = 'https'; + url.search = `ref=${ref}`; return url; } catch (e) { diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts deleted file mode 100644 index 1b39da2753..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubV3ReaderProcessor } from './GithubV3ReaderProcessor'; - -describe('GithubV3ReaderProcessor', () => { - // it('should build raw url') -}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts deleted file mode 100644 index 2316c36629..0000000000 --- a/plugins/catalog-backend/src/ingestion/processors/GithubV3ReaderProcessor.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'node-fetch'; -import * as result from './results'; -import { LocationProcessor, LocationProcessorEmit } from './types'; - -const privateToken = process.env.GITHUB_PRIVATE_TOKEN; - -export class GithubV3ReaderProcessor implements LocationProcessor { - getRequestOptions(): RequestInit { - const requestOptions: RequestInit = { - headers: { - Accept: 'application/vnd.github.v3.raw', - }, - }; - - if (privateToken) { - requestOptions.headers.Authorization = `token ${privateToken}`; - } - return requestOptions; - } - - async readLocation( - location: LocationSpec, - optional: boolean, - emit: LocationProcessorEmit, - ): Promise { - if (location.type !== 'github/v3') { - return false; - } - - try { - const url = this.buildRawUrl(location.target); - - const response = await fetch(url.toString(), this.getRequestOptions()); - - if (response.ok) { - const buffer = await response.buffer(); - - emit(result.data(location, buffer)); - } else { - const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; - if (response.status === 404) { - if (!optional) { - emit(result.notFoundError(location, message)); - } - } else { - emit(result.generalError(location, message)); - } - } - } catch (e) { - const message = `Unable to read ${location.type} ${location.target}, ${e}`; - emit(result.generalError(location, message)); - } - - return true; - } - - // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - blobKeyword, - ref, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'github.com' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - blobKeyword !== 'blob' || - !restOfPath.join('/').match(/\.yaml$/) - ) { - throw new Error('Wrong GitHub URL'); - } - - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; - url.protocol = 'https'; - url.search = `ref=${ref}`; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } -} From af33400021dae4d0f699a13f3b436bbd7f2fee05 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 12:22:46 +0700 Subject: [PATCH 03/41] catalog(github): fix test --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 1 - .../src/ingestion/processors/GithubReaderProcessor.test.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index bbdc7d8933..0fe87440fb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,7 +26,6 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; -import { GithubV3ReaderProcessor } from './processors/GithubV3ReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index 5d64caf5b9..0b3057f97e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -32,7 +32,7 @@ describe('GithubReaderProcessor', () => { target: 'https://api.com/a/b/blob/master/path/to/c.yaml', url: null, err: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL', + 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong GitHub URL or Invalid file path', }, { target: 'com/a/b/blob/master/path/to/c.yaml', From a05a8f805d32e5da44190229c49d60cd5800119e Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 8 Jul 2020 09:25:27 +0200 Subject: [PATCH 04/41] feat(proxy): implement proxy backend plugin --- packages/backend/package.json | 1 + packages/backend/src/index.ts | 5 +- packages/backend/src/plugins/proxy.ts | 25 ++++++++++ plugins/proxy-backend/.eslintrc.js | 3 ++ plugins/proxy-backend/README.md | 37 ++++++++++++++ plugins/proxy-backend/package.json | 50 +++++++++++++++++++ plugins/proxy-backend/src/index.ts | 17 +++++++ plugins/proxy-backend/src/run.ts | 33 ++++++++++++ plugins/proxy-backend/src/service/router.ts | 42 ++++++++++++++++ .../src/service/standaloneServer.ts | 48 ++++++++++++++++++ plugins/proxy-backend/src/setupTests.ts | 19 +++++++ yarn.lock | 29 ++++++++--- 12 files changed, 300 insertions(+), 9 deletions(-) create mode 100644 packages/backend/src/plugins/proxy.ts create mode 100644 plugins/proxy-backend/.eslintrc.js create mode 100644 plugins/proxy-backend/README.md create mode 100644 plugins/proxy-backend/package.json create mode 100644 plugins/proxy-backend/src/index.ts create mode 100644 plugins/proxy-backend/src/run.ts create mode 100644 plugins/proxy-backend/src/service/router.ts create mode 100644 plugins/proxy-backend/src/service/standaloneServer.ts create mode 100644 plugins/proxy-backend/src/setupTests.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 64ca2c3113..54cf1e7e1a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,6 +25,7 @@ "@backstage/plugin-auth-backend": "^0.1.1-alpha.12", "@backstage/plugin-catalog-backend": "^0.1.1-alpha.12", "@backstage/plugin-identity-backend": "^0.1.1-alpha.12", + "@backstage/plugin-proxy-backend": "^0.1.1-alpha.12", "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.12", "@backstage/plugin-sentry-backend": "^0.1.1-alpha.12", "dockerode": "^3.2.0", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 3246c5f6f5..a50b1e38be 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -35,6 +35,7 @@ import catalog from './plugins/catalog'; import identity from './plugins/identity'; import scaffolder from './plugins/scaffolder'; import sentry from './plugins/sentry'; +import proxy from './plugins/proxy'; import { PluginEnvironment } from './types'; function makeCreateEnv(loadedConfigs: AppConfig[]) { @@ -63,6 +64,7 @@ async function main() { const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); const authEnv = useHotMemoize(module, () => createEnv('auth')); const identityEnv = useHotMemoize(module, () => createEnv('identity')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); const service = createServiceBuilder(module) .loadConfig(configReader) @@ -73,7 +75,8 @@ async function main() { await sentry(getRootLogger().child({ type: 'plugin', plugin: 'sentry' })), ) .addRouter('/auth', await auth(authEnv)) - .addRouter('/identity', await identity(identityEnv)); + .addRouter('/identity', await identity(identityEnv)) + .addRouter('/', await proxy(proxyEnv)); await service.start().catch(err => { console.log(err); diff --git a/packages/backend/src/plugins/proxy.ts b/packages/backend/src/plugins/proxy.ts new file mode 100644 index 0000000000..4964de130e --- /dev/null +++ b/packages/backend/src/plugins/proxy.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// @ts-ignore +import { createRouter } from '@backstage/plugin-proxy-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ logger, config }); +} diff --git a/plugins/proxy-backend/.eslintrc.js b/plugins/proxy-backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/proxy-backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/proxy-backend/README.md b/plugins/proxy-backend/README.md new file mode 100644 index 0000000000..315178dac6 --- /dev/null +++ b/plugins/proxy-backend/README.md @@ -0,0 +1,37 @@ +# Proxy backend plugin + +This is the backend plugin that enables proxy definitions to be declared in and read from app-config.yaml. + +Relies on the `http-proxy-middleware` package. + +## Getting Started + +This backend plugin can be started in a standalone mode from directly in this package +with `yarn start`. However, it will have limited functionality and that process is +most convenient when developing the plugin itself. + +To run it within the backend do: + +1. Register the router in `packages/backend/src/index.ts`: + +``` +const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); + +const service = createServiceBuilder(module) + .loadConfig(configReader) + /** several different routers */ + .addRouter('/', await proxy(proxyEnv)); +``` + +2. Start the backend + +```bash +yarn workspace example-backend start +``` + +This will launch the full example backend. + +## Links + +- (http-proxy-middleware)[https://www.npmjs.com/package/http-proxy-middleware] +- (The Backstage homepage)[https://backstage.io] diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json new file mode 100644 index 0000000000..88348b4779 --- /dev/null +++ b/plugins/proxy-backend/package.json @@ -0,0 +1,50 @@ +{ + "name": "@backstage/plugin-proxy-backend", + "version": "0.1.1-alpha.12", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.1.1-alpha.12", + "@backstage/config": "^0.1.1-alpha.12", + "@types/express": "^4.17.6", + "@types/http-proxy-middleware": "^0.19.3", + "express": "^4.17.1", + "express-promise-router": "^3.0.3", + "http-proxy-middleware": "^1.0.4", + "morgan": "^1.10.0", + "node-fetch": "^2.6.0", + "uuid": "^8.0.0", + "winston": "^3.2.1", + "yaml": "^1.9.2", + "yn": "^4.0.0", + "yup": "^0.29.1" + }, + "devDependencies": { + "@backstage/cli": "^0.1.1-alpha.12", + "@types/node-fetch": "^2.5.7", + "@types/supertest": "^2.0.8", + "@types/uuid": "^8.0.0", + "@types/yup": "^0.28.2", + "jest-fetch-mock": "^3.0.3", + "supertest": "^4.0.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/proxy-backend/src/index.ts b/plugins/proxy-backend/src/index.ts new file mode 100644 index 0000000000..7612c392a2 --- /dev/null +++ b/plugins/proxy-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; diff --git a/plugins/proxy-backend/src/run.ts b/plugins/proxy-backend/src/run.ts new file mode 100644 index 0000000000..b96989e4b8 --- /dev/null +++ b/plugins/proxy-backend/src/run.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getRootLogger } from '@backstage/backend-common'; +import yn from 'yn'; +import { startStandaloneServer } from './service/standaloneServer'; + +const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000; +const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); +const logger = getRootLogger(); + +startStandaloneServer({ port, enableCors, logger }).catch(err => { + logger.error(err); + process.exit(1); +}); + +process.on('SIGINT', () => { + logger.info('CTRL+C pressed; exiting.'); + process.exit(0); +}); diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts new file mode 100644 index 0000000000..0f5cd27502 --- /dev/null +++ b/plugins/proxy-backend/src/service/router.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { errorHandler } from '@backstage/backend-common'; +import { AppConfig } from '@backstage/config'; +import express from 'express'; +import Router from 'express-promise-router'; +import { createProxyMiddleware } from 'http-proxy-middleware'; +import { Logger } from 'winston'; + +export interface RouterOptions { + logger: Logger; + config: AppConfig; +} + +export async function createRouter( + options: RouterOptions, +): Promise { + const router = Router(); + router.use(express.json()); + + const proxyConfig = options.config.data.proxy ?? {}; + Object.entries(proxyConfig).forEach(([route, proxyRouteConfig]) => { + router.use(createProxyMiddleware(route, proxyRouteConfig)); + }); + + router.use(errorHandler()); + return router; +} diff --git a/plugins/proxy-backend/src/service/standaloneServer.ts b/plugins/proxy-backend/src/service/standaloneServer.ts new file mode 100644 index 0000000000..22b64aaaf5 --- /dev/null +++ b/plugins/proxy-backend/src/service/standaloneServer.ts @@ -0,0 +1,48 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceBuilder } from '@backstage/backend-common'; +import { Server } from 'http'; +import { Logger } from 'winston'; +import { createRouter } from './router'; + +export interface ServerOptions { + port: number; + enableCors: boolean; + logger: Logger; +} + +export async function startStandaloneServer( + options: ServerOptions, +): Promise { + const logger = options.logger.child({ service: 'catalog-backend' }); + + logger.debug('Creating application...'); + + logger.debug('Starting application server...'); + const router = await createRouter({ + logger, + }); + const service = createServiceBuilder(module) + .enableCors({ origin: 'http://localhost:3000' }) + .addRouter('/proxy', router); + return await service.start().catch(err => { + logger.error(err); + process.exit(1); + }); +} + +module.hot?.accept(); diff --git a/plugins/proxy-backend/src/setupTests.ts b/plugins/proxy-backend/src/setupTests.ts new file mode 100644 index 0000000000..f7b6ca962d --- /dev/null +++ b/plugins/proxy-backend/src/setupTests.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +require('jest-fetch-mock').enableMocks(); + +export {}; diff --git a/yarn.lock b/yarn.lock index 8f5d9e8cec..a26245c3b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3594,7 +3594,7 @@ resolved "https://registry.npmjs.org/@types/http-errors/-/http-errors-1.6.3.tgz#619a55768eab98299e8f76747339f3373f134e69" integrity sha512-4KCE/agIcoQ9bIfa4sBxbZdnORzRjIw8JNQPLfqoNv7wQl/8f8mRbW68Q8wBsQFoJkPUHGlQYZ9sqi5WpfGSEQ== -"@types/http-proxy-middleware@*": +"@types/http-proxy-middleware@*", "@types/http-proxy-middleware@^0.19.3": version "0.19.3" resolved "https://registry.npmjs.org/@types/http-proxy-middleware/-/http-proxy-middleware-0.19.3.tgz#b2eb96fbc0f9ac7250b5d9c4c53aade049497d03" integrity sha512-lnBTx6HCOUeIJMLbI/LaL5EmdKLhczJY5oeXZpX/cXE4rRqb3RmV7VcMpiEfYkmTjipv3h7IAyIINe4plEv7cA== @@ -3960,13 +3960,6 @@ "@types/node" "*" rollup "^0.63.4" -"@types/sanitize-html@^1.23.3": - version "1.23.3" - resolved "https://registry.npmjs.org/@types/sanitize-html/-/sanitize-html-1.23.3.tgz#26527783aba3bf195ad8a3c3e51bd3713526fc0d" - integrity sha512-Isg8N0ifKdDq6/kaNlIcWfapDXxxquMSk2XC5THsOICRyOIhQGds95XH75/PL/g9mExi4bL8otIqJM/Wo96WxA== - dependencies: - htmlparser2 "^4.1.0" - "@types/serve-static@*": version "1.13.3" resolved "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1" @@ -10118,6 +10111,17 @@ http-proxy-middleware@0.19.1: lodash "^4.17.11" micromatch "^3.1.10" +http-proxy-middleware@^1.0.4: + version "1.0.4" + resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-1.0.4.tgz#425ea177986a0cda34f9c81ec961c719adb6c2a9" + integrity sha512-8wiqujNWlsZNbeTSSWMLUl/u70xbJ5VYRwPR8RcAbvsNxzAZbgwLzRvT96btbm3fAitZUmo5i8LY6WKGyHDgvA== + dependencies: + "@types/http-proxy" "^1.17.4" + http-proxy "^1.18.1" + is-glob "^4.0.1" + lodash "^4.17.15" + micromatch "^4.0.2" + http-proxy@^1.17.0: version "1.18.0" resolved "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.0.tgz#dbe55f63e75a347db7f3d99974f2692a314a6a3a" @@ -10127,6 +10131,15 @@ http-proxy@^1.17.0: follow-redirects "^1.0.0" requires-port "^1.0.0" +http-proxy@^1.18.1: + version "1.18.1" + resolved "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" + integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== + dependencies: + eventemitter3 "^4.0.0" + follow-redirects "^1.0.0" + requires-port "^1.0.0" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" From 237502140ba7de938bfb8da58cfbc47ecdcd1c97 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 8 Jul 2020 09:25:56 +0200 Subject: [PATCH 05/41] feat(proxy): start using proxy for circleci --- app-config.yaml | 7 +++++++ packages/app/src/apis.ts | 2 +- plugins/circleci/package.json | 9 --------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 39527c20bd..b0b7666f24 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -10,5 +10,12 @@ backend: methods: [GET, POST, PUT, DELETE] credentials: true +proxy: + '/circleci/api': + target: 'https://circleci.com/api/v1.1' + changeOrigin: true + pathRewrite: + '^/circleci/api/': '/' + organization: name: Spotify diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index bc7170c93a..3f9765f5e2 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -72,7 +72,7 @@ export const apis = (config: ConfigApi) => { ); builder.add(storageApiRef, WebStorage.create({ errorApi })); - builder.add(circleCIApiRef, new CircleCIApi()); + builder.add(circleCIApiRef, new CircleCIApi(`${backendUrl}/circleci/api`)); builder.add(featureFlagsApiRef, new FeatureFlags()); builder.add(lighthouseApiRef, new LighthouseRestApi('http://localhost:3003')); diff --git a/plugins/circleci/package.json b/plugins/circleci/package.json index 89f6335dcc..a27cd6b81c 100644 --- a/plugins/circleci/package.json +++ b/plugins/circleci/package.json @@ -10,15 +10,6 @@ "main": "dist/index.esm.js", "types": "dist/index.d.ts" }, - "proxy": { - "/circleci/api": { - "target": "https://circleci.com/api/v1.1", - "changeOrigin": true, - "pathRewrite": { - "^/circleci/api/": "/" - } - } - }, "scripts": { "build": "backstage-cli plugin:build", "lint": "backstage-cli lint", From 14537751b3f22a2f6cdd5ff6d6fb78917dc4dd5b Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Wed, 8 Jul 2020 09:36:11 +0200 Subject: [PATCH 06/41] fix(proxy): private package for now --- plugins/proxy-backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json index 88348b4779..6eb8dab99d 100644 --- a/plugins/proxy-backend/package.json +++ b/plugins/proxy-backend/package.json @@ -1,10 +1,10 @@ { "name": "@backstage/plugin-proxy-backend", - "version": "0.1.1-alpha.12", + "version": "0.1.1-alpha.13", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", - "private": false, + "private": true, "publishConfig": { "access": "public", "main": "dist/index.cjs.js", From 48addc9259367cc5de2700bba57e17fe8bc7b322 Mon Sep 17 00:00:00 2001 From: danztran Date: Wed, 8 Jul 2020 14:57:03 +0700 Subject: [PATCH 07/41] catalog(github): add test get request options --- .../processors/GithubReaderProcessor.test.ts | 26 ++++++++++++++++++- .../processors/GithubReaderProcessor.ts | 9 ++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts index 0b3057f97e..b836a592ee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts @@ -60,6 +60,30 @@ describe('GithubReaderProcessor', () => { }); it('should return request options', () => { - // todo + const tests = [ + { + token: '0123456789', + expect: { + headers: { + Accept: 'application/vnd.github.v3.raw', + Authorization: 'token 0123456789', + }, + }, + }, + { + token: '', + expect: { + headers: { + Accept: 'application/vnd.github.v3.raw', + }, + }, + }, + ]; + + for (const test of tests) { + process.env.GITHUB_PRIVATE_TOKEN = test.token; + const processor = new GithubReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index 11610234bd..a0e089aaae 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -19,15 +19,16 @@ import fetch, { RequestInit, HeadersInit } from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; -const privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; - export class GithubReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + getRequestOptions(): RequestInit { const headers: HeadersInit = { Accept: 'application/vnd.github.v3.raw', }; - if (privateToken) { - headers.Authorization = `token ${privateToken}`; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; } const requestOptions: RequestInit = { From ed0bd8f6165c1ebb060e50201bff51df6845d8d5 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Thu, 9 Jul 2020 01:54:31 +0200 Subject: [PATCH 08/41] fix(proxy): types and versions --- plugins/proxy-backend/package.json | 13 ++++++++++--- plugins/proxy-backend/src/service/router.ts | 6 +++--- .../proxy-backend/src/service/standaloneServer.ts | 10 ++++++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json index 6eb8dab99d..521f190eba 100644 --- a/plugins/proxy-backend/package.json +++ b/plugins/proxy-backend/package.json @@ -20,8 +20,9 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.1.1-alpha.12", - "@backstage/config": "^0.1.1-alpha.12", + "@backstage/backend-common": "^0.1.1-alpha.13", + "@backstage/config": "^0.1.1-alpha.13", + "@backstage/config-loader": "^0.1.1-alpha.13", "@types/express": "^4.17.6", "@types/http-proxy-middleware": "^0.19.3", "express": "^4.17.1", @@ -36,7 +37,7 @@ "yup": "^0.29.1" }, "devDependencies": { - "@backstage/cli": "^0.1.1-alpha.12", + "@backstage/cli": "^0.1.1-alpha.13", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.8", "@types/uuid": "^8.0.0", @@ -44,6 +45,12 @@ "jest-fetch-mock": "^3.0.3", "supertest": "^4.0.2" }, + "workspaces": { + "nohoist": [ + "http-proxy-middleware", + "@types/http-proxy-middleware" + ] + }, "files": [ "dist" ] diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts index 0f5cd27502..a8fd70c10d 100644 --- a/plugins/proxy-backend/src/service/router.ts +++ b/plugins/proxy-backend/src/service/router.ts @@ -15,7 +15,7 @@ */ import { errorHandler } from '@backstage/backend-common'; -import { AppConfig } from '@backstage/config'; +import { Config } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; import { createProxyMiddleware } from 'http-proxy-middleware'; @@ -23,7 +23,7 @@ import { Logger } from 'winston'; export interface RouterOptions { logger: Logger; - config: AppConfig; + config: Config; } export async function createRouter( @@ -32,7 +32,7 @@ export async function createRouter( const router = Router(); router.use(express.json()); - const proxyConfig = options.config.data.proxy ?? {}; + const proxyConfig = options.config.get('proxy') ?? {}; Object.entries(proxyConfig).forEach(([route, proxyRouteConfig]) => { router.use(createProxyMiddleware(route, proxyRouteConfig)); }); diff --git a/plugins/proxy-backend/src/service/standaloneServer.ts b/plugins/proxy-backend/src/service/standaloneServer.ts index 22b64aaaf5..84c803eeac 100644 --- a/plugins/proxy-backend/src/service/standaloneServer.ts +++ b/plugins/proxy-backend/src/service/standaloneServer.ts @@ -18,6 +18,8 @@ import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; import { createRouter } from './router'; +import { ConfigReader } from '@backstage/config'; +import { loadConfig } from '@backstage/config-loader'; export interface ServerOptions { port: number; @@ -28,17 +30,21 @@ export interface ServerOptions { export async function startStandaloneServer( options: ServerOptions, ): Promise { - const logger = options.logger.child({ service: 'catalog-backend' }); + const logger = options.logger.child({ service: 'proxy-backend' }); logger.debug('Creating application...'); - logger.debug('Starting application server...'); + const config = ConfigReader.fromConfigs(await loadConfig()); const router = await createRouter({ + config, logger, }); const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) .addRouter('/proxy', router); + + logger.debug('Starting application server...'); + return await service.start().catch(err => { logger.error(err); process.exit(1); From d4d9c33ca32d9e0f60664c8e2d43fa1958e5dcf4 Mon Sep 17 00:00:00 2001 From: danztran Date: Thu, 9 Jul 2020 23:59:56 +0700 Subject: [PATCH 09/41] catalog(github): split to github api reader processor --- ...st.ts => GithubApiReaderProcessor.test.ts} | 8 +- .../processors/GithubApiReaderProcessor.ts | 121 ++++++++++++++++++ .../processors/GithubReaderProcessor.ts | 45 ++----- 3 files changed, 134 insertions(+), 40 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/{GithubReaderProcessor.test.ts => GithubApiReaderProcessor.test.ts} (91%) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts rename to plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts index b836a592ee..4dd3fd359f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.test.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { GithubReaderProcessor } from './GithubReaderProcessor'; +import { GithubApiReaderProcessor } from './GithubApiReaderProcessor'; -describe('GithubReaderProcessor', () => { +describe('GithubApiReaderProcessor', () => { it('should build raw api', () => { - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); const tests = [ { @@ -82,7 +82,7 @@ describe('GithubReaderProcessor', () => { for (const test of tests) { process.env.GITHUB_PRIVATE_TOKEN = test.token; - const processor = new GithubReaderProcessor(); + const processor = new GithubApiReaderProcessor(); expect(processor.getRequestOptions()).toEqual(test.expect); } }); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts new file mode 100644 index 0000000000..ff61d004ca --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GithubApiReaderProcessor.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class GithubApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { + Accept: 'application/vnd.github.v3.raw', + }; + + if (this.privateToken !== '') { + headers.Authorization = `token ${this.privateToken}`; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'github/api') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString(), this.getRequestOptions()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://github.com/a/b/blob/master/path/to/c.yaml + // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master + buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ref, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + url.hostname !== 'github.com' || + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong GitHub URL or Invalid file path'); + } + + // transform to api + url.pathname = [ + empty, + 'repos', + userOrOrg, + repoName, + 'contents', + ...restOfPath, + ].join('/'); + url.hostname = 'api.github.com'; + url.protocol = 'https'; + url.search = `ref=${ref}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts index a0e089aaae..b83c9a16f3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts @@ -15,29 +15,11 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import fetch from 'node-fetch'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; export class GithubReaderProcessor implements LocationProcessor { - private privateToken: string = process.env.GITHUB_PRIVATE_TOKEN || ''; - - getRequestOptions(): RequestInit { - const headers: HeadersInit = { - Accept: 'application/vnd.github.v3.raw', - }; - - if (this.privateToken !== '') { - headers.Authorization = `token ${this.privateToken}`; - } - - const requestOptions: RequestInit = { - headers, - }; - - return requestOptions; - } - async readLocation( location: LocationSpec, optional: boolean, @@ -52,7 +34,7 @@ export class GithubReaderProcessor implements LocationProcessor { // TODO(freben): Should "hard" errors thrown by this line be treated as // notFound instead of fatal? - const response = await fetch(url.toString(), this.getRequestOptions()); + const response = await fetch(url.toString()); if (response.ok) { const data = await response.buffer(); @@ -76,9 +58,9 @@ export class GithubReaderProcessor implements LocationProcessor { } // Converts - // from: https://github.com/a/b/blob/master/path/to/c.yaml - // to: https://api.github.com/repos/a/b/contents/path/to/c.yaml?ref=master - buildRawUrl(target: string): URL { + // from: https://github.com/a/b/blob/master/c.yaml + // to: https://raw.githubusercontent.com/a/b/master/c.yaml + private buildRawUrl(target: string): URL { try { const url = new URL(target); @@ -87,7 +69,6 @@ export class GithubReaderProcessor implements LocationProcessor { userOrOrg, repoName, blobKeyword, - ref, ...restOfPath ] = url.pathname.split('/'); @@ -99,21 +80,13 @@ export class GithubReaderProcessor implements LocationProcessor { blobKeyword !== 'blob' || !restOfPath.join('/').match(/\.yaml$/) ) { - throw new Error('Wrong GitHub URL or Invalid file path'); + throw new Error('Wrong GitHub URL'); } - // transform to api - url.pathname = [ - empty, - 'repos', - userOrOrg, - repoName, - 'contents', - ...restOfPath, - ].join('/'); - url.hostname = 'api.github.com'; + // Removing the "blob" part + url.pathname = [empty, userOrOrg, repoName, ...restOfPath].join('/'); + url.hostname = 'raw.githubusercontent.com'; url.protocol = 'https'; - url.search = `ref=${ref}`; return url; } catch (e) { From 77b5160254397d379b649c97f2f6c68626e9f083 Mon Sep 17 00:00:00 2001 From: danztran Date: Fri, 10 Jul 2020 00:05:44 +0700 Subject: [PATCH 10/41] catalog(github): add github api to default processors --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 0fe87440fb..8ec575d6ec 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -57,6 +58,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GithubApiReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), From 4a990863fdea57163a5b3077fd504b5813ceead0 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Fri, 10 Jul 2020 00:56:40 +0200 Subject: [PATCH 11/41] fix(proxy): add test --- .../proxy-backend/src/service/router.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 plugins/proxy-backend/src/service/router.test.ts diff --git a/plugins/proxy-backend/src/service/router.test.ts b/plugins/proxy-backend/src/service/router.test.ts new file mode 100644 index 0000000000..699ed9b99d --- /dev/null +++ b/plugins/proxy-backend/src/service/router.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createRouter } from './router'; +import winston from 'winston'; +import { ConfigReader } from '@backstage/config'; +import { loadConfig } from '@backstage/config-loader'; + +describe('createRouter', () => { + it('works', async () => { + const logger = winston.createLogger(); + const config = ConfigReader.fromConfigs(await loadConfig()); + const router = await createRouter({ + config, + logger, + }); + expect(router).toBeDefined(); + }); +}); From ab69c9e47bf38fbc8a41ae046e7f53415a4dc352 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Fri, 10 Jul 2020 09:29:26 +0200 Subject: [PATCH 12/41] feat(catalog): allow using different branches --- .../register-component/src/util/validate.test.ts | 14 -------------- plugins/register-component/src/util/validate.ts | 3 --- 2 files changed, 17 deletions(-) diff --git a/plugins/register-component/src/util/validate.test.ts b/plugins/register-component/src/util/validate.test.ts index 43e05a4671..d062655bd2 100644 --- a/plugins/register-component/src/util/validate.test.ts +++ b/plugins/register-component/src/util/validate.test.ts @@ -30,20 +30,6 @@ describe('ComponentIdValidators', () => { expect(ComponentIdValidators.httpsValidator(arg)).toBe(expected); }); }); - describe('masterValidator', () => { - const errorMessage = 'Must reference a file on the master branch.'; - test.each([ - [true, '/blob/master/'], - [true, 'http://example.com/blob/master/'], - [errorMessage, 'blob/master/'], - [errorMessage, '/blob/master'], - [errorMessage, '/master/'], - [errorMessage, ''], - [errorMessage, undefined], - ])('should return %p for %s', (expected: string | boolean, arg: any) => { - expect(ComponentIdValidators.masterValidator(arg)).toBe(expected); - }); - }); describe('yamlValidator', () => { const errorMessage = "Must end with '.yaml'."; test.each([ diff --git a/plugins/register-component/src/util/validate.ts b/plugins/register-component/src/util/validate.ts index 175d9378df..78d20995f5 100644 --- a/plugins/register-component/src/util/validate.ts +++ b/plugins/register-component/src/util/validate.ts @@ -18,9 +18,6 @@ export const ComponentIdValidators = { httpsValidator: (value: any) => (typeof value === 'string' && value.match(/^https:\/\//) !== null) || 'Must start with https://.', - masterValidator: (value: any) => - (typeof value === 'string' && value.match(/\/blob\/master\//) !== null) || - 'Must reference a file on the master branch.', yamlValidator: (value: any) => (typeof value === 'string' && value.match(/.yaml$/) !== null) || "Must end with '.yaml'.", From 09dfdbad5c7ae3bbbb7a3643132623a0ada49285 Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Fri, 10 Jul 2020 09:24:32 +0200 Subject: [PATCH 13/41] feat: github actions api type with a mock implementation --- packages/app/src/apis.ts | 5 +++ plugins/github-actions/package.json | 1 + .../src/api/GithubActionsApi.ts | 28 ++++++++++++ .../MockGithubActionsClient.ts} | 12 ++--- plugins/github-actions/src/api/index.test.ts | 44 +++++++++++++++++++ .../src/{apis/builds => api}/index.ts | 3 +- .../src/{apis/builds => api}/types.ts | 0 .../BuildDetailsPage/BuildDetailsPage.tsx | 8 ++-- .../BuildInfoCard/BuildInfoCard.tsx | 8 ++-- .../BuildListPage/BuildListPage.tsx | 8 ++-- .../BuildStatusIndicator.tsx | 2 +- plugins/github-actions/src/index.ts | 1 + 12 files changed, 98 insertions(+), 22 deletions(-) create mode 100644 plugins/github-actions/src/api/GithubActionsApi.ts rename plugins/github-actions/src/{apis/builds/BuildsClient.ts => api/MockGithubActionsClient.ts} (77%) create mode 100644 plugins/github-actions/src/api/index.test.ts rename plugins/github-actions/src/{apis/builds => api}/index.ts (88%) rename plugins/github-actions/src/{apis/builds => api}/types.ts (100%) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index abc490b658..df17886216 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -58,6 +58,10 @@ import { import { scaffolderApiRef, ScaffolderApi } from '@backstage/plugin-scaffolder'; import { rollbarApiRef, RollbarClient } from '@backstage/plugin-rollbar'; +import { + MockGithubActionsClient, + githubActionsApiRef, +} from '@backstage/plugin-github-actions'; export const apis = (config: ConfigApi) => { // eslint-disable-next-line no-console @@ -75,6 +79,7 @@ export const apis = (config: ConfigApi) => { builder.add(storageApiRef, WebStorage.create({ errorApi })); builder.add(circleCIApiRef, new CircleCIApi()); + builder.add(githubActionsApiRef, new MockGithubActionsClient()); builder.add(featureFlagsApiRef, new FeatureFlags()); builder.add(lighthouseApiRef, new LighthouseRestApi('http://localhost:3003')); diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 456bd826c0..a4528c42a5 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@backstage/core": "^0.1.1-alpha.13", + "@backstage/core-api": "^0.1.1-alpha.13", "@backstage/theme": "^0.1.1-alpha.13", "@material-ui/core": "^4.9.1", "@material-ui/icons": "^4.9.1", diff --git a/plugins/github-actions/src/api/GithubActionsApi.ts b/plugins/github-actions/src/api/GithubActionsApi.ts new file mode 100644 index 0000000000..7c52bbcb70 --- /dev/null +++ b/plugins/github-actions/src/api/GithubActionsApi.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createApiRef } from '@backstage/core'; +import { Build, BuildDetails } from './types'; + +export const githubActionsApiRef = createApiRef({ + id: 'plugin.githubactions.service', + description: 'Used by the Github Actions plugin to make requests', +}); + +export type GithubActionsApi = { + listBuilds: (_entityUri: string) => Promise; + getBuild: (_buildUri: string) => Promise; +}; diff --git a/plugins/github-actions/src/apis/builds/BuildsClient.ts b/plugins/github-actions/src/api/MockGithubActionsClient.ts similarity index 77% rename from plugins/github-actions/src/apis/builds/BuildsClient.ts rename to plugins/github-actions/src/api/MockGithubActionsClient.ts index 6fbc6a6c53..89ecad3931 100644 --- a/plugins/github-actions/src/apis/builds/BuildsClient.ts +++ b/plugins/github-actions/src/api/MockGithubActionsClient.ts @@ -14,20 +14,16 @@ * limitations under the License. */ +import { GithubActionsApi } from './GithubActionsApi'; import { Build, BuildDetails, BuildStatus } from './types'; -export class BuildsClient { - static create(): BuildsClient { - return new BuildsClient(); - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async listBuilds(_entityUri: string): Promise { +export class MockGithubActionsClient implements GithubActionsApi { + async listBuilds(): Promise { return []; } // eslint-disable-next-line @typescript-eslint/no-unused-vars - async getBuild(_buildUri: string): Promise { + async getBuild(): Promise { return { build: { commitId: 'TODO', diff --git a/plugins/github-actions/src/api/index.test.ts b/plugins/github-actions/src/api/index.test.ts new file mode 100644 index 0000000000..70319b8120 --- /dev/null +++ b/plugins/github-actions/src/api/index.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MockGithubActionsClient } from './MockGithubActionsClient'; +import { BuildStatus } from './types'; + +describe('Github Actions API', () => { + let client: MockGithubActionsClient; + beforeEach(() => { + client = new MockGithubActionsClient(); + }); + describe('Mock client', () => { + it('gets a list of builds by a project id', async () => { + await expect(client.listBuilds()).resolves.toEqual([]); + }); + it('gets a build info by its id', async () => { + await expect(client.getBuild()).resolves.toEqual({ + build: { + commitId: 'TODO', + branch: 'TODO', + uri: 'TODO', + status: BuildStatus.Running, + message: 'TODO', + }, + author: 'TODO', + logUrl: 'TODO', + overviewUrl: 'TODO', + }); + }); + }); +}); diff --git a/plugins/github-actions/src/apis/builds/index.ts b/plugins/github-actions/src/api/index.ts similarity index 88% rename from plugins/github-actions/src/apis/builds/index.ts rename to plugins/github-actions/src/api/index.ts index 9ce2150893..66c2c053fe 100644 --- a/plugins/github-actions/src/apis/builds/index.ts +++ b/plugins/github-actions/src/api/index.ts @@ -14,5 +14,6 @@ * limitations under the License. */ -export { BuildsClient } from './BuildsClient'; +export * from './GithubActionsApi'; +export * from './MockGithubActionsClient'; export * from './types'; diff --git a/plugins/github-actions/src/apis/builds/types.ts b/plugins/github-actions/src/api/types.ts similarity index 100% rename from plugins/github-actions/src/apis/builds/types.ts rename to plugins/github-actions/src/api/types.ts diff --git a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx index bb07e0d6aa..2244a4c432 100644 --- a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx +++ b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx @@ -32,8 +32,9 @@ import { import React from 'react'; import { useParams } from 'react-router-dom'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; +import { useApi } from '@backstage/core-api'; +import { githubActionsApiRef } from '../../api'; const useStyles = makeStyles(theme => ({ root: { @@ -48,12 +49,11 @@ const useStyles = makeStyles(theme => ({ }, })); -const client = BuildsClient.create(); - export const BuildDetailsPage = () => { + const api = useApi(githubActionsApiRef); const classes = useStyles(); const { buildUri } = useParams(); - const status = useAsync(() => client.getBuild(buildUri), [buildUri]); + const status = useAsync(() => api.getBuild(buildUri), [buildUri]); if (status.loading) { return ; diff --git a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx index a04189debc..2e4cbb03bd 100644 --- a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx +++ b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx @@ -27,10 +27,9 @@ import { } from '@material-ui/core'; import React from 'react'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; - -const client = BuildsClient.create(); +import { githubActionsApiRef } from '../../api'; +import { useApi } from '@backstage/core-api'; const useStyles = makeStyles(theme => ({ root: { @@ -43,7 +42,8 @@ const useStyles = makeStyles(theme => ({ export const BuildInfoCard = () => { const classes = useStyles(); - const status = useAsync(() => client.listBuilds('entity:spotify:backstage')); + const api = useApi(githubActionsApiRef); + const status = useAsync(() => api.listBuilds('entity:spotify:backstage')); let content: JSX.Element; diff --git a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx index fc784108d4..d68f756129 100644 --- a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx +++ b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx @@ -31,10 +31,9 @@ import { } from '@material-ui/core'; import React from 'react'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; - -const client = BuildsClient.create(); +import { githubActionsApiRef } from '../../api'; +import { useApi } from '@backstage/core-api'; const LongText = ({ text, max }: { text: string; max: number }) => { if (text.length < max) { @@ -57,8 +56,9 @@ const useStyles = makeStyles(theme => ({ })); const PageContents = () => { + const api = useApi(githubActionsApiRef); const { loading, error, value } = useAsync(() => - client.listBuilds('entity:spotify:backstage'), + api.listBuilds('entity:spotify:backstage'), ); if (loading) { diff --git a/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx b/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx index 332a03d67a..1a198f4df2 100644 --- a/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx +++ b/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx @@ -21,7 +21,7 @@ import SuccessIcon from '@material-ui/icons/CheckCircle'; import FailureIcon from '@material-ui/icons/Error'; import UnknownIcon from '@material-ui/icons/Help'; import React from 'react'; -import { BuildStatus } from '../../apis/builds'; +import { BuildStatus } from '../../api/types'; type Props = { status?: BuildStatus; diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 3a0a0fe2d3..d67bc6a864 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -15,3 +15,4 @@ */ export { plugin } from './plugin'; +export * from './api'; From 3d4fcc4559b2677da470ed97decadaea1b31a5fd Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 10:48:46 +0200 Subject: [PATCH 14/41] techdocs: set custom docStorageUrl via appConfig (#1575) * feat(techdocs): read docStorageUrl from appConfig * docs: updated techdocs plugin README to mention custom baseurl * fix: added config file * fix: use JSON in env variable * fix: add to app-config.yaml * typescript wrestling * Nicer config handling * fix: switch to using useApi(configApiRef) * docs: corrected to new config name * docs: mention app-config.yaml config option * fix: removed unused dep @backstage/config Co-authored-by: Sebastian Qvarfordt --- app-config.yaml | 3 ++ plugins/techdocs/README.md | 32 +++++++++++++++++++ plugins/techdocs/src/config.js | 17 ---------- .../techdocs/src/reader/components/Reader.tsx | 12 ++++--- .../reader/transformers/addBaseUrl.test.ts | 6 ++-- .../src/reader/transformers/addBaseUrl.ts | 8 ++--- .../reader/transformers/onCssReady.test.ts | 8 ++--- .../src/reader/transformers/onCssReady.ts | 6 ++-- 8 files changed, 57 insertions(+), 35 deletions(-) delete mode 100644 plugins/techdocs/src/config.js diff --git a/app-config.yaml b/app-config.yaml index 39527c20bd..79967842ef 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -12,3 +12,6 @@ backend: organization: name: Spotify + +techdocs: + storageUrl: https://techdocs-mock-sites.storage.googleapis.com diff --git a/plugins/techdocs/README.md b/plugins/techdocs/README.md index 36d00fc970..8ac34f1943 100644 --- a/plugins/techdocs/README.md +++ b/plugins/techdocs/README.md @@ -15,3 +15,35 @@ Your plugin has been added to the example app in this repository, meaning you'll You can also serve the plugin in isolation by running `yarn start` in the plugin directory. This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. + +## Configuration + +### Custom Storage URL + +TechDocs currently reads a static HTML file, generated by Mkdocs (see our `plugins/techdocs/mkdocs/container` folder for more documentation) and stored on an external server, and loads that into Backstage. By default, we have set up a mock server with some example documentation sites over in Google Cloud Storage: + +```md +# Base URL + +https://techdocs-mock-sites.storage.googleapis.com + +# Home Page for the "mkdocs" docs + +https://techdocs-mock-sites.storage.googleapis.com/mkdocs/index.html + +# Home Page for the "backstage-microsite" docs + +https://techdocs-mock-sites.storage.googleapis.com/backstage-microsite/index.html +``` + +Using your own setup (or ours which is being worked on as of Q3 2020), you can point it to your own server with your own hosted documentation sites. The only requirement is that it the output is from [Mkdocs](https://mkdocs.org) with the Material theme. You can always use our documentation generation tool located at `plugins/techdocs/mkdocs/container` for easy setup. + +To point TechDocs to your own server, simply update the `techdocs.storageUrl` value in your `app-config.yaml` file or set the environment variable `APP_CONFIG_techdocs_storageUrl` in your application: + +```bash +git clone git@github.com:spotify/backstage.git +cd backstage/ +yarn install +export APP_CONFIG_techdocs_storageUrl='"http://example-docs-site-server.com"' +yarn start +``` diff --git a/plugins/techdocs/src/config.js b/plugins/techdocs/src/config.js deleted file mode 100644 index 296b9fc14c..0000000000 --- a/plugins/techdocs/src/config.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export const docStorageURL = - 'https://techdocs-mock-sites.storage.googleapis.com'; diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index c42a3657cf..a6b23bd454 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -15,6 +15,7 @@ */ import React from 'react'; +import { useApi, configApiRef } from '@backstage/core'; import { useShadowDom } from '..'; import { useAsync } from 'react-use'; import { AsyncState } from 'react-use/lib/useAsync'; @@ -30,7 +31,6 @@ import transformer, { onCssReady, sanitizeDOM, } from '../transformers'; -import { docStorageURL } from '../../config'; import URLFormatter from '../urlFormatter'; import { TechDocsNotFound } from './TechDocsNotFound'; import { TechDocsPageWrapper } from './TechDocsPageWrapper'; @@ -69,12 +69,16 @@ const useEnforcedTrailingSlash = (): void => { export const Reader = () => { useEnforcedTrailingSlash(); + const docStorageUrl = + useApi(configApiRef).getOptionalString('techdocs.storageUrl') ?? + 'https://techdocs-mock-sites.storage.googleapis.com'; + const location = useLocation(); const { componentId, '*': path } = useParams(); const [shadowDomRef, shadowRoot] = useShadowDom(); const navigate = useNavigate(); const normalizedUrl = new URLFormatter( - `${docStorageURL}${location.pathname.replace('/docs', '')}`, + `${docStorageUrl}${location.pathname.replace('/docs', '')}`, ).formatBaseURL(); const state = useFetch(`${normalizedUrl}index.html`); @@ -91,7 +95,7 @@ export const Reader = () => { const transformedElement = transformer(state.value as string, [ sanitizeDOM(), addBaseUrl({ - docStorageURL, + docStorageUrl, componentId, path, }), @@ -137,7 +141,7 @@ export const Reader = () => { }, }), onCssReady({ - docStorageURL, + docStorageUrl, onLoading: (dom: Element) => { (dom as HTMLElement).style.setProperty('opacity', '0'); }, diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts index ceb2313b03..ce903c5d2f 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.test.ts @@ -41,7 +41,7 @@ describe('addBaseUrl', () => { const shadowDom = createTestShadowDom(FIXTURES.FIXTURE_STANDARD_PAGE, { preTransformers: [ addBaseUrl({ - docStorageURL: DOC_STORAGE_URL, + docStorageUrl: DOC_STORAGE_URL, componentId: 'example-docs', path: '', }), @@ -76,7 +76,7 @@ describe('addBaseUrl', () => { { preTransformers: [ addBaseUrl({ - docStorageURL: DOC_STORAGE_URL, + docStorageUrl: DOC_STORAGE_URL, componentId: 'example-docs', path: 'examplepath', }), @@ -112,7 +112,7 @@ describe('addBaseUrl', () => { { preTransformers: [ addBaseUrl({ - docStorageURL: DOC_STORAGE_URL, + docStorageUrl: DOC_STORAGE_URL, componentId: 'example-docs', path: 'examplepath/', }), diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts index 91986db729..42ed531aea 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts @@ -18,13 +18,13 @@ import URLFormatter from '../urlFormatter'; import type { Transformer } from './index'; type AddBaseUrlOptions = { - docStorageURL: string; + docStorageUrl: string; componentId: string; path: string; }; export const addBaseUrl = ({ - docStorageURL, + docStorageUrl, componentId, path, }: AddBaseUrlOptions): Transformer => { @@ -38,8 +38,8 @@ export const addBaseUrl = ({ .forEach((elem: T) => { const urlFormatter = new URLFormatter( path.length < 1 || path.endsWith('/') - ? `${docStorageURL}/${componentId}/${path}` - : `${docStorageURL}/${componentId}/${path}/`, + ? `${docStorageUrl}/${componentId}/${path}` + : `${docStorageUrl}/${componentId}/${path}/`, ); elem.setAttribute( diff --git a/plugins/techdocs/src/reader/transformers/onCssReady.test.ts b/plugins/techdocs/src/reader/transformers/onCssReady.test.ts index 234100fc4b..13965bcf4b 100644 --- a/plugins/techdocs/src/reader/transformers/onCssReady.test.ts +++ b/plugins/techdocs/src/reader/transformers/onCssReady.test.ts @@ -23,7 +23,7 @@ import { } from '../../test-utils'; import { addBaseUrl, onCssReady } from '../transformers'; -const docStorageURL: string = +const docStorageUrl: string = 'https://techdocs-mock-sites.storage.googleapis.com'; jest.useFakeTimers(); @@ -45,7 +45,7 @@ describe('onCssReady', () => { preTransformers: [], postTransformers: [ onCssReady({ - docStorageURL, + docStorageUrl, onLoading, onLoaded, }), @@ -65,12 +65,12 @@ describe('onCssReady', () => { preTransformers: [], postTransformers: [ addBaseUrl({ - docStorageURL, + docStorageUrl, componentId: 'mkdocs', path: '', }), onCssReady({ - docStorageURL, + docStorageUrl, onLoading, onLoaded, }), diff --git a/plugins/techdocs/src/reader/transformers/onCssReady.ts b/plugins/techdocs/src/reader/transformers/onCssReady.ts index 2d355574b6..dd50879459 100644 --- a/plugins/techdocs/src/reader/transformers/onCssReady.ts +++ b/plugins/techdocs/src/reader/transformers/onCssReady.ts @@ -17,20 +17,20 @@ import type { Transformer } from './index'; type OnCssReadyOptions = { - docStorageURL: string; + docStorageUrl: string; onLoading: (dom: Element) => void; onLoaded: (dom: Element) => void; }; export const onCssReady = ({ - docStorageURL, + docStorageUrl, onLoading, onLoaded, }: OnCssReadyOptions): Transformer => { return dom => { const cssPages = Array.from( dom.querySelectorAll('head > link[rel="stylesheet"]'), - ).filter(elem => elem.getAttribute('href')?.startsWith(docStorageURL)); + ).filter(elem => elem.getAttribute('href')?.startsWith(docStorageUrl)); let count = cssPages.length; From 5c487c127b4cc37300d07f0caaaee3e64279baff Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Fri, 10 Jul 2020 15:10:13 +0200 Subject: [PATCH 15/41] feat(github-actions): list builds based on mocked data --- .../BuildListPage/BuildListPage.tsx | 136 ++----- .../BuildListTable/BuildListTable.tsx | 176 +++++++++ .../src/components/BuildListTable/index.ts | 17 + .../components/BuildListTable/useBuilds.ts | 333 ++++++++++++++++++ 4 files changed, 558 insertions(+), 104 deletions(-) create mode 100644 plugins/github-actions/src/components/BuildListTable/BuildListTable.tsx create mode 100644 plugins/github-actions/src/components/BuildListTable/index.ts create mode 100644 plugins/github-actions/src/components/BuildListTable/useBuilds.ts diff --git a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx index fc784108d4..9f6807d8bd 100644 --- a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx +++ b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx @@ -14,115 +14,43 @@ * limitations under the License. */ -import { Link } from '@backstage/core'; import { - LinearProgress, - makeStyles, - Paper, - Table, - TableBody, - TableCell, - TableContainer, - TableHead, - TableRow, - Theme, - Tooltip, - Typography, -} from '@material-ui/core'; + Header, + HeaderLabel, + pageTheme, + Page, + Content, + ContentHeader, + SupportButton, +} from '@backstage/core'; +import { Grid } from '@material-ui/core'; import React from 'react'; -import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; -import { BuildStatusIndicator } from '../BuildStatusIndicator'; -const client = BuildsClient.create(); - -const LongText = ({ text, max }: { text: string; max: number }) => { - if (text.length < max) { - return {text}; - } - return ( - - {text.slice(0, max)}... - - ); -}; - -const useStyles = makeStyles(theme => ({ - root: { - padding: theme.spacing(2), - }, - title: { - padding: theme.spacing(1, 0, 2, 0), - }, -})); - -const PageContents = () => { - const { loading, error, value } = useAsync(() => - client.listBuilds('entity:spotify:backstage'), - ); - - if (loading) { - return ; - } - - if (error) { - return ( - - Failed to load builds, {error.message}{' '} - - ); - } - - return ( - - - - - Status - Branch - Message - Commit - - - - {value!.map(build => ( - - - - - - - - - - - - - - - - - - - {build.commitId.slice(0, 10)} - - - - ))} - -
- - ); -}; +import { BuildListTable } from '../BuildListTable'; export const BuildListPage = () => { - const classes = useStyles(); return ( -
- - CI/CD Builds - - -
+ +
+ + +
+ + + + This plugin allows you to view and interact with your builds within + the GitHub Actions environment. + + + + + + + + + ); }; diff --git a/plugins/github-actions/src/components/BuildListTable/BuildListTable.tsx b/plugins/github-actions/src/components/BuildListTable/BuildListTable.tsx new file mode 100644 index 0000000000..dc593afdbc --- /dev/null +++ b/plugins/github-actions/src/components/BuildListTable/BuildListTable.tsx @@ -0,0 +1,176 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { FC } from 'react'; +import { Link, Typography, Box, IconButton } from '@material-ui/core'; +import RetryIcon from '@material-ui/icons/Replay'; +import GitHubIcon from '@material-ui/icons/GitHub'; +import { Link as RouterLink } from 'react-router-dom'; +import { + StatusError, + StatusWarning, + StatusOK, + StatusPending, + StatusRunning, + Table, + TableColumn, +} from '@backstage/core'; +import { useBuilds } from './useBuilds'; + +export type Build = { + id: string; + buildName: string; + buildUrl?: string; + source: { + branchName: string; + commit: { + hash: string; + url: string; + }; + }; + status: string; + onRestartClick: () => void; +}; + +// retried, canceled, infrastructure_fail, timedout, not_run, running, failed, queued, scheduled, not_running, no_tests, fixed, success +const getStatusComponent = (status: string | undefined = '') => { + switch (status.toLowerCase()) { + case 'queued': + case 'scheduled': + return ; + case 'running': + return ; + case 'failed': + return ; + case 'success': + return ; + case 'canceled': + default: + return ; + } +}; + +const generatedColumns: TableColumn[] = [ + { + title: 'ID', + field: 'id', + type: 'numeric', + width: '150px', + }, + { + title: 'Build', + field: 'buildName', + highlight: true, + render: (row: Partial) => ( + + {row.buildName} + + ), + }, + { + title: 'Source', + render: (row: Partial) => ( + <> +

{row.source?.branchName}

+

{row.source?.commit.hash}

+ + ), + }, + { + title: 'Status', + render: (row: Partial) => ( + + {getStatusComponent(row.status)} + + {row.status} + + ), + }, + { + title: 'Actions', + render: (row: Partial) => ( + + + + ), + width: '10%', + }, +]; + +type Props = { + loading: boolean; + retry: () => void; + builds?: Build[]; + projectName: string; + page: number; + onChangePage: (page: number) => void; + total: number; + pageSize: number; + onChangePageSize: (pageSize: number) => void; +}; + +const BuildListTableView: FC = ({ + projectName, + loading, + pageSize, + page, + retry, + builds, + onChangePage, + onChangePageSize, + total, +}) => { + return ( + , + tooltip: 'Refresh Data', + isFreeAction: true, + onClick: () => retry(), + }, + ]} + data={builds ?? []} + onChangePage={onChangePage} + onChangeRowsPerPage={onChangePageSize} + title={ + + + + {projectName} + + } + columns={generatedColumns} + /> + ); +}; + +const noop = () => {}; + +export const BuildListTable = () => { + const [tableProps] = useBuilds(); + return ( + + ); +}; diff --git a/plugins/github-actions/src/components/BuildListTable/index.ts b/plugins/github-actions/src/components/BuildListTable/index.ts new file mode 100644 index 0000000000..089d53db94 --- /dev/null +++ b/plugins/github-actions/src/components/BuildListTable/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { BuildListTable } from './BuildListTable'; +export type { Build } from './BuildListTable'; diff --git a/plugins/github-actions/src/components/BuildListTable/useBuilds.ts b/plugins/github-actions/src/components/BuildListTable/useBuilds.ts new file mode 100644 index 0000000000..20f58a92d8 --- /dev/null +++ b/plugins/github-actions/src/components/BuildListTable/useBuilds.ts @@ -0,0 +1,333 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useCallback, useState } from 'react'; +import { useAsyncRetry } from 'react-use'; +import { Build } from './BuildListTable'; + +// TODO(shmidt-i): use real APIs +const useEntityGHSettings = () => ({ repo: 'test', owner: 'shmidt-i-test' }); +const buildsMock = { + total_count: 1, + workflow_runs: [ + { + id: 30433642, + node_id: 'MDEyOldvcmtmbG93IFJ1bjI2OTI4OQ==', + head_branch: 'master', + head_sha: 'acb5820ced9479c074f688cc328bf03f341a511d', + run_number: 562, + event: 'push', + status: 'queued', + conclusion: null, + workflow_id: 159038, + url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642', + html_url: 'https://github.com/octo-org/octo-repo/actions/runs/30433642', + pull_requests: [], + created_at: '2020-01-22T19:33:08Z', + updated_at: '2020-01-22T19:33:08Z', + jobs_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/jobs', + logs_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/logs', + check_suite_url: + 'https://api.github.com/repos/octo-org/octo-repo/check-suites/414944374', + artifacts_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/artifacts', + cancel_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/cancel', + rerun_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/runs/30433642/rerun', + workflow_url: + 'https://api.github.com/repos/octo-org/octo-repo/actions/workflows/159038', + head_commit: { + id: 'acb5820ced9479c074f688cc328bf03f341a511d', + tree_id: 'd23f6eedb1e1b9610bbc754ddb5197bfe7271223', + message: 'Create linter.yml', + timestamp: '2020-01-22T19:33:05Z', + author: { + name: 'Octo Cat', + email: 'octocat@github.com', + }, + committer: { + name: 'GitHub', + email: 'noreply@github.com', + }, + }, + repository: { + id: 1296269, + node_id: 'MDEwOlJlcG9zaXRvcnkxMjk2MjY5', + name: 'Hello-World', + full_name: 'octocat/Hello-World', + owner: { + login: 'octocat', + id: 1, + node_id: 'MDQ6VXNlcjE=', + avatar_url: 'https://github.com/images/error/octocat_happy.gif', + gravatar_id: '', + url: 'https://api.github.com/users/octocat', + html_url: 'https://github.com/octocat', + followers_url: 'https://api.github.com/users/octocat/followers', + following_url: + 'https://api.github.com/users/octocat/following{/other_user}', + gists_url: 'https://api.github.com/users/octocat/gists{/gist_id}', + starred_url: + 'https://api.github.com/users/octocat/starred{/owner}{/repo}', + subscriptions_url: + 'https://api.github.com/users/octocat/subscriptions', + organizations_url: 'https://api.github.com/users/octocat/orgs', + repos_url: 'https://api.github.com/users/octocat/repos', + events_url: 'https://api.github.com/users/octocat/events{/privacy}', + received_events_url: + 'https://api.github.com/users/octocat/received_events', + type: 'User', + site_admin: false, + }, + private: false, + html_url: 'https://github.com/octocat/Hello-World', + description: 'This your first repo!', + fork: false, + url: 'https://api.github.com/repos/octocat/Hello-World', + archive_url: + 'http://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}', + assignees_url: + 'http://api.github.com/repos/octocat/Hello-World/assignees{/user}', + blobs_url: + 'http://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}', + branches_url: + 'http://api.github.com/repos/octocat/Hello-World/branches{/branch}', + collaborators_url: + 'http://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}', + comments_url: + 'http://api.github.com/repos/octocat/Hello-World/comments{/number}', + commits_url: + 'http://api.github.com/repos/octocat/Hello-World/commits{/sha}', + compare_url: + 'http://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}', + contents_url: + 'http://api.github.com/repos/octocat/Hello-World/contents/{+path}', + contributors_url: + 'http://api.github.com/repos/octocat/Hello-World/contributors', + deployments_url: + 'http://api.github.com/repos/octocat/Hello-World/deployments', + downloads_url: + 'http://api.github.com/repos/octocat/Hello-World/downloads', + events_url: 'http://api.github.com/repos/octocat/Hello-World/events', + forks_url: 'http://api.github.com/repos/octocat/Hello-World/forks', + git_commits_url: + 'http://api.github.com/repos/octocat/Hello-World/git/commits{/sha}', + git_refs_url: + 'http://api.github.com/repos/octocat/Hello-World/git/refs{/sha}', + git_tags_url: + 'http://api.github.com/repos/octocat/Hello-World/git/tags{/sha}', + git_url: 'git:github.com/octocat/Hello-World.git', + issue_comment_url: + 'http://api.github.com/repos/octocat/Hello-World/issues/comments{/number}', + issue_events_url: + 'http://api.github.com/repos/octocat/Hello-World/issues/events{/number}', + issues_url: + 'http://api.github.com/repos/octocat/Hello-World/issues{/number}', + keys_url: + 'http://api.github.com/repos/octocat/Hello-World/keys{/key_id}', + labels_url: + 'http://api.github.com/repos/octocat/Hello-World/labels{/name}', + languages_url: + 'http://api.github.com/repos/octocat/Hello-World/languages', + merges_url: 'http://api.github.com/repos/octocat/Hello-World/merges', + milestones_url: + 'http://api.github.com/repos/octocat/Hello-World/milestones{/number}', + notifications_url: + 'http://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}', + pulls_url: + 'http://api.github.com/repos/octocat/Hello-World/pulls{/number}', + releases_url: + 'http://api.github.com/repos/octocat/Hello-World/releases{/id}', + ssh_url: 'git@github.com:octocat/Hello-World.git', + stargazers_url: + 'http://api.github.com/repos/octocat/Hello-World/stargazers', + statuses_url: + 'http://api.github.com/repos/octocat/Hello-World/statuses/{sha}', + subscribers_url: + 'http://api.github.com/repos/octocat/Hello-World/subscribers', + subscription_url: + 'http://api.github.com/repos/octocat/Hello-World/subscription', + tags_url: 'http://api.github.com/repos/octocat/Hello-World/tags', + teams_url: 'http://api.github.com/repos/octocat/Hello-World/teams', + trees_url: + 'http://api.github.com/repos/octocat/Hello-World/git/trees{/sha}', + }, + head_repository: { + id: 217723378, + node_id: 'MDEwOlJlcG9zaXRvcnkyMTc3MjMzNzg=', + name: 'octo-repo', + full_name: 'octo-org/octo-repo', + private: true, + owner: { + login: 'octocat', + id: 1, + node_id: 'MDQ6VXNlcjE=', + avatar_url: 'https://github.com/images/error/octocat_happy.gif', + gravatar_id: '', + url: 'https://api.github.com/users/octocat', + html_url: 'https://github.com/octocat', + followers_url: 'https://api.github.com/users/octocat/followers', + following_url: + 'https://api.github.com/users/octocat/following{/other_user}', + gists_url: 'https://api.github.com/users/octocat/gists{/gist_id}', + starred_url: + 'https://api.github.com/users/octocat/starred{/owner}{/repo}', + subscriptions_url: + 'https://api.github.com/users/octocat/subscriptions', + organizations_url: 'https://api.github.com/users/octocat/orgs', + repos_url: 'https://api.github.com/users/octocat/repos', + events_url: 'https://api.github.com/users/octocat/events{/privacy}', + received_events_url: + 'https://api.github.com/users/octocat/received_events', + type: 'User', + site_admin: false, + }, + html_url: 'https://github.com/octo-org/octo-repo', + description: null, + fork: false, + url: 'https://api.github.com/repos/octo-org/octo-repo', + forks_url: 'https://api.github.com/repos/octo-org/octo-repo/forks', + keys_url: + 'https://api.github.com/repos/octo-org/octo-repo/keys{/key_id}', + collaborators_url: + 'https://api.github.com/repos/octo-org/octo-repo/collaborators{/collaborator}', + teams_url: 'https://api.github.com/repos/octo-org/octo-repo/teams', + hooks_url: 'https://api.github.com/repos/octo-org/octo-repo/hooks', + issue_events_url: + 'https://api.github.com/repos/octo-org/octo-repo/issues/events{/number}', + events_url: 'https://api.github.com/repos/octo-org/octo-repo/events', + assignees_url: + 'https://api.github.com/repos/octo-org/octo-repo/assignees{/user}', + branches_url: + 'https://api.github.com/repos/octo-org/octo-repo/branches{/branch}', + tags_url: 'https://api.github.com/repos/octo-org/octo-repo/tags', + blobs_url: + 'https://api.github.com/repos/octo-org/octo-repo/git/blobs{/sha}', + git_tags_url: + 'https://api.github.com/repos/octo-org/octo-repo/git/tags{/sha}', + git_refs_url: + 'https://api.github.com/repos/octo-org/octo-repo/git/refs{/sha}', + trees_url: + 'https://api.github.com/repos/octo-org/octo-repo/git/trees{/sha}', + statuses_url: + 'https://api.github.com/repos/octo-org/octo-repo/statuses/{sha}', + languages_url: + 'https://api.github.com/repos/octo-org/octo-repo/languages', + stargazers_url: + 'https://api.github.com/repos/octo-org/octo-repo/stargazers', + contributors_url: + 'https://api.github.com/repos/octo-org/octo-repo/contributors', + subscribers_url: + 'https://api.github.com/repos/octo-org/octo-repo/subscribers', + subscription_url: + 'https://api.github.com/repos/octo-org/octo-repo/subscription', + commits_url: + 'https://api.github.com/repos/octo-org/octo-repo/commits{/sha}', + git_commits_url: + 'https://api.github.com/repos/octo-org/octo-repo/git/commits{/sha}', + comments_url: + 'https://api.github.com/repos/octo-org/octo-repo/comments{/number}', + issue_comment_url: + 'https://api.github.com/repos/octo-org/octo-repo/issues/comments{/number}', + contents_url: + 'https://api.github.com/repos/octo-org/octo-repo/contents/{+path}', + compare_url: + 'https://api.github.com/repos/octo-org/octo-repo/compare/{base}...{head}', + merges_url: 'https://api.github.com/repos/octo-org/octo-repo/merges', + archive_url: + 'https://api.github.com/repos/octo-org/octo-repo/{archive_format}{/ref}', + downloads_url: + 'https://api.github.com/repos/octo-org/octo-repo/downloads', + issues_url: + 'https://api.github.com/repos/octo-org/octo-repo/issues{/number}', + pulls_url: + 'https://api.github.com/repos/octo-org/octo-repo/pulls{/number}', + milestones_url: + 'https://api.github.com/repos/octo-org/octo-repo/milestones{/number}', + notifications_url: + 'https://api.github.com/repos/octo-org/octo-repo/notifications{?since,all,participating}', + labels_url: + 'https://api.github.com/repos/octo-org/octo-repo/labels{/name}', + releases_url: + 'https://api.github.com/repos/octo-org/octo-repo/releases{/id}', + deployments_url: + 'https://api.github.com/repos/octo-org/octo-repo/deployments', + }, + }, + ], +}; + +export function useBuilds() { + const { repo, owner } = useEntityGHSettings(); + + const [total, setTotal] = useState(0); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(5); + const getBuilds = useCallback(async (_options: any) => buildsMock, []); + + const restartBuild = async (_buildId: number) => {}; + + const { loading, value: builds, retry } = useAsyncRetry( + () => + getBuilds({ + perPage: pageSize, + page: page, + }).then((allBuilds): Build[] => { + setTotal(allBuilds.total_count); + // Transformation here + return allBuilds.workflow_runs.map(run => ({ + buildName: run.head_commit.message, + id: `${run.id}`, + onRestartClick: () => {}, + source: { + branchName: run.head_branch, + commit: { + hash: run.head_commit.id, + url: run.head_repository.branches_url.replace( + '{/branch}', + run.head_branch, + ), + }, + }, + status: run.status, + buildUrl: run.url, + })); + }), + [page, pageSize, getBuilds], + ); + + const projectName = `${owner}/${repo}`; + return [ + { + page, + pageSize, + loading, + builds, + projectName, + total, + }, + { + getBuilds, + setPage, + setPageSize, + restartBuild, + retry, + }, + ] as const; +} From 5e93d376b515f2260656daeac3cefbacf7efa79e Mon Sep 17 00:00:00 2001 From: ebarrios Date: Fri, 10 Jul 2020 15:48:03 +0200 Subject: [PATCH 16/41] Removed Mock implementation and tried to add the real one --- plugins/github-actions/package.json | 2 + .../src/api/GithubActionsApi.ts | 29 +++ .../src/api/GithubActionsClient.ts | 154 ++++++++++++ plugins/github-actions/src/api/index.test.ts | 44 ++++ plugins/github-actions/src/api/index.ts | 19 ++ plugins/github-actions/src/api/types.ts | 224 ++++++++++++++++++ .../BuildDetailsPage/BuildDetailsPage.tsx | 11 +- .../BuildInfoCard/BuildInfoCard.tsx | 8 +- .../BuildListPage/BuildListPage.tsx | 20 +- .../BuildStatusIndicator.tsx | 2 +- plugins/github-actions/src/index.ts | 1 + 11 files changed, 497 insertions(+), 17 deletions(-) create mode 100644 plugins/github-actions/src/api/GithubActionsApi.ts create mode 100644 plugins/github-actions/src/api/GithubActionsClient.ts create mode 100644 plugins/github-actions/src/api/index.test.ts create mode 100644 plugins/github-actions/src/api/index.ts create mode 100644 plugins/github-actions/src/api/types.ts diff --git a/plugins/github-actions/package.json b/plugins/github-actions/package.json index 456bd826c0..8ac474169f 100644 --- a/plugins/github-actions/package.json +++ b/plugins/github-actions/package.json @@ -21,7 +21,9 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/catalog-model": "^0.1.1-alpha.13", "@backstage/core": "^0.1.1-alpha.13", + "@backstage/core-api": "^0.1.1-alpha.13", "@backstage/theme": "^0.1.1-alpha.13", "@material-ui/core": "^4.9.1", "@material-ui/icons": "^4.9.1", diff --git a/plugins/github-actions/src/api/GithubActionsApi.ts b/plugins/github-actions/src/api/GithubActionsApi.ts new file mode 100644 index 0000000000..58892173ed --- /dev/null +++ b/plugins/github-actions/src/api/GithubActionsApi.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createApiRef } from '@backstage/core'; +import { Build, BuildDetails } from './types'; +import { Entity } from '@backstage/catalog-model'; + +export const githubActionsApiRef = createApiRef({ + id: 'plugin.githubactions.service', + description: 'Used by the Github Actions plugin to make requests', +}); + +export type GithubActionsApi = { + listBuilds: (entity: Entity, token: Promise) => Promise; + getBuild: (buildUri: string, token: Promise) => Promise; +}; diff --git a/plugins/github-actions/src/api/GithubActionsClient.ts b/plugins/github-actions/src/api/GithubActionsClient.ts new file mode 100644 index 0000000000..0c9a98e57c --- /dev/null +++ b/plugins/github-actions/src/api/GithubActionsClient.ts @@ -0,0 +1,154 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubActionsApi } from './GithubActionsApi'; +import { Build, BuildDetails, BuildStatus, WorkflowRun } from './types'; +import { Entity } from '@backstage/catalog-model'; + +export class GithubActionsClient implements GithubActionsApi { + async listBuilds(entity: Entity, token: Promise): Promise { + // ### Feedback request ### + // I asumed the following: (maybe not the best. Ideally this should come from the link to the component.yaml file) + // entity.metadata.namespace => org name + // entity.metadata.name => repo name + // entityUri -> entity:spotify:backstage + + let url: string; + if (entity.metadata.name !== '') { + url = `https://api.github.com/repos/${entity.metadata.namespace}/${entity.metadata.name}/runs`; + } else { + url = 'https://api.github.com/repos/spotify/backstage/actions/runs'; + } + + const response = await fetch(url, { + headers: new Headers({ + Authorization: `Bearer ${await token}`, + }), + }); + + if (response.status > 200) { + return [ + { + commitId: 'Error', + message: 'ResponseCode > 200', + branch: 'Error', + status: BuildStatus.Failure, + uri: 'Error', + }, + ]; + } + + const data = await response.json(); + + const newData: WorkflowRun[] = data.workflow_runs; + + const endData: Build[] = []; + + newData.forEach((element, index) => { + const transData: Build = { + commitId: '', + message: '', + branch: '', + status: BuildStatus.Null, + uri: '', + }; + transData.commitId = String(element.head_commit.id); + transData.branch = element.head_branch; + + // ### Feedback request ### + // TODO: I am not sure about this part. Looks ugly. Maybe there is a better way of doing this. + if (element.conclusion === 'success') { + transData.status = BuildStatus.Success; + } else if (element.conclusion === 'failure') { + transData.status = BuildStatus.Failure; + } else if (element.conclusion === 'pending') { + transData.status = BuildStatus.Pending; + } else if (element.conclusion === 'running') { + transData.status = BuildStatus.Running; + } else { + if (element.status === 'in_progress') { + transData.status = BuildStatus.Running; + } else { + transData.status = BuildStatus.Null; + } + } + transData.message = element.head_commit.message; + transData.uri = element.url; + endData[index] = transData; + }); + + return endData; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async getBuild( + buildUri: string, + token: Promise, + ): Promise { + const response = await fetch(buildUri, { + headers: new Headers({ + Authorization: `Bearer ${await token}`, + }), + }); + const buildBlank: Build = { + commitId: '', + message: '', + branch: '', + status: BuildStatus.Null, + uri: '', + }; + + const dataBlank: BuildDetails = { + build: buildBlank, + author: '', + logUrl: '', + overviewUrl: '', + }; + + if (response.status > 200) { + return dataBlank; + } + + const data = await response.json(); + + const newData: WorkflowRun = data; + + dataBlank.author = newData.head_commit.author.name; + dataBlank.build.branch = newData.head_branch; + dataBlank.build.commitId = newData.head_commit.id; + dataBlank.build.message = newData.head_commit.message; + + // ### Feedback request ### + // TODO: I am not sure about this part. Look ugly. Maybe there is a better way of doing this. + if (newData.status === 'completed') { + dataBlank.build.status = BuildStatus.Success; + } else if (newData.status === 'in_progress') { + dataBlank.build.status = BuildStatus.Running; + } else if (newData.status === 'pending') { + dataBlank.build.status = BuildStatus.Pending; + } else if (newData.status === 'failure') { + dataBlank.build.status = BuildStatus.Failure; + } else { + dataBlank.build.status = BuildStatus.Null; + } + + dataBlank.build.uri = newData.url; + dataBlank.logUrl = newData.logs_url; + dataBlank.overviewUrl = newData.html_url; + + return dataBlank; + } +} diff --git a/plugins/github-actions/src/api/index.test.ts b/plugins/github-actions/src/api/index.test.ts new file mode 100644 index 0000000000..feff353115 --- /dev/null +++ b/plugins/github-actions/src/api/index.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GithubActionsClient } from './GithubActionsClient'; +import { BuildStatus } from './types'; + +describe('Github Actions API', () => { + let client: GithubActionsClient; + beforeEach(() => { + client = new GithubActionsClient(); + }); + describe('Mock client', () => { + it('gets a list of builds by a project id', async () => { + await expect(client.listBuilds()).resolves.toEqual([]); + }); + it('gets a build info by its id', async () => { + await expect(client.getBuild()).resolves.toEqual({ + build: { + commitId: 'TODO', + branch: 'TODO', + uri: 'TODO', + status: BuildStatus.Running, + message: 'TODO', + }, + author: 'TODO', + logUrl: 'TODO', + overviewUrl: 'TODO', + }); + }); + }); +}); diff --git a/plugins/github-actions/src/api/index.ts b/plugins/github-actions/src/api/index.ts new file mode 100644 index 0000000000..9383250bfb --- /dev/null +++ b/plugins/github-actions/src/api/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './GithubActionsApi'; +export * from './GithubActionsClient'; +export * from './types'; diff --git a/plugins/github-actions/src/api/types.ts b/plugins/github-actions/src/api/types.ts new file mode 100644 index 0000000000..8a7b2ca548 --- /dev/null +++ b/plugins/github-actions/src/api/types.ts @@ -0,0 +1,224 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export enum BuildStatus { + Null, + Success, + Failure, + Pending, + Running, +} + +export type Build = { + commitId: string; + message: string; + branch: string; + status: BuildStatus; + uri: string; +}; + +export type BuildDetails = { + build: Build; + author: string; + logUrl: string; + overviewUrl: string; +}; + +export interface Author { + name: string; + email: string; +} + +export interface Committer { + name: string; + email: string; +} + +export interface HeadCommit { + id: string; + tree_id: string; + message: string; + timestamp: Date; + author: Author; + committer: Committer; +} + +export interface Owner { + login: string; + id: number; + node_id: string; + avatar_url: string; + gravatar_id: string; + url: string; + html_url: string; + followers_url: string; + following_url: string; + gists_url: string; + starred_url: string; + subscriptions_url: string; + organizations_url: string; + repos_url: string; + events_url: string; + received_events_url: string; + type: string; + site_admin: boolean; +} + +export interface Repository { + id: number; + node_id: string; + name: string; + full_name: string; + private: boolean; + owner: Owner; + html_url: string; + description?: any; + fork: boolean; + url: string; + forks_url: string; + keys_url: string; + collaborators_url: string; + teams_url: string; + hooks_url: string; + issue_events_url: string; + events_url: string; + assignees_url: string; + branches_url: string; + tags_url: string; + blobs_url: string; + git_tags_url: string; + git_refs_url: string; + trees_url: string; + statuses_url: string; + languages_url: string; + stargazers_url: string; + contributors_url: string; + subscribers_url: string; + subscription_url: string; + commits_url: string; + git_commits_url: string; + comments_url: string; + issue_comment_url: string; + contents_url: string; + compare_url: string; + merges_url: string; + archive_url: string; + downloads_url: string; + issues_url: string; + pulls_url: string; + milestones_url: string; + notifications_url: string; + labels_url: string; + releases_url: string; + deployments_url: string; +} + +export interface Owner2 { + login: string; + id: number; + node_id: string; + avatar_url: string; + gravatar_id: string; + url: string; + html_url: string; + followers_url: string; + following_url: string; + gists_url: string; + starred_url: string; + subscriptions_url: string; + organizations_url: string; + repos_url: string; + events_url: string; + received_events_url: string; + type: string; + site_admin: boolean; +} + +export interface HeadRepository { + id: number; + node_id: string; + name: string; + full_name: string; + private: boolean; + owner: Owner2; + html_url: string; + description?: any; + fork: boolean; + url: string; + forks_url: string; + keys_url: string; + collaborators_url: string; + teams_url: string; + hooks_url: string; + issue_events_url: string; + events_url: string; + assignees_url: string; + branches_url: string; + tags_url: string; + blobs_url: string; + git_tags_url: string; + git_refs_url: string; + trees_url: string; + statuses_url: string; + languages_url: string; + stargazers_url: string; + contributors_url: string; + subscribers_url: string; + subscription_url: string; + commits_url: string; + git_commits_url: string; + comments_url: string; + issue_comment_url: string; + contents_url: string; + compare_url: string; + merges_url: string; + archive_url: string; + downloads_url: string; + issues_url: string; + pulls_url: string; + milestones_url: string; + notifications_url: string; + labels_url: string; + releases_url: string; + deployments_url: string; +} + +export interface WorkflowRun { + id: number; + node_id: string; + head_branch: string; + head_sha: string; + run_number: number; + event: string; + status: string; + conclusion: string; + workflow_id: number; + url: string; + html_url: string; + pull_requests: any[]; + created_at: Date; + updated_at: Date; + jobs_url: string; + logs_url: string; + check_suite_url: string; + artifacts_url: string; + cancel_url: string; + rerun_url: string; + workflow_url: string; + head_commit: HeadCommit; + repository: Repository; + head_repository: HeadRepository; +} diff --git a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx index bb07e0d6aa..84e9332837 100644 --- a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx +++ b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx @@ -32,8 +32,9 @@ import { import React from 'react'; import { useParams } from 'react-router-dom'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; +import { useApi, githubAuthApiRef } from '@backstage/core-api'; +import { githubActionsApiRef } from '../../api'; const useStyles = makeStyles(theme => ({ root: { @@ -48,12 +49,14 @@ const useStyles = makeStyles(theme => ({ }, })); -const client = BuildsClient.create(); - export const BuildDetailsPage = () => { + const api = useApi(githubActionsApiRef); + const githubApi = useApi(githubAuthApiRef); + const token = githubApi.getAccessToken('repo'); + const classes = useStyles(); const { buildUri } = useParams(); - const status = useAsync(() => client.getBuild(buildUri), [buildUri]); + const status = useAsync(() => api.getBuild(buildUri, token), [buildUri]); if (status.loading) { return ; diff --git a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx index a04189debc..2e4cbb03bd 100644 --- a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx +++ b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx @@ -27,10 +27,9 @@ import { } from '@material-ui/core'; import React from 'react'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; - -const client = BuildsClient.create(); +import { githubActionsApiRef } from '../../api'; +import { useApi } from '@backstage/core-api'; const useStyles = makeStyles(theme => ({ root: { @@ -43,7 +42,8 @@ const useStyles = makeStyles(theme => ({ export const BuildInfoCard = () => { const classes = useStyles(); - const status = useAsync(() => client.listBuilds('entity:spotify:backstage')); + const api = useApi(githubActionsApiRef); + const status = useAsync(() => api.listBuilds('entity:spotify:backstage')); let content: JSX.Element; diff --git a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx index fc784108d4..ae979d0cc0 100644 --- a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx +++ b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx @@ -29,12 +29,12 @@ import { Tooltip, Typography, } from '@material-ui/core'; -import React from 'react'; +import React, { FC } from 'react'; import { useAsync } from 'react-use'; -import { BuildsClient } from '../../apis/builds'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; - -const client = BuildsClient.create(); +import { githubActionsApiRef } from '../../api'; +import { useApi, githubAuthApiRef } from '@backstage/core-api'; +import { Entity } from '@backstage/catalog-model'; const LongText = ({ text, max }: { text: string; max: number }) => { if (text.length < max) { @@ -56,9 +56,13 @@ const useStyles = makeStyles(theme => ({ }, })); -const PageContents = () => { +const PageContents: FC<{ entity: Entity }> = ({ entity }) => { + const api = useApi(githubActionsApiRef); + const githubApi = useApi(githubAuthApiRef); + const token = githubApi.getAccessToken('repo'); + const { loading, error, value } = useAsync(() => - client.listBuilds('entity:spotify:backstage'), + api.listBuilds(entity, token), ); if (loading) { @@ -115,14 +119,14 @@ const PageContents = () => { ); }; -export const BuildListPage = () => { +export const BuildListPage: FC<{ entity: Entity }> = ({ entity }) => { const classes = useStyles(); return (
CI/CD Builds - +
); }; diff --git a/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx b/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx index 332a03d67a..1a198f4df2 100644 --- a/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx +++ b/plugins/github-actions/src/components/BuildStatusIndicator/BuildStatusIndicator.tsx @@ -21,7 +21,7 @@ import SuccessIcon from '@material-ui/icons/CheckCircle'; import FailureIcon from '@material-ui/icons/Error'; import UnknownIcon from '@material-ui/icons/Help'; import React from 'react'; -import { BuildStatus } from '../../apis/builds'; +import { BuildStatus } from '../../api/types'; type Props = { status?: BuildStatus; diff --git a/plugins/github-actions/src/index.ts b/plugins/github-actions/src/index.ts index 3a0a0fe2d3..d67bc6a864 100644 --- a/plugins/github-actions/src/index.ts +++ b/plugins/github-actions/src/index.ts @@ -15,3 +15,4 @@ */ export { plugin } from './plugin'; +export * from './api'; From 05519f880b9204e881c04f387ae566d4ec6c977d Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Fri, 10 Jul 2020 15:51:46 +0200 Subject: [PATCH 17/41] fix: lint --- .../src/components/BuildListTable/useBuilds.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/github-actions/src/components/BuildListTable/useBuilds.ts b/plugins/github-actions/src/components/BuildListTable/useBuilds.ts index 20f58a92d8..2ac5ef98d6 100644 --- a/plugins/github-actions/src/components/BuildListTable/useBuilds.ts +++ b/plugins/github-actions/src/components/BuildListTable/useBuilds.ts @@ -279,16 +279,13 @@ export function useBuilds() { const [total, setTotal] = useState(0); const [page, setPage] = useState(0); const [pageSize, setPageSize] = useState(5); - const getBuilds = useCallback(async (_options: any) => buildsMock, []); + const getBuilds = useCallback(async () => buildsMock, []); - const restartBuild = async (_buildId: number) => {}; + const restartBuild = async () => {}; const { loading, value: builds, retry } = useAsyncRetry( () => - getBuilds({ - perPage: pageSize, - page: page, - }).then((allBuilds): Build[] => { + getBuilds().then((allBuilds): Build[] => { setTotal(allBuilds.total_count); // Transformation here return allBuilds.workflow_runs.map(run => ({ From 353cf5a14f7c30b68aec96c28c4ce87857f15405 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:01:45 +0200 Subject: [PATCH 18/41] techdocs: implement serve for the techdocs-cli (#1581) * feat(techdocs-cli): point techdocs export to localhost:3000/api * feat(techdocs-cli): add http-proxy package * feat(techdocs-cli): add react-dev-utils package * feat(techdocs-cli): implement serve and serve:mkdocs commands * fix: use [module] prefix convention in console.log * docs: added commands * wip * Fix type errors Co-authored-by: Sebastian Qvarfordt --- packages/techdocs-cli/README.md | 10 ++ packages/techdocs-cli/bin/build | 2 +- packages/techdocs-cli/package.json | 2 + packages/techdocs-cli/src/index.ts | 120 +++++++++++++------- packages/techdocs-cli/src/lib/httpServer.ts | 62 ++++++++-- yarn.lock | 9 ++ 6 files changed, 156 insertions(+), 49 deletions(-) diff --git a/packages/techdocs-cli/README.md b/packages/techdocs-cli/README.md index 8d86363c12..f751497e83 100644 --- a/packages/techdocs-cli/README.md +++ b/packages/techdocs-cli/README.md @@ -4,6 +4,16 @@ Check out the [TechDocs README](https://github.com/spotify/backstage/blob/master **WIP: This cli is a work in progress. It is not ready for use yet. Follow our progress on [the Backstage Discord](https://discord.gg/MUpMjP2) under #docs-like-code or on [our GitHub Milestone](https://github.com/spotify/backstage/milestone/15).** +## Commands + +```bash +# Serve localhost:3000 (and localhost:8000) +yarn serve + +# Serve localhost:8000 containing your Mkdocs documentation. +yarn serve:mkdocs +``` + ## Getting Started You'll need Docker installed and running to use this. You will also need to build the container located at `plugins/techdocs/mkdocs/container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: diff --git a/packages/techdocs-cli/bin/build b/packages/techdocs-cli/bin/build index c0a1cacdfc..a86954ace9 100755 --- a/packages/techdocs-cli/bin/build +++ b/packages/techdocs-cli/bin/build @@ -22,7 +22,7 @@ TECHDOCS_PREVIEW_DEST=$ROOT_DIR/packages/techdocs-cli/dist/techdocs-preview-bund backstage-cli build --outputs cjs # Create export of the TechDocs plugin -yarn workspace @backstage/plugin-techdocs export +APP_CONFIG_techdocs_storageUrl='"http://localhost:3000/api"' yarn workspace @backstage/plugin-techdocs export # Copy over export to techdocs-cli dist/ folder cp -r $TECHDOCS_PREVIEW_SOURCE $TECHDOCS_PREVIEW_DEST diff --git a/packages/techdocs-cli/package.json b/packages/techdocs-cli/package.json index c8985367b0..97c5cbe049 100644 --- a/packages/techdocs-cli/package.json +++ b/packages/techdocs-cli/package.json @@ -49,6 +49,8 @@ "chalk": "^4.1.0", "commander": "^5.1.0", "fs-extra": "^9.0.1", + "http-proxy": "^1.18.1", + "react-dev-utils": "^10.2.1", "serve-handler": "^6.1.3" } } diff --git a/packages/techdocs-cli/src/index.ts b/packages/techdocs-cli/src/index.ts index f9ab1c45de..5b4f947a73 100644 --- a/packages/techdocs-cli/src/index.ts +++ b/packages/techdocs-cli/src/index.ts @@ -13,17 +13,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { spawn, ChildProcess } from 'child_process'; import program from 'commander'; import { version } from './lib/version'; // import chalk from 'chalk'; -import { spawn } from 'child_process'; import path from 'path'; -// import HTTPServer from './lib/httpServer'; +import HTTPServer from './lib/httpServer'; +import openBrowser from 'react-dev-utils/openBrowser'; -const run = (workingDirectory: string, name: string, args: string[] = []) => { - const child = spawn(name, args, { +const run = ( + workingDirectory: string, + name: string, + args: string[] = [], +): ChildProcess => { + const [stdin, stdout, stderr] = [ + 'inherit' as const, + 'pipe' as const, + 'inherit' as const, + ]; + + const childProcess = spawn(name, args, { cwd: workingDirectory, - stdio: ['inherit', 'inherit', 'inherit'], + stdio: [stdin, stdout, stderr], shell: true, env: { ...process.env, @@ -31,56 +42,87 @@ const run = (workingDirectory: string, name: string, args: string[] = []) => { }, }); - child.once('error', error => { + childProcess.once('error', error => { console.error(error); + childProcess.kill(); }); - child.once('exit', code => { - console.log('exited!', code); + + childProcess.once('exit', () => { + process.exit(0); + }); + + return childProcess; +}; + +const runMkdocsServer = (options?: { + devAddr: string; +}): Promise => { + const devAddr = options?.devAddr ?? '0.0.0.0:8000'; + + return new Promise(resolve => { + const childProcess = run(process.env.PWD!, 'docker', [ + 'run', + '-it', + '-w', + '/content', + '-v', + '$(pwd):/content', + '-p', + '8000:8000', + 'mkdocs:local-dev', + 'serve', + '-a', + devAddr, + ]); + + childProcess.stdout?.on('data', rawData => { + const data = rawData.toString().split('\n')[0]; + console.log('[mkdocs] ', data); + + if (data.includes(`Serving on http://${devAddr}`)) { + resolve(childProcess); + } + }); }); }; const main = (argv: string[]) => { program.name('techdocs-cli').version(version); + program + .command('serve:mkdocs') + .description('Serve a documentation project locally') + .action(() => { + runMkdocsServer().then(() => { + openBrowser('http://localhost:8000'); + }); + }); + program .command('serve') .description('Serve a documentation project locally') .action(() => { - // const techdocsPreviewBundlePath = path.join( - // __dirname, - // '..', - // 'dist', - // 'techdocs-preview-bundle', - // ); + // Mkdocs server + const mkdocsServer = runMkdocsServer(); - // new HTTPServer(techdocsPreviewBundlePath, 3000).serve(); - - run(process.env.PWD!, 'docker', [ - 'run', - '-it', - '-w', - '/content', - '-v', - '$(pwd):/content', - '-p', - '8000:8000', - 'mkdocs:local-dev', - 'serve', - '-a', - '0.0.0.0:8000', - ]); - - const pluginPath = path.join( - require.resolve('@backstage/plugin-techdocs'), - '..', + // Local Backstage Preview + const techdocsPreviewBundlePath = path.join( + __dirname, '..', + 'dist', + 'techdocs-preview-bundle', ); - run( - pluginPath, - path.join(require.resolve('@backstage/cli'), '../../bin/backstage-cli'), - ['plugin:serve'], - ); + const httpServer = new HTTPServer(techdocsPreviewBundlePath, 3000) + .serve() + .catch(err => { + console.error(err); + mkdocsServer.then(childProcess => childProcess.kill()); + }); + + Promise.all([mkdocsServer, httpServer]).then(() => { + openBrowser('http://localhost:3000/docs/local-dev/'); + }); }); program.parse(argv); diff --git a/packages/techdocs-cli/src/lib/httpServer.ts b/packages/techdocs-cli/src/lib/httpServer.ts index 6ca02fc7e9..6e6578840e 100644 --- a/packages/techdocs-cli/src/lib/httpServer.ts +++ b/packages/techdocs-cli/src/lib/httpServer.ts @@ -16,20 +16,64 @@ import serveHandler from 'serve-handler'; import http from 'http'; +import httpProxy from 'http-proxy'; export default class HTTPServer { - constructor(public dir: string, public port: number) {} + proxyEndpoint: string; - serve() { - const server = http.createServer((request, response) => { - return serveHandler(request, response, { - public: this.dir, - trailingSlash: true, - }); + constructor(public dir: string, public port: number) { + this.proxyEndpoint = '/api/'; + } + + private createProxy() { + const proxy = httpProxy.createProxyServer({ + target: 'http://localhost:8000', }); - server.listen(this.port, () => { - console.log('Running at http://localhost:3000'); + return (request: http.IncomingMessage): [httpProxy, string] => { + const [, ...pathChunks] = + request.url?.substring(this.proxyEndpoint.length).split('/') ?? []; + const forwardPath = pathChunks.join('/'); + + return [proxy, forwardPath]; + }; + } + + public async serve(): Promise { + return new Promise((resolve, reject) => { + const proxyHandler = this.createProxy(); + + const server = http.createServer( + (request: http.IncomingMessage, response: http.ServerResponse) => { + if (request.url?.startsWith(this.proxyEndpoint)) { + const [proxy, forwardPath] = proxyHandler(request); + + proxy.on('error', (error: Error) => { + reject(error); + }); + + request.url = forwardPath; + return proxy.web(request, response); + } + + return serveHandler(request, response, { + public: this.dir, + trailingSlash: true, + rewrites: [{ source: '**', destination: 'index.html' }], + }); + }, + ); + + server.listen(this.port, () => { + console.log( + '[techdocs-preview-bundle] Running local version of Backstage at http://localhost:3000', + ); + resolve(server); + }); + + server.on('error', (error: Error) => { + reject(error); + }); }); } } diff --git a/yarn.lock b/yarn.lock index fbdde05008..0fb5368314 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10182,6 +10182,15 @@ http-proxy@^1.17.0: follow-redirects "^1.0.0" requires-port "^1.0.0" +http-proxy@^1.18.1: + version "1.18.1" + resolved "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" + integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== + dependencies: + eventemitter3 "^4.0.0" + follow-redirects "^1.0.0" + requires-port "^1.0.0" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" From 4f709b4a250060097630c379e5dae08b1919b0c3 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:05:14 +0200 Subject: [PATCH 19/41] fix: migrate techdocs-container to packages/ --- .../mkdocs/container => packages/techdocs-container}/Dockerfile | 0 .../techdocs/mkdocs => packages/techdocs-container}/README.md | 0 .../mkdocs => packages/techdocs-container}/mock-docs/.gitignore | 0 .../techdocs-container}/mock-docs/docs/index.md | 0 .../mkdocs => packages/techdocs-container}/mock-docs/mkdocs.yml | 0 .../techdocs-container}/mock-docs/sub-docs/docs/index.md | 0 .../techdocs-container}/mock-docs/sub-docs/mkdocs.yml | 0 .../techdocs-container}/techdocs-core/.gitignore | 0 .../techdocs-container}/techdocs-core/README.md | 0 .../techdocs-container}/techdocs-core/requirements.txt | 0 .../techdocs-container}/techdocs-core/setup.py | 0 .../techdocs-container}/techdocs-core/src/__init__.py | 0 .../techdocs-container}/techdocs-core/src/core.py | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/Dockerfile (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/README.md (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/mock-docs/.gitignore (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/mock-docs/docs/index.md (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/mock-docs/mkdocs.yml (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/mock-docs/sub-docs/docs/index.md (100%) rename {plugins/techdocs/mkdocs => packages/techdocs-container}/mock-docs/sub-docs/mkdocs.yml (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/.gitignore (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/README.md (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/requirements.txt (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/setup.py (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/src/__init__.py (100%) rename {plugins/techdocs/mkdocs/container => packages/techdocs-container}/techdocs-core/src/core.py (100%) diff --git a/plugins/techdocs/mkdocs/container/Dockerfile b/packages/techdocs-container/Dockerfile similarity index 100% rename from plugins/techdocs/mkdocs/container/Dockerfile rename to packages/techdocs-container/Dockerfile diff --git a/plugins/techdocs/mkdocs/README.md b/packages/techdocs-container/README.md similarity index 100% rename from plugins/techdocs/mkdocs/README.md rename to packages/techdocs-container/README.md diff --git a/plugins/techdocs/mkdocs/mock-docs/.gitignore b/packages/techdocs-container/mock-docs/.gitignore similarity index 100% rename from plugins/techdocs/mkdocs/mock-docs/.gitignore rename to packages/techdocs-container/mock-docs/.gitignore diff --git a/plugins/techdocs/mkdocs/mock-docs/docs/index.md b/packages/techdocs-container/mock-docs/docs/index.md similarity index 100% rename from plugins/techdocs/mkdocs/mock-docs/docs/index.md rename to packages/techdocs-container/mock-docs/docs/index.md diff --git a/plugins/techdocs/mkdocs/mock-docs/mkdocs.yml b/packages/techdocs-container/mock-docs/mkdocs.yml similarity index 100% rename from plugins/techdocs/mkdocs/mock-docs/mkdocs.yml rename to packages/techdocs-container/mock-docs/mkdocs.yml diff --git a/plugins/techdocs/mkdocs/mock-docs/sub-docs/docs/index.md b/packages/techdocs-container/mock-docs/sub-docs/docs/index.md similarity index 100% rename from plugins/techdocs/mkdocs/mock-docs/sub-docs/docs/index.md rename to packages/techdocs-container/mock-docs/sub-docs/docs/index.md diff --git a/plugins/techdocs/mkdocs/mock-docs/sub-docs/mkdocs.yml b/packages/techdocs-container/mock-docs/sub-docs/mkdocs.yml similarity index 100% rename from plugins/techdocs/mkdocs/mock-docs/sub-docs/mkdocs.yml rename to packages/techdocs-container/mock-docs/sub-docs/mkdocs.yml diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/.gitignore b/packages/techdocs-container/techdocs-core/.gitignore similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/.gitignore rename to packages/techdocs-container/techdocs-core/.gitignore diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/README.md b/packages/techdocs-container/techdocs-core/README.md similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/README.md rename to packages/techdocs-container/techdocs-core/README.md diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/requirements.txt b/packages/techdocs-container/techdocs-core/requirements.txt similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/requirements.txt rename to packages/techdocs-container/techdocs-core/requirements.txt diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/setup.py b/packages/techdocs-container/techdocs-core/setup.py similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/setup.py rename to packages/techdocs-container/techdocs-core/setup.py diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/src/__init__.py b/packages/techdocs-container/techdocs-core/src/__init__.py similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/src/__init__.py rename to packages/techdocs-container/techdocs-core/src/__init__.py diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/src/core.py b/packages/techdocs-container/techdocs-core/src/core.py similarity index 100% rename from plugins/techdocs/mkdocs/container/techdocs-core/src/core.py rename to packages/techdocs-container/techdocs-core/src/core.py From 3e1efc1e43ced03e9b3d2e1ce20a00f725e4ce8d Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:06:45 +0200 Subject: [PATCH 20/41] fix(.github/workflows): add packages/techdocs-container --- .github/workflows/techdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/techdocs.yml b/.github/workflows/techdocs.yml index 505f63c879..75dc43fba1 100644 --- a/.github/workflows/techdocs.yml +++ b/.github/workflows/techdocs.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - '.github/workflows/techdocs.yml' + - 'packages/techdocs-container/**' - 'packages/techdocs-cli/**' - 'plugins/techdocs/**' - 'plugins/techdocs-backend/**' From decf400b4b9b4e389e7e910fff5b0f2b47f24386 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:11:11 +0200 Subject: [PATCH 21/41] fix(.github/workflows): only validate Dockerfile, not publish (it happens via webhooks) --- .github/workflows/techdocs.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/techdocs.yml b/.github/workflows/techdocs.yml index 75dc43fba1..a313a89f5b 100644 --- a/.github/workflows/techdocs.yml +++ b/.github/workflows/techdocs.yml @@ -29,13 +29,8 @@ jobs: - name: Build and push Docker images uses: docker/build-push-action@v1.1.0 with: - path: plugins/techdocs/mkdocs/container - registry: docker.pkg.github.com - repository: ${{ github.repository }}/mkdocs - username: ${{ github.actor }} - password: ${{ github.token }} - tag_with_ref: true - push: true + path: packages/techdocs-container + push: false # Lint Python code for techdocs-core package - name: prepare python environment From 68d472f702a7c61bc9b4a7a5b6e5d20d628df1ee Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:17:22 +0200 Subject: [PATCH 22/41] docs: updated to packages/techdocs-container --- .github/workflows/techdocs.yml | 2 +- packages/techdocs-cli/README.md | 12 ++++++------ packages/techdocs-container/README.md | 18 +++++++++++++----- .../techdocs-container/techdocs-core/README.md | 2 +- plugins/techdocs/README.md | 4 ++-- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/techdocs.yml b/.github/workflows/techdocs.yml index a313a89f5b..8c13915ebd 100644 --- a/.github/workflows/techdocs.yml +++ b/.github/workflows/techdocs.yml @@ -19,7 +19,7 @@ jobs: python-version: [3.7] env: - TECHDOCS_CORE_PATH: ./plugins/techdocs/mkdocs/container/techdocs-core + TECHDOCS_CORE_PATH: ./packages/techdocs-container/techdocs-core name: Python ${{ matrix.node-version }} on ${{ matrix.os }} steps: diff --git a/packages/techdocs-cli/README.md b/packages/techdocs-cli/README.md index f751497e83..fa862d5e2b 100644 --- a/packages/techdocs-cli/README.md +++ b/packages/techdocs-cli/README.md @@ -16,25 +16,25 @@ yarn serve:mkdocs ## Getting Started -You'll need Docker installed and running to use this. You will also need to build the container located at `plugins/techdocs/mkdocs/container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: +You'll need Docker installed and running to use this. You will also need to build the container located at `/packages/techdocs-container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: ```bash -docker build plugins/techdocs/mkdocs/container -t mkdocs:local-dev +docker build packages/techdocs-container -t mkdocs:local-dev ``` From that point, you can invoke the CLI from any project with a docs folder. Try out our example! ```bash -cd plugins/techdocs/mkdocs/mock-docs +cd packages/techdocs-container/mock-docs npx @techdocs/cli serve ``` ## Local Development -You'll need Docker installed and running to use this. You will also need to build the container located at `plugins/techdocs/mkdocs/container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: +You'll need Docker installed and running to use this. You will also need to build the container located at `packages/techdocs-container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: ```bash -docker build plugins/techdocs/mkdocs/container -t mkdocs:local-dev +docker build packages/techdocs-container -t mkdocs:local-dev ``` Once that is built, you'll need to manually create an `alias` for running the CLI locally: @@ -52,7 +52,7 @@ alias techdocs="[HERE]" From that point, you can invoke `techdocs` from any project with a docs folder. Try out our example! ```bash -cd plugins/techdocs/mkdocs/mock-docs +cd packages/techdocs-container/mock-docs techdocs serve ``` diff --git a/packages/techdocs-container/README.md b/packages/techdocs-container/README.md index 5c2a7725aa..01cbcc1992 100644 --- a/packages/techdocs-container/README.md +++ b/packages/techdocs-container/README.md @@ -1,15 +1,23 @@ -# MkDocs +# techdocs-container -Welcome to MkDocs. This is the TechDocs implementation of MkDocs. +This is the Docker container that powers the creation of static documentation sites that are supported by [TechDocs](https://github.com/spotify/backstage/blob/master/plugins/techdocs). **WIP: This is a work in progress. It is not ready for use yet. Follow our progress on [the Backstage Discord](https://discord.gg/MUpMjP2) under #docs-like-code or on [our GitHub Milestone](https://github.com/spotify/backstage/milestone/15).** -## Getting started +## Getting Started + +Using the TechDocs CLI, we can invoke the latest version of `techdocs-container` via Docker Hub: ```bash -docker build ./container -t mkdocs-container +npx @techdocs/cli serve:container +``` -docker run -w /content -v $(pwd)/mock-docs:/content -p 8000:8000 -it mkdocs-container serve -a 0.0.0.0:8000 +## Local Development + +```bash +docker build ./container -t techdocs-container + +docker run -w /content -v $(pwd)/mock-docs:/content -p 8000:8000 -it techdocs-container serve -a 0.0.0.0:8000 ``` Then open up `http://localhost:8000` on your local machine. diff --git a/packages/techdocs-container/techdocs-core/README.md b/packages/techdocs-container/techdocs-core/README.md index 143acf3c3b..12b3acb1e2 100644 --- a/packages/techdocs-container/techdocs-core/README.md +++ b/packages/techdocs-container/techdocs-core/README.md @@ -33,7 +33,7 @@ You'll then have the `techdocs-core` package available to use in Mkdocs and `pip In the parent `Dockerfile` we add this folder to the build and install the package locally in the container. In the future, we'll probably move away from this approach and have it download directly from a Python registry (and this folder will publish to one). -See the `README.md` located in the `mkdocs/` folder for more details on how to build and run the Docker container. +See the `README.md` located in the `techdocs-container/` folder for more details on how to build and run the Docker container. ## Linting diff --git a/plugins/techdocs/README.md b/plugins/techdocs/README.md index 8ac34f1943..fb30f463dd 100644 --- a/plugins/techdocs/README.md +++ b/plugins/techdocs/README.md @@ -20,7 +20,7 @@ It is only meant for local development, and the setup for it can be found inside ### Custom Storage URL -TechDocs currently reads a static HTML file, generated by Mkdocs (see our `plugins/techdocs/mkdocs/container` folder for more documentation) and stored on an external server, and loads that into Backstage. By default, we have set up a mock server with some example documentation sites over in Google Cloud Storage: +TechDocs currently reads a static HTML file, generated by Mkdocs (see our `packages/techdocs-container` folder for more documentation) and stored on an external server, and loads that into Backstage. By default, we have set up a mock server with some example documentation sites over in Google Cloud Storage: ```md # Base URL @@ -36,7 +36,7 @@ https://techdocs-mock-sites.storage.googleapis.com/mkdocs/index.html https://techdocs-mock-sites.storage.googleapis.com/backstage-microsite/index.html ``` -Using your own setup (or ours which is being worked on as of Q3 2020), you can point it to your own server with your own hosted documentation sites. The only requirement is that it the output is from [Mkdocs](https://mkdocs.org) with the Material theme. You can always use our documentation generation tool located at `plugins/techdocs/mkdocs/container` for easy setup. +Using your own setup (or ours which is being worked on as of Q3 2020), you can point it to your own server with your own hosted documentation sites. The only requirement is that it the output is from [Mkdocs](https://mkdocs.org) with the Material theme. You can always use our documentation generation tool located at `/packages/techdocs-container` for easy setup. To point TechDocs to your own server, simply update the `techdocs.storageUrl` value in your `app-config.yaml` file or set the environment variable `APP_CONFIG_techdocs_storageUrl` in your application: From f5a5fac7cffed131bd19374a0e264eceab933f89 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:18:16 +0200 Subject: [PATCH 23/41] docs: correction --- .github/workflows/techdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/techdocs.yml b/.github/workflows/techdocs.yml index 8c13915ebd..491b5693de 100644 --- a/.github/workflows/techdocs.yml +++ b/.github/workflows/techdocs.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/checkout@v2 # Build Docker Image - - name: Build and push Docker images + - name: Build Docker image uses: docker/build-push-action@v1.1.0 with: path: packages/techdocs-container From 449c962aea15d5e0d76928c0e8f14a388f3735fa Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:21:51 +0200 Subject: [PATCH 24/41] docs: removed prefix slash --- plugins/techdocs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/README.md b/plugins/techdocs/README.md index fb30f463dd..b763679a1c 100644 --- a/plugins/techdocs/README.md +++ b/plugins/techdocs/README.md @@ -36,7 +36,7 @@ https://techdocs-mock-sites.storage.googleapis.com/mkdocs/index.html https://techdocs-mock-sites.storage.googleapis.com/backstage-microsite/index.html ``` -Using your own setup (or ours which is being worked on as of Q3 2020), you can point it to your own server with your own hosted documentation sites. The only requirement is that it the output is from [Mkdocs](https://mkdocs.org) with the Material theme. You can always use our documentation generation tool located at `/packages/techdocs-container` for easy setup. +Using your own setup (or ours which is being worked on as of Q3 2020), you can point it to your own server with your own hosted documentation sites. The only requirement is that it the output is from [Mkdocs](https://mkdocs.org) with the Material theme. You can always use our documentation generation tool located at `packages/techdocs-container` for easy setup. To point TechDocs to your own server, simply update the `techdocs.storageUrl` value in your `app-config.yaml` file or set the environment variable `APP_CONFIG_techdocs_storageUrl` in your application: From 2331be17c74cf4133a98cc51c3b7402ea137a84a Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 16:41:05 +0200 Subject: [PATCH 25/41] fix: set techdocs plugin to public (#1585) --- plugins/techdocs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 98fbac1492..4dcbeec668 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -4,7 +4,7 @@ "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", - "private": true, + "private": false, "publishConfig": { "access": "public", "main": "dist/index.esm.js", From 066ba28acf214ebee3277457af6662917e23db21 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 17:58:34 +0200 Subject: [PATCH 26/41] docs(techdocs-cli): update to use from npm registry (#1586) --- packages/techdocs-cli/README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/packages/techdocs-cli/README.md b/packages/techdocs-cli/README.md index fa862d5e2b..960574db37 100644 --- a/packages/techdocs-cli/README.md +++ b/packages/techdocs-cli/README.md @@ -31,29 +31,15 @@ npx @techdocs/cli serve ## Local Development -You'll need Docker installed and running to use this. You will also need to build the container located at `packages/techdocs-container` under the tag `mkdocs:local-dev`, as you can see in the commands from below: +You'll need Docker installed and running to use this. You will also need to build the container located at `packages/techdocs-container` under the tag `mkdocs:local-dev` (for now until we deploy the container to a centralized Docker registry), as you can see in the commands from below: ```bash docker build packages/techdocs-container -t mkdocs:local-dev ``` -Once that is built, you'll need to manually create an `alias` for running the CLI locally: - -```bash -cd packages/techdocs-cli -echo "$(pwd)/bin/techdocs" - -# Copy the value from above and add it in [HERE] below -# For more convenience, add it to your ~/.zshrc or ~/.bash_profile -# otherwise you'll lose it when you open a new Terminal -alias techdocs="[HERE]" -``` - -From that point, you can invoke `techdocs` from any project with a docs folder. Try out our example! - ```bash cd packages/techdocs-container/mock-docs -techdocs serve +npx techdocs serve ``` You should have a `localhost:3000` serving TechDocs in Backstage, as well as `localhost:8000` serving Mkdocs (which won't open up and be exposed to the user). From 3026db08f06f6464533f4e39bd1bad92113a7aa3 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Fri, 10 Jul 2020 18:17:29 +0200 Subject: [PATCH 27/41] fix: remove @backstage/plugin-techdocs as dep --- packages/techdocs-cli/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/techdocs-cli/package.json b/packages/techdocs-cli/package.json index 97c5cbe049..2e06c4a71e 100644 --- a/packages/techdocs-cli/package.json +++ b/packages/techdocs-cli/package.json @@ -45,7 +45,6 @@ }, "dependencies": { "@backstage/cli": "^0.1.1-alpha.13", - "@backstage/plugin-techdocs": "^0.1.1-alpha.13", "chalk": "^4.1.0", "commander": "^5.1.0", "fs-extra": "^9.0.1", From 2fd5d2ffa50e606f8beaba5e51dfaf5db73b697f Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Fri, 10 Jul 2020 16:58:59 +0200 Subject: [PATCH 28/41] feat: make it work Co-authored-by: Ivan Shmidt --- packages/app/src/apis.ts | 4 +- .../src/api/GithubActionsApi.ts | 12 ++- .../src/api/GithubActionsClient.ts | 73 +++++---------- .../src/api/MockGithubActionsClient.ts | 40 -------- plugins/github-actions/src/api/index.test.ts | 44 --------- .../BuildDetailsPage/BuildDetailsPage.tsx | 19 ++-- .../BuildInfoCard/BuildInfoCard.tsx | 91 ++++++++++--------- .../BuildListPage/BuildListPage.tsx | 29 +++--- plugins/github-actions/src/plugin.ts | 2 +- 9 files changed, 110 insertions(+), 204 deletions(-) delete mode 100644 plugins/github-actions/src/api/MockGithubActionsClient.ts delete mode 100644 plugins/github-actions/src/api/index.test.ts diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index df17886216..e80f031bf3 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -59,7 +59,7 @@ import { scaffolderApiRef, ScaffolderApi } from '@backstage/plugin-scaffolder'; import { rollbarApiRef, RollbarClient } from '@backstage/plugin-rollbar'; import { - MockGithubActionsClient, + GithubActionsClient, githubActionsApiRef, } from '@backstage/plugin-github-actions'; @@ -79,7 +79,7 @@ export const apis = (config: ConfigApi) => { builder.add(storageApiRef, WebStorage.create({ errorApi })); builder.add(circleCIApiRef, new CircleCIApi()); - builder.add(githubActionsApiRef, new MockGithubActionsClient()); + builder.add(githubActionsApiRef, new GithubActionsClient()); builder.add(featureFlagsApiRef, new FeatureFlags()); builder.add(lighthouseApiRef, new LighthouseRestApi('http://localhost:3003')); diff --git a/plugins/github-actions/src/api/GithubActionsApi.ts b/plugins/github-actions/src/api/GithubActionsApi.ts index c47cff7eda..dd75f34bf7 100644 --- a/plugins/github-actions/src/api/GithubActionsApi.ts +++ b/plugins/github-actions/src/api/GithubActionsApi.ts @@ -16,8 +16,6 @@ import { createApiRef } from '@backstage/core'; import { Build, BuildDetails } from './types'; -import { Entity } from '@backstage/catalog-model'; - export const githubActionsApiRef = createApiRef({ id: 'plugin.githubactions.service', @@ -25,6 +23,14 @@ export const githubActionsApiRef = createApiRef({ }); export type GithubActionsApi = { - listBuilds: (entity: Entity, token: Promise) => Promise; + listBuilds: ({ + owner, + repo, + token, + }: { + owner: string; + repo: string; + token: string; + }) => Promise; getBuild: (buildUri: string, token: Promise) => Promise; }; diff --git a/plugins/github-actions/src/api/GithubActionsClient.ts b/plugins/github-actions/src/api/GithubActionsClient.ts index 0c9a98e57c..93ace7fa38 100644 --- a/plugins/github-actions/src/api/GithubActionsClient.ts +++ b/plugins/github-actions/src/api/GithubActionsClient.ts @@ -16,26 +16,34 @@ import { GithubActionsApi } from './GithubActionsApi'; import { Build, BuildDetails, BuildStatus, WorkflowRun } from './types'; -import { Entity } from '@backstage/catalog-model'; + +const statusToBuildStatus: { [status: string]: BuildStatus } = { + success: BuildStatus.Success, + failure: BuildStatus.Failure, + pending: BuildStatus.Pending, + running: BuildStatus.Running, + in_progress: BuildStatus.Running, + completed: BuildStatus.Success, +}; + +const conclusionToStatus = (conslusion: string): BuildStatus => + statusToBuildStatus[conslusion] ?? BuildStatus.Null; export class GithubActionsClient implements GithubActionsApi { - async listBuilds(entity: Entity, token: Promise): Promise { - // ### Feedback request ### - // I asumed the following: (maybe not the best. Ideally this should come from the link to the component.yaml file) - // entity.metadata.namespace => org name - // entity.metadata.name => repo name - // entityUri -> entity:spotify:backstage - - let url: string; - if (entity.metadata.name !== '') { - url = `https://api.github.com/repos/${entity.metadata.namespace}/${entity.metadata.name}/runs`; - } else { - url = 'https://api.github.com/repos/spotify/backstage/actions/runs'; - } + async listBuilds({ + owner, + repo, + token, + }: { + owner: string; + repo: string; + token: string; + }): Promise { + const url = `https://api.github.com/repos/${owner}/${repo}/actions/runs`; const response = await fetch(url, { headers: new Headers({ - Authorization: `Bearer ${await token}`, + Authorization: `Bearer ${token}`, }), }); @@ -67,24 +75,7 @@ export class GithubActionsClient implements GithubActionsApi { }; transData.commitId = String(element.head_commit.id); transData.branch = element.head_branch; - - // ### Feedback request ### - // TODO: I am not sure about this part. Looks ugly. Maybe there is a better way of doing this. - if (element.conclusion === 'success') { - transData.status = BuildStatus.Success; - } else if (element.conclusion === 'failure') { - transData.status = BuildStatus.Failure; - } else if (element.conclusion === 'pending') { - transData.status = BuildStatus.Pending; - } else if (element.conclusion === 'running') { - transData.status = BuildStatus.Running; - } else { - if (element.status === 'in_progress') { - transData.status = BuildStatus.Running; - } else { - transData.status = BuildStatus.Null; - } - } + transData.status = conclusionToStatus(element.conclusion); transData.message = element.head_commit.message; transData.uri = element.url; endData[index] = transData; @@ -130,21 +121,7 @@ export class GithubActionsClient implements GithubActionsApi { dataBlank.build.branch = newData.head_branch; dataBlank.build.commitId = newData.head_commit.id; dataBlank.build.message = newData.head_commit.message; - - // ### Feedback request ### - // TODO: I am not sure about this part. Look ugly. Maybe there is a better way of doing this. - if (newData.status === 'completed') { - dataBlank.build.status = BuildStatus.Success; - } else if (newData.status === 'in_progress') { - dataBlank.build.status = BuildStatus.Running; - } else if (newData.status === 'pending') { - dataBlank.build.status = BuildStatus.Pending; - } else if (newData.status === 'failure') { - dataBlank.build.status = BuildStatus.Failure; - } else { - dataBlank.build.status = BuildStatus.Null; - } - + dataBlank.build.status = conclusionToStatus(newData.status); dataBlank.build.uri = newData.url; dataBlank.logUrl = newData.logs_url; dataBlank.overviewUrl = newData.html_url; diff --git a/plugins/github-actions/src/api/MockGithubActionsClient.ts b/plugins/github-actions/src/api/MockGithubActionsClient.ts deleted file mode 100644 index 89ecad3931..0000000000 --- a/plugins/github-actions/src/api/MockGithubActionsClient.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubActionsApi } from './GithubActionsApi'; -import { Build, BuildDetails, BuildStatus } from './types'; - -export class MockGithubActionsClient implements GithubActionsApi { - async listBuilds(): Promise { - return []; - } - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async getBuild(): Promise { - return { - build: { - commitId: 'TODO', - branch: 'TODO', - uri: 'TODO', - status: BuildStatus.Running, - message: 'TODO', - }, - author: 'TODO', - logUrl: 'TODO', - overviewUrl: 'TODO', - }; - } -} diff --git a/plugins/github-actions/src/api/index.test.ts b/plugins/github-actions/src/api/index.test.ts deleted file mode 100644 index feff353115..0000000000 --- a/plugins/github-actions/src/api/index.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubActionsClient } from './GithubActionsClient'; -import { BuildStatus } from './types'; - -describe('Github Actions API', () => { - let client: GithubActionsClient; - beforeEach(() => { - client = new GithubActionsClient(); - }); - describe('Mock client', () => { - it('gets a list of builds by a project id', async () => { - await expect(client.listBuilds()).resolves.toEqual([]); - }); - it('gets a build info by its id', async () => { - await expect(client.getBuild()).resolves.toEqual({ - build: { - commitId: 'TODO', - branch: 'TODO', - uri: 'TODO', - status: BuildStatus.Running, - message: 'TODO', - }, - author: 'TODO', - logUrl: 'TODO', - overviewUrl: 'TODO', - }); - }); - }); -}); diff --git a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx index 84e9332837..707c670f40 100644 --- a/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx +++ b/plugins/github-actions/src/components/BuildDetailsPage/BuildDetailsPage.tsx @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Link } from '@backstage/core'; import { Button, ButtonGroup, @@ -30,10 +29,10 @@ import { Typography, } from '@material-ui/core'; import React from 'react'; -import { useParams } from 'react-router-dom'; +import { useLocation } from 'react-router-dom'; import { useAsync } from 'react-use'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; -import { useApi, githubAuthApiRef } from '@backstage/core-api'; +import { Link, useApi, githubAuthApiRef } from '@backstage/core'; import { githubActionsApiRef } from '../../api'; const useStyles = makeStyles(theme => ({ @@ -55,8 +54,12 @@ export const BuildDetailsPage = () => { const token = githubApi.getAccessToken('repo'); const classes = useStyles(); - const { buildUri } = useParams(); - const status = useAsync(() => api.getBuild(buildUri, token), [buildUri]); + const location = useLocation(); + const status = useAsync( + () => + api.getBuild(decodeURIComponent(location.search.split('uri=')[1]), token), + [location.search], + ); if (status.loading) { return ; @@ -73,7 +76,7 @@ export const BuildDetailsPage = () => { return (
- + < @@ -127,12 +130,12 @@ export const BuildDetailsPage = () => { > {details?.overviewUrl && ( )} {details?.logUrl && ( )} diff --git a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx index 2e4cbb03bd..cbbc1da339 100644 --- a/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx +++ b/plugins/github-actions/src/components/BuildInfoCard/BuildInfoCard.tsx @@ -14,7 +14,6 @@ * limitations under the License. */ -import { Link } from '@backstage/core'; import { LinearProgress, makeStyles, @@ -29,7 +28,7 @@ import React from 'react'; import { useAsync } from 'react-use'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; import { githubActionsApiRef } from '../../api'; -import { useApi } from '@backstage/core-api'; +import { Link, useApi, githubAuthApiRef } from '@backstage/core'; const useStyles = makeStyles(theme => ({ root: { @@ -40,63 +39,69 @@ const useStyles = makeStyles(theme => ({ }, })); -export const BuildInfoCard = () => { - const classes = useStyles(); +const BuildInfoCardContent = () => { const api = useApi(githubActionsApiRef); - const status = useAsync(() => api.listBuilds('entity:spotify:backstage')); + const githubApi = useApi(githubAuthApiRef); - let content: JSX.Element; + const status = useAsync(async () => { + const token = await githubApi.getAccessToken('repo'); + return api.listBuilds({ owner: 'spotify', repo: 'backstage', token }); + }); if (status.loading) { - content = ; + return ; } else if (status.error) { - content = ( + return ( Failed to load builds, {status.error.message} ); - } else { - const [build] = - status.value?.filter(({ branch }) => branch === 'master') ?? []; - - content = ( -
- - - - Message - - - - {build?.message} - - - - - - Commit ID - - {build?.commitId} - - - - Status - - - - - - -
- ); } + const [build] = + status.value?.filter(({ branch }) => branch === 'master') ?? []; + + return ( + + + + + Message + + + + {build?.message} + + + + + + Commit ID + + {build?.commitId} + + + + Status + + + + + + +
+ ); +}; + +export const BuildInfoCard = () => { + const classes = useStyles(); + return (
Master Build - {content} +
); }; diff --git a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx index 85568c8c07..236bc182ef 100644 --- a/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx +++ b/plugins/github-actions/src/components/BuildListPage/BuildListPage.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Link } from '@backstage/core'; +import { Link, useApi, githubAuthApiRef } from '@backstage/core'; import { LinearProgress, makeStyles, @@ -29,13 +29,10 @@ import { Tooltip, Typography, } from '@material-ui/core'; -import React, { FC } from 'react'; +import React from 'react'; import { useAsync } from 'react-use'; import { BuildStatusIndicator } from '../BuildStatusIndicator'; -import { githubActionsApiRef } from '../../api'; -import { useApi, githubAuthApiRef } from '@backstage/core-api'; -import { Entity } from '@backstage/catalog-model'; - +import { githubActionsApiRef, Build } from '../../api'; const LongText = ({ text, max }: { text: string; max: number }) => { if (text.length < max) { @@ -57,14 +54,15 @@ const useStyles = makeStyles(theme => ({ }, })); -const PageContents: FC<{ entity: Entity }> = ({ entity }) => { +const PageContents = ({ owner, repo }: { owner: string; repo: string }) => { const api = useApi(githubActionsApiRef); const githubApi = useApi(githubAuthApiRef); - const token = githubApi.getAccessToken('repo'); - const { loading, error, value } = useAsync(() => - api.listBuilds(entity, token), - ); + const { loading, error, value } = useAsync(async () => { + const token = await githubApi.getAccessToken('repo'); + + return api.listBuilds({ owner, repo, token }); + }, [githubApi, owner, repo]); if (loading) { return ; @@ -90,7 +88,7 @@ const PageContents: FC<{ entity: Entity }> = ({ entity }) => { - {value!.map(build => ( + {value?.map((build: Build) => ( @@ -101,7 +99,7 @@ const PageContents: FC<{ entity: Entity }> = ({ entity }) => { - + @@ -120,14 +118,15 @@ const PageContents: FC<{ entity: Entity }> = ({ entity }) => { ); }; -export const BuildListPage: FC<{ entity: Entity }> = ({ entity }) => { +export const BuildListPage = () => { const classes = useStyles(); + return (
CI/CD Builds - +
); }; diff --git a/plugins/github-actions/src/plugin.ts b/plugins/github-actions/src/plugin.ts index 638c8d2c8c..8897fe53db 100644 --- a/plugins/github-actions/src/plugin.ts +++ b/plugins/github-actions/src/plugin.ts @@ -24,7 +24,7 @@ export const rootRouteRef = createRouteRef({ title: 'GitHub Actions', }); export const buildRouteRef = createRouteRef({ - path: '/github-actions/builds/:buildUri', + path: '/github-actions/builds', title: 'GitHub Actions Build', }); From f39abc0fe7c5b29ad1e34a31065bdb7b5571b0d5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 11 Jul 2020 20:22:12 +0200 Subject: [PATCH 29/41] package: enforce node 12 --- package.json | 2 +- packages/backend/package.json | 2 +- packages/cli/templates/default-app/package.json.hbs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 8b5a7a24c8..cc45d541d4 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=12.0.0" + "node": "12" }, "scripts": { "start": "yarn workspace example-app start", diff --git a/packages/backend/package.json b/packages/backend/package.json index 2e9cd1ff49..097c6446e9 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -6,7 +6,7 @@ "private": true, "license": "Apache-2.0", "engines": { - "node": ">=12" + "node": "12" }, "scripts": { "build": "backstage-cli backend:build", diff --git a/packages/cli/templates/default-app/package.json.hbs b/packages/cli/templates/default-app/package.json.hbs index c70e00e4d0..1414ef341a 100644 --- a/packages/cli/templates/default-app/package.json.hbs +++ b/packages/cli/templates/default-app/package.json.hbs @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "engines": { - "node": ">=12.0.0" + "node": "12" }, "scripts": { "start": "yarn workspace app start", From 423ff518f978ea60187be8f181ba343aa8b57efd Mon Sep 17 00:00:00 2001 From: Nikita Nek Dudnik Date: Mon, 13 Jul 2020 10:29:35 +0200 Subject: [PATCH 30/41] fix: correct response status usage --- plugins/github-actions/src/api/GithubActionsClient.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/github-actions/src/api/GithubActionsClient.ts b/plugins/github-actions/src/api/GithubActionsClient.ts index 93ace7fa38..58d7280c1f 100644 --- a/plugins/github-actions/src/api/GithubActionsClient.ts +++ b/plugins/github-actions/src/api/GithubActionsClient.ts @@ -47,11 +47,11 @@ export class GithubActionsClient implements GithubActionsApi { }), }); - if (response.status > 200) { + if (!response.ok) { return [ { commitId: 'Error', - message: 'ResponseCode > 200', + message: 'Response status is not OK', branch: 'Error', status: BuildStatus.Failure, uri: 'Error', @@ -109,7 +109,7 @@ export class GithubActionsClient implements GithubActionsApi { overviewUrl: '', }; - if (response.status > 200) { + if (!response.ok) { return dataBlank; } From 87698e2fe6f8df2c9c99b5b8cd8802972a743feb Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 10:53:30 +0200 Subject: [PATCH 31/41] chore(scaffolder): making the templater field mandatory --- .../src/kinds/TemplateEntityV1alpha1.test.ts | 10 ++++++++-- .../catalog-model/src/kinds/TemplateEntityV1alpha1.ts | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts index 52951c793a..ce9ed8bedd 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts @@ -32,7 +32,8 @@ describe('TemplateEntityV1alpah1', () => { name: 'test', }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', schema: { $schema: 'http://json-schema.org/draft-07/schema#', required: ['storePath', 'owner'], @@ -78,7 +79,7 @@ describe('TemplateEntityV1alpah1', () => { await expect(policy.enforce(entity)).rejects.toThrow(/type/); }); - it('acceptps any other type', async () => { + it('accepts any other type', async () => { (entity as any).spec.type = 'hallo'; await expect(policy.enforce(entity)).resolves.toBe(entity); }); @@ -87,4 +88,9 @@ describe('TemplateEntityV1alpah1', () => { (entity as any).spec.type = ''; await expect(policy.enforce(entity)).rejects.toThrow(/type/); }); + + it('rejects missing templater', async () => { + (entity as any).spec.templater = ''; + await expect(policy.enforce(entity)).rejects.toThrow(/templater/); + }); }); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts index c2f071b7c3..8aa79d57db 100644 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts @@ -26,6 +26,7 @@ export interface TemplateEntityV1alpha1 extends Entity { kind: typeof KIND; spec: { type: string; + templater: string; path?: string; schema: JSONSchema; }; @@ -43,6 +44,7 @@ export class TemplateEntityV1alpha1Policy implements EntityPolicy { type: yup.string().required().min(1), path: yup.string(), schema: yup.object().required(), + templater: yup.string().required(), }) .required(), }); From 021b03669a712afacb210a3ec13a2711dd9be1ed Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 13 Jul 2020 11:19:20 +0200 Subject: [PATCH 32/41] docs(proxy): add link to plugin creation docs --- docs/getting-started/structure-of-a-plugin.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/getting-started/structure-of-a-plugin.md b/docs/getting-started/structure-of-a-plugin.md index fcfb2c0d46..a277628b6c 100644 --- a/docs/getting-started/structure-of-a-plugin.md +++ b/docs/getting-started/structure-of-a-plugin.md @@ -95,4 +95,13 @@ There are two things needed for a Backstage app to start making use of a plugin. Luckily these two steps happen automatically when you create a plugin with the Backstage CLI. +## Talking to the outside world + +If your plugin needs to communicate with services outside the backstage +environment you will probably face challenges like CORS policies and/or +backend-side authorization. To smooth this process out you can use proxy - +either the one you already have (like nginx/haproxy/etc) or the proxy-backend +plugin that we provide for the backstage backend. +[Read more](../../plugins/proxy-backend/README.md) + [Back to Getting Started](README.md) From ac27ab635bf7a2d01cbd760e752c52da106e7ed7 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 13 Jul 2020 11:21:43 +0200 Subject: [PATCH 33/41] fix(proxy): versions, private --- plugins/proxy-backend/package.json | 10 +--------- plugins/proxy-backend/src/service/router.ts | 6 +----- yarn.lock | 17 ++++++++--------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/plugins/proxy-backend/package.json b/plugins/proxy-backend/package.json index 521f190eba..9a9352ba83 100644 --- a/plugins/proxy-backend/package.json +++ b/plugins/proxy-backend/package.json @@ -4,7 +4,6 @@ "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", - "private": true, "publishConfig": { "access": "public", "main": "dist/index.cjs.js", @@ -24,10 +23,9 @@ "@backstage/config": "^0.1.1-alpha.13", "@backstage/config-loader": "^0.1.1-alpha.13", "@types/express": "^4.17.6", - "@types/http-proxy-middleware": "^0.19.3", "express": "^4.17.1", "express-promise-router": "^3.0.3", - "http-proxy-middleware": "^1.0.4", + "http-proxy-middleware": "^0.19.1", "morgan": "^1.10.0", "node-fetch": "^2.6.0", "uuid": "^8.0.0", @@ -45,12 +43,6 @@ "jest-fetch-mock": "^3.0.3", "supertest": "^4.0.2" }, - "workspaces": { - "nohoist": [ - "http-proxy-middleware", - "@types/http-proxy-middleware" - ] - }, "files": [ "dist" ] diff --git a/plugins/proxy-backend/src/service/router.ts b/plugins/proxy-backend/src/service/router.ts index a8fd70c10d..2c8af574bd 100644 --- a/plugins/proxy-backend/src/service/router.ts +++ b/plugins/proxy-backend/src/service/router.ts @@ -14,11 +14,10 @@ * limitations under the License. */ -import { errorHandler } from '@backstage/backend-common'; import { Config } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; -import { createProxyMiddleware } from 'http-proxy-middleware'; +import createProxyMiddleware from 'http-proxy-middleware'; import { Logger } from 'winston'; export interface RouterOptions { @@ -30,13 +29,10 @@ export async function createRouter( options: RouterOptions, ): Promise { const router = Router(); - router.use(express.json()); - const proxyConfig = options.config.get('proxy') ?? {}; Object.entries(proxyConfig).forEach(([route, proxyRouteConfig]) => { router.use(createProxyMiddleware(route, proxyRouteConfig)); }); - router.use(errorHandler()); return router; } diff --git a/yarn.lock b/yarn.lock index 35b06e3d21..ca6bdad85b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3599,7 +3599,7 @@ resolved "https://registry.npmjs.org/@types/http-errors/-/http-errors-1.6.3.tgz#619a55768eab98299e8f76747339f3373f134e69" integrity sha512-4KCE/agIcoQ9bIfa4sBxbZdnORzRjIw8JNQPLfqoNv7wQl/8f8mRbW68Q8wBsQFoJkPUHGlQYZ9sqi5WpfGSEQ== -"@types/http-proxy-middleware@*", "@types/http-proxy-middleware@^0.19.3": +"@types/http-proxy-middleware@*": version "0.19.3" resolved "https://registry.npmjs.org/@types/http-proxy-middleware/-/http-proxy-middleware-0.19.3.tgz#b2eb96fbc0f9ac7250b5d9c4c53aade049497d03" integrity sha512-lnBTx6HCOUeIJMLbI/LaL5EmdKLhczJY5oeXZpX/cXE4rRqb3RmV7VcMpiEfYkmTjipv3h7IAyIINe4plEv7cA== @@ -10173,16 +10173,15 @@ http-proxy-middleware@0.19.1: lodash "^4.17.11" micromatch "^3.1.10" -http-proxy-middleware@^1.0.4: - version "1.0.4" - resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-1.0.4.tgz#425ea177986a0cda34f9c81ec961c719adb6c2a9" - integrity sha512-8wiqujNWlsZNbeTSSWMLUl/u70xbJ5VYRwPR8RcAbvsNxzAZbgwLzRvT96btbm3fAitZUmo5i8LY6WKGyHDgvA== +http-proxy-middleware@^0.19.1: + version "0.19.2" + resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.19.2.tgz#ee73dcc8348165afefe8de2ff717751d181608ee" + integrity sha512-aYk1rTKqLTus23X3L96LGNCGNgWpG4cG0XoZIT1GUPhhulEHX/QalnO6Vbo+WmKWi4AL2IidjuC0wZtbpg0yhQ== dependencies: - "@types/http-proxy" "^1.17.4" http-proxy "^1.18.1" - is-glob "^4.0.1" - lodash "^4.17.15" - micromatch "^4.0.2" + is-glob "^4.0.0" + lodash "^4.17.11" + micromatch "^3.1.10" http-proxy@^1.17.0: version "1.18.0" From f1b479c772e17ca70220c1574cc5197e82434d86 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Mon, 13 Jul 2020 11:22:08 +0200 Subject: [PATCH 34/41] fix(proxy): mount on /proxy --- packages/app/src/apis.ts | 5 ++++- packages/backend/src/index.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/app/src/apis.ts b/packages/app/src/apis.ts index 9ab4a125a6..3d8a381635 100644 --- a/packages/app/src/apis.ts +++ b/packages/app/src/apis.ts @@ -74,7 +74,10 @@ export const apis = (config: ConfigApi) => { ); builder.add(storageApiRef, WebStorage.create({ errorApi })); - builder.add(circleCIApiRef, new CircleCIApi(`${backendUrl}/circleci/api`)); + builder.add( + circleCIApiRef, + new CircleCIApi(`${backendUrl}/proxy/circleci/api`), + ); builder.add(featureFlagsApiRef, new FeatureFlags()); builder.add(lighthouseApiRef, new LighthouseRestApi('http://localhost:3003')); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 7c9de7c3bb..ea5142da19 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -86,7 +86,7 @@ async function main() { .addRouter('/auth', await auth(authEnv)) .addRouter('/identity', await identity(identityEnv)) .addRouter('/techdocs', await techdocs(techdocsEnv)) - .addRouter('/', await proxy(proxyEnv)); + .addRouter('/proxy', await proxy(proxyEnv)); await service.start().catch(err => { console.log(err); From 46c60cb54a36d43d45ec707045aeb9a90ac576b4 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 11:53:10 +0200 Subject: [PATCH 35/41] chore(scaffolder): Updating typescript issues by making the templater a required key --- .../scaffolder/stages/prepare/github.test.ts | 3 +- .../scaffolder/stages/prepare/helpers.test.ts | 15 ++- .../stages/prepare/preparers.test.ts | 6 +- .../src/scaffolder/stages/publish/index.ts | 1 + .../scaffolder/stages/templater/helpers.ts | 16 +++ .../src/scaffolder/stages/templater/index.ts | 1 + .../stages/templater/templaters.test.ts | 125 ++++++++++++++++++ .../scaffolder/stages/templater/templaters.ts | 45 +++++++ .../src/scaffolder/stages/templater/types.ts | 15 ++- 9 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 6259023476..a61e5de867 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -47,7 +47,8 @@ describe('GitHubPreparer', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts index c5380fa5d1..8b174a1edf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts @@ -39,7 +39,8 @@ describe('Helpers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', @@ -86,7 +87,8 @@ describe('Helpers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', @@ -131,7 +133,8 @@ describe('Helpers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', @@ -177,7 +180,8 @@ describe('Helpers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', @@ -221,7 +225,8 @@ describe('Helpers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts index 9807c63c08..b79adffb18 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts @@ -35,8 +35,9 @@ describe('Preparers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + templater: 'cookiecutter', path: '.', + type: 'website', schema: { $schema: 'http://json-schema.org/draft-07/schema#', required: ['storePath', 'owner'], @@ -88,7 +89,8 @@ describe('Preparers', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: '.', schema: { $schema: 'http://json-schema.org/draft-07/schema#', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts index dcfd2c9c34..38a96480c2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export * from './github'; +export * from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index 9ef85224d4..0267d5eeab 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -16,6 +16,8 @@ import { Writable, PassThrough } from 'stream'; import Docker from 'dockerode'; import fs from 'fs'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { InputError } from '@backstage/backend-common'; export type RunDockerContainerOptions = { imageName: string; @@ -26,6 +28,20 @@ export type RunDockerContainerOptions = { dockerClient: Docker; }; +/** + * Gets the templater key to use for templating from the entity + * @param entity Template entity + */ +export const getTemplaterKey = (entity: TemplateEntityV1alpha1): string => { + const { templater } = entity.spec; + + if (!templater) { + throw new InputError('Template does not have a required templating key'); + } + + return templater; +}; + /** * * @param options the options object diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts index 0813d0ab86..56358f5c5b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts @@ -16,3 +16,4 @@ export * from './cookiecutter'; export * from './types'; export * from './helpers'; +export * from './templaters'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts new file mode 100644 index 0000000000..bf2f2ed9c6 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Templaters } from '.'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { CookieCutter } from './cookiecutter'; + +describe('Templaters', () => { + const mockTemplate: TemplateEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: { + 'backstage.io/managed-by-location': + 'file:/Users/blam/dev/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml', + }, + name: 'react-ssr-template', + title: 'React SSR Template', + description: + 'Next.js application skeleton for creating isomorphic web applications.', + uid: '7357f4c5-aa58-4a1e-9670-18931eef771f', + etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw', + generation: 1, + }, + spec: { + templater: 'cookiecutter', + path: '.', + type: 'website', + schema: { + $schema: 'http://json-schema.org/draft-07/schema#', + required: ['storePath', 'owner'], + properties: { + owner: { + type: 'string', + title: 'Owner', + description: 'Who is going to own this component', + }, + storePath: { + type: 'string', + title: 'Store path', + description: 'GitHub store path in org/repo format', + }, + }, + }, + }, + }; + it('should throw an error when the templater is not registered', () => { + const templaters = new Templaters(); + + expect(() => templaters.get(mockTemplate)).toThrow( + expect.objectContaining({ + message: 'No templater registered for template: "cookiecutter"', + }), + ); + }); + it('should return the correct templater when the templater matches', () => { + const templaters = new Templaters(); + const templater = new CookieCutter(); + + templaters.register('cookiecutter', templater); + + expect(templaters.get(mockTemplate)).toBe(templater); + }); + + it('should throw an error if the templater does not exist in the entity', () => { + const brokenTemplate: TemplateEntityV1alpha1 = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: {}, + name: 'react-ssr-template', + title: 'React SSR Template', + description: + 'Next.js application skeleton for creating isomorphic web applications.', + uid: '7357f4c5-aa58-4a1e-9670-18931eef771f', + etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw', + generation: 1, + }, + spec: { + type: 'website', + path: '.', + templater: '', + schema: { + $schema: 'http://json-schema.org/draft-07/schema#', + required: ['storePath', 'owner'], + properties: { + owner: { + type: 'string', + title: 'Owner', + description: 'Who is going to own this component', + }, + storePath: { + type: 'string', + title: 'Store path', + description: 'GitHub store path in org/repo format', + }, + }, + }, + }, + }; + + const templaters = new Templaters(); + + expect(() => templaters.get(brokenTemplate)).toThrow( + expect.objectContaining({ + name: 'InputError', + message: expect.stringContaining( + 'Template does not have a required templating key', + ), + }), + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts new file mode 100644 index 0000000000..549b898af8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + TemplaterBase, + SupportedTemplatingKey, + TemplaterBuilder, +} from './types'; + +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { getTemplaterKey } from './helpers'; + +export class Templaters implements TemplaterBuilder { + private preparerMap = new Map(); + + register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) { + this.preparerMap.set(templaterKey, templater); + } + + get(template: TemplateEntityV1alpha1): TemplaterBase { + const templaterKey = getTemplaterKey(template); + const preparer = this.preparerMap.get(templaterKey); + + if (!preparer) { + throw new Error( + `No templater registered for template: "${templaterKey}"`, + ); + } + + return preparer; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts index 519c24e381..6a0fe60858 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - import type { Writable } from 'stream'; import Docker from 'dockerode'; import { JsonValue } from '@backstage/config'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; /** * Currently the required template values. The owner @@ -55,3 +55,16 @@ export type TemplaterBase = { export type TemplaterConfig = { templater?: TemplaterBase; }; + +/** + * List of supported templating options + */ +export type SupportedTemplatingKey = 'cookiecutter' | string; + +/** + * The templater builder holds the templaters ready for run time + */ +export type TemplaterBuilder = { + register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void; + get(template: TemplateEntityV1alpha1): TemplaterBase; +}; From 2fdd40698ac8e5653822540665919004dacbbd3c Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 11:54:32 +0200 Subject: [PATCH 36/41] chore(scaffolder): renaming the processor key to templater --- .../sample-templates/react-ssr-template/template.yaml | 2 +- .../sample-templates/springboot-template/template.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml index 39100308bf..363d488448 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml @@ -8,7 +8,7 @@ metadata: - Recommended - React spec: - processor: cookiecutter + templater: cookiecutter type: website path: '.' schema: diff --git a/plugins/scaffolder-backend/sample-templates/springboot-template/template.yaml b/plugins/scaffolder-backend/sample-templates/springboot-template/template.yaml index 1bc19db2db..19ec7935d9 100644 --- a/plugins/scaffolder-backend/sample-templates/springboot-template/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/springboot-template/template.yaml @@ -8,7 +8,7 @@ metadata: - Recommended - Java spec: - processor: cookiecutter + templater: cookiecutter type: service path: '.' schema: From 1d4d07e07dbfdc2f32d7393574a442d58f4ee5c2 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 11:55:05 +0200 Subject: [PATCH 37/41] chore(scaffolder): refactor out the templaters into a map that is dictated by the router --- .../src/scaffolder/jobs/processor.test.ts | 3 ++- .../scaffolder-backend/src/service/router.ts | 24 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts index 191c949c22..8a875e4e2b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts @@ -37,7 +37,8 @@ describe('JobProcessor', () => { generation: 1, }, spec: { - type: 'cookiecutter', + type: 'website', + templater: 'cookiecutter', path: './template', schema: { $schema: 'http://json-schema.org/draft-07/schema#', diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index abe12c94eb..225662142f 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -16,23 +16,24 @@ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/config'; -import { Octokit } from '@octokit/rest'; import Docker from 'dockerode'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import { - GithubPublisher, JobProcessor, PreparerBuilder, RequiredTemplateValues, StageContext, - TemplaterBase, + TemplaterBuilder, + Publisher, } from '../scaffolder'; export interface RouterOptions { preparers: PreparerBuilder; - templater: TemplaterBase; + templaters: TemplaterBuilder; + publisher: Publisher; + logger: Logger; dockerClient: Docker; } @@ -42,9 +43,14 @@ export async function createRouter( ): Promise { const router = Router(); - const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); - const { preparers, templater, logger: parentLogger, dockerClient } = options; - const githubPulisher = new GithubPublisher({ client: githubClient }); + const { + preparers, + templaters, + publisher, + logger: parentLogger, + dockerClient, + } = options; + const logger = parentLogger.child({ plugin: 'scaffolder' }); const jobProcessor = new JobProcessor(); @@ -106,6 +112,7 @@ export async function createRouter( { name: 'Run the templater', handler: async (ctx: StageContext<{ skeletonDir: string }>) => { + const templater = templaters.get(ctx.entity); const { resultDir } = await templater.run({ directory: ctx.skeletonDir, dockerClient, @@ -120,7 +127,8 @@ export async function createRouter( name: 'Publish template', handler: async (ctx: StageContext<{ resultDir: string }>) => { ctx.logger.info('Should not store the template'); - const { remoteUrl } = await githubPulisher.publish({ + const { remoteUrl } = await publisher.publish({ + entity: ctx.entity, values: ctx.values, directory: ctx.resultDir, }); From d91c10f654475a60829fa33a5c81018e517a319a Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 11:55:39 +0200 Subject: [PATCH 38/41] feat(scaffolder): now the scaffolder is passed a list of templaters which is decided by the template definition --- packages/backend/package.json | 1 + packages/backend/src/plugins/scaffolder.ts | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 097c6446e9..69b97b6335 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -29,6 +29,7 @@ "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.13", "@backstage/plugin-sentry-backend": "^0.1.1-alpha.13", "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.13", + "@octokit/rest": "^18.0.0", "dockerode": "^3.2.0", "express": "^4.17.1", "knex": "^0.21.1", diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 1b0ee46969..70f3d88f3e 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -20,19 +20,34 @@ import { FilePreparer, GithubPreparer, Preparers, + GithubPublisher, + Templaters, } from '@backstage/plugin-scaffolder-backend'; +import { Octokit } from '@octokit/rest'; import type { PluginEnvironment } from '../types'; import Docker from 'dockerode'; export default async function createPlugin({ logger }: PluginEnvironment) { - const templater = new CookieCutter(); + const cookiecutterTemplater = new CookieCutter(); + const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); + const filePreparer = new FilePreparer(); const githubPreparer = new GithubPreparer(); const preparers = new Preparers(); - const dockerClient = new Docker(); preparers.register('file', filePreparer); preparers.register('github', githubPreparer); - return await createRouter({ preparers, templater, logger, dockerClient }); + const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); + const publisher = new GithubPublisher({ client: githubClient }); + + const dockerClient = new Docker(); + return await createRouter({ + preparers, + templaters, + publisher, + logger, + dockerClient, + }); } From 567e5d74cd2159ff4910cbbf9e46f56a778b78c2 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 13 Jul 2020 13:48:54 +0200 Subject: [PATCH 39/41] chore(scaffolder): make the paths a little less specific --- .../src/scaffolder/stages/prepare/preparers.test.ts | 2 +- .../src/scaffolder/stages/templater/templaters.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts index b79adffb18..0aaef2633d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts @@ -24,7 +24,7 @@ describe('Preparers', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'file:/Users/blam/dev/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml', + 'file:/Users/bingo/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml', }, name: 'react-ssr-template', title: 'React SSR Template', diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts index bf2f2ed9c6..1c4fe90207 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts @@ -24,7 +24,7 @@ describe('Templaters', () => { metadata: { annotations: { 'backstage.io/managed-by-location': - 'file:/Users/blam/dev/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml', + 'file:/Users/bingo/spotify/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml', }, name: 'react-ssr-template', title: 'React SSR Template', From fd935dc4a38ddc6bb75db1ce3a269fba2d4c16aa Mon Sep 17 00:00:00 2001 From: ellinors Date: Mon, 13 Jul 2020 14:40:31 +0200 Subject: [PATCH 40/41] fix: updated techdocs container docs --- packages/techdocs-container/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/techdocs-container/README.md b/packages/techdocs-container/README.md index 01cbcc1992..57bdefe894 100644 --- a/packages/techdocs-container/README.md +++ b/packages/techdocs-container/README.md @@ -9,15 +9,15 @@ This is the Docker container that powers the creation of static documentation si Using the TechDocs CLI, we can invoke the latest version of `techdocs-container` via Docker Hub: ```bash -npx @techdocs/cli serve:container +npx @techdocs/cli serve ``` ## Local Development ```bash -docker build ./container -t techdocs-container +docker build . -t mkdocs:local-dev -docker run -w /content -v $(pwd)/mock-docs:/content -p 8000:8000 -it techdocs-container serve -a 0.0.0.0:8000 +docker run -w /content -v $(pwd)/mock-docs:/content -p 8000:8000 -it mkdocs:local-dev serve -a 0.0.0.0:8000 ``` Then open up `http://localhost:8000` on your local machine. From 329305087c4ac648c7faee256ea52ad69b6eebb0 Mon Sep 17 00:00:00 2001 From: ellinors Date: Mon, 13 Jul 2020 15:11:08 +0200 Subject: [PATCH 41/41] fix: add new techdocs folders to codeowners --- .github/CODEOWNERS | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 751ec4914d..d9f9d2ccbe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,8 @@ # The last matching pattern takes precedence. # https://help.github.com/articles/about-codeowners/ -* @spotify/backstage-core -/plugins/techdocs @spotify/techdocs-core -/packages/techdocs-cli @spotify/techdocs-core +* @spotify/backstage-core +/plugins/techdocs @spotify/techdocs-core +/plugins/techdocs-backend @spotify/techdocs-core +/packages/techdocs-cli @spotify/techdocs-core +/packages/techdocs-container @spotify/techdocs-core