From bc15e7a7103cb0e49d5ea6e7eb01919feec2e1ad Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Tue, 23 Jun 2020 11:56:56 +0200 Subject: [PATCH 1/9] feat(scaffolder): cookiecutter templater wip Co-authored-by: Ben Lambert --- plugins/scaffolder-backend/package.json | 1 + .../hooks/post_gen_project.sh | 17 ++++++---------- .../hooks/pre_gen_project.sh | 9 --------- .../package.json | 2 +- plugins/scaffolder-backend/scripts/Dockerfile | 11 ++++++++++ .../src/scaffolder/templater/cookiecutter.ts | 20 +++++++++++++++++-- .../scaffolder-backend/src/service/router.ts | 9 ++++++--- yarn.lock | 7 +++++++ 8 files changed, 50 insertions(+), 26 deletions(-) delete mode 100644 plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh create mode 100644 plugins/scaffolder-backend/scripts/Dockerfile diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 0edf7a8fe9..e85a65e744 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.9", + "@types/dockerode": "^2.5.32", "@types/fs-extra": "^9.0.1", "@types/supertest": "^2.0.8", "supertest": "^4.0.2", diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh index 033a102fed..75c35f2034 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh @@ -1,12 +1,7 @@ -#!/bin/bash +#!/bin/sh -# package name is "__component_id__" so that yarn doesn't throw an error -# about invalid characters when running yarn commands. here we replace it with the actual name -sed -i -e "s/__component_id__/{{ cookiecutter.component_id }}/g" package.json - -# node_modules was moved out of the template folder, during the pre_gen hook, -# to avoid cookie_cutter from copying all of them. time to move it back -mv ../../node_modules.tmp ../../\{\{cookiecutter.component_id\}\}/node_modules 2>/dev/null ||: - -# move back the build directory that was moved out in the pre_gen hook (if it exists) -mv ../../build.tmp ../../\{\{cookiecutter.component_id\}\}/build 2>/dev/null ||: +# Move all template files to the root folder +mv ./* ../ +cd .. +rm -rf {{cookiecutter.component_id}} +# # # # # # # # diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh deleted file mode 100644 index 142fda9ad9..0000000000 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/pre_gen_project.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# no way to ignore files in cookiecutter, so move node_modules out while building -# to avoid cookiecutter from copying all of them -mv ../../\{\{cookiecutter.component_id\}\}/node_modules ../../node_modules.tmp 2>/dev/null ||: - -# cookicutter really doesn't like the next.js build directory, so if the app has -# been built from inside the template folder, that folders needs to be moved out as well -mv ../../\{\{cookiecutter.component_id\}\}/build ../../build.tmp 2>/dev/null ||: diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json index a0ae26e267..6ae3e63a17 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/package.json @@ -1,5 +1,5 @@ { - "name": "__component_id__", + "name": "{{ cookiecutter.component_id }}", "version": "0.0.0", "description": "{{ cookiecutter.description }}", "license": "UNLICENSED", diff --git a/plugins/scaffolder-backend/scripts/Dockerfile b/plugins/scaffolder-backend/scripts/Dockerfile new file mode 100644 index 0000000000..b33c7cbe64 --- /dev/null +++ b/plugins/scaffolder-backend/scripts/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine:3.7 + +RUN apk add --update \ + git \ + python \ + python-dev \ + py-pip \ + g++ && \ + pip install cookiecutter && \ + apk del g++ py-pip python-dev && \ + rm -rf /var/cache/apk/* diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index ee5265fde8..cabb1d4f1e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -16,9 +16,18 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; * limitations under the License. */ import fs from 'fs-extra'; +import Docker from 'dockerode'; export class CookieCutter implements TemplaterBase { + private docker:Docker; + constructor() { + this.docker = new Docker(); + } + public async run(options: TemplaterRunOptions): Promise { + + + // first we need to make cookiecutter.json in the directory provided with the input values. const cookieInfo = { _copy_without_render: ['.github/workflows/*'], @@ -26,7 +35,14 @@ export class CookieCutter implements TemplaterBase { }; await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo); - return ''; - // run cookie cutter with new json + const realTemplatePath = await fs.promises.realpath(options.directory); + const outDir = realTemplatePath + '/result'; + + + await this.docker.run('backstage/cookiecutter', ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], process.stdout, {Volumes: { '/result': {}, '/template': {}}, HostConfig: { + Binds: [`${outDir}:/result`, `${realTemplatePath}:/template`], + }}); + + return outDir; } } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index c9b92494fd..e71593b6ba 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -64,12 +64,15 @@ export async function createRouter( const preparer = preparers.get(mockEntity); // Run the preparer for the mock entity to produce a temporary directory with template in - const path = await preparer.prepare(mockEntity); + const skeletonPath = await preparer.prepare(mockEntity); // Run the templater on the mock directory with values from the post body - await templater.run({ directory: path, values: { component_id: 'test' } }); + const templatedPath = await templater.run({ + directory: skeletonPath, + values: { component_id: 'test', description: "Something for now" }, + }); - console.warn(path); + console.warn(templatedPath); }); const app = express(); diff --git a/yarn.lock b/yarn.lock index fd351efe36..88a9861af6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3520,6 +3520,13 @@ resolved "https://registry.npmjs.org/@types/diff/-/diff-4.0.2.tgz#2e9bb89f9acc3ab0108f0f3dc4dbdcf2fff8a99c" integrity sha512-mIenTfsIe586/yzsyfql69KRnA75S8SVXQbTLpDejRrjH0QSJcpu3AUOi/Vjnt9IOsXKxPhJfGpQUNMueIU1fQ== +"@types/dockerode@^2.5.32": + version "2.5.32" + resolved "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.32.tgz#52d3628f605f8ea65202541c59a8a6dd166384fd" + integrity sha512-TfaGOoOHxsjkWRj2sPoQ3FLmTC5mVMhZ4kzZy13U7mjtIDoloE4e7AMj5jPLbffWB6Csy5DF5e0lC9M+tnKz/A== + dependencies: + "@types/node" "*" + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d" From 12db9bc694dba19bf992e91632f8283ac1738ca3 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Tue, 23 Jun 2020 14:17:28 +0200 Subject: [PATCH 2/9] chore: add tests, merge cookiecutter.json Co-authored-by: Ben Lambert --- plugins/scaffolder-backend/package.json | 1 + .../react-ssr-template/cookiecutter.json | 3 + .../scaffolder/templater/cookiecutter.test.ts | 119 ++++++++++++++++++ .../src/scaffolder/templater/cookiecutter.ts | 42 +++++-- 4 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json create mode 100644 plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index e85a65e744..b356f026e5 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -23,6 +23,7 @@ "dependencies": { "@backstage/backend-common": "^0.1.1-alpha.9", "@backstage/catalog-model": "^0.1.1-alpha.9", + "@backstage/config": "^0.1.1-alpha.9", "@types/express": "^4.17.6", "compression": "^1.7.4", "cors": "^2.8.5", diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json b/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json new file mode 100644 index 0000000000..bc6e4b5347 --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/cookiecutter.json @@ -0,0 +1,3 @@ +{ + "_copy_without_render": [".github/workflows/*"] +} diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts new file mode 100644 index 0000000000..8362a759bf --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { CookieCutter } from './cookiecutter'; +import fs from 'fs-extra'; +import os from 'os'; + +const mockDocker = { run: jest.fn() }; +jest.mock( + 'dockerode', + () => + class { + constructor() { + return mockDocker; + } + }, +); + +describe('CookieCutter Templater', () => { + const cookie = new CookieCutter(); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should write a cookiecutter.json file with the values from the entitiy', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ directory: tempdir, values }); + + const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); + + expect(cookieCutterJson).toEqual(expect.objectContaining(values)); + }); + + it('should merge any value that is in the cookiecutter.json path already', async () => { + const tempdir = os.tmpdir(); + const existingJson = { + _copy_without_render: ['./github/workflows/*'], + }; + await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson); + + const values = { + component_id: 'hello', + description: 'im something cool', + }; + + await cookie.run({ directory: tempdir, values }); + + const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); + + expect(cookieCutterJson).toEqual({ ...existingJson, ...values }); + }); + + it('should run the correct docker container with the correct bindings for the volumes', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ directory: tempdir, values }); + + const realpath = await fs.realpath(tempdir); + + // TODO(blam): This might change when we publish our own cookiecutter image + // to @backstage/cookiecutter in docker hub. + expect(mockDocker.run).toHaveBeenCalledWith( + 'backstage/cookiecutter', + ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], + process.stdout, + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${realpath}:/template`, + `${realpath}/result:/result`, + ]), + }, + Volumes: { + '/template': {}, + '/result': {}, + }, + }), + ); + }); + + it('should return the result path to the end templated folder', async () => { + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + const path = await cookie.run({ directory: tempdir, values }); + + const realpath = await fs.realpath(tempdir); + + expect(path).toBe(`${realpath}/result`); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index cabb1d4f1e..e927108dc4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -17,32 +17,52 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; */ import fs from 'fs-extra'; import Docker from 'dockerode'; +import { JsonValue } from '@backstage/config'; export class CookieCutter implements TemplaterBase { - private docker:Docker; + private docker: Docker; constructor() { this.docker = new Docker(); } + private async fetchTemplateCookieCutter( + directory: string, + ): Promise> { + try { + return await fs.readJSON(`${directory}/cookiecutter.json`); + } catch (ex) { + return {}; + } + } + public async run(options: TemplaterRunOptions): Promise { - - - - // first we need to make cookiecutter.json in the directory provided with the input values. + // First lets grab the default cookiecutter.json file + const cookieCutterJson = await this.fetchTemplateCookieCutter( + options.directory, + ); + const cookieInfo = { - _copy_without_render: ['.github/workflows/*'], + ...cookieCutterJson, ...options.values, }; await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo); + const realTemplatePath = await fs.promises.realpath(options.directory); - const outDir = realTemplatePath + '/result'; + const outDir = `${realTemplatePath}/result`; + await this.docker.run( + 'backstage/cookiecutter', + ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], + process.stdout, + { + Volumes: { '/result': {}, '/template': {} }, + HostConfig: { + Binds: [`${outDir}:/result`, `${realTemplatePath}:/template`], + }, + }, + ); - await this.docker.run('backstage/cookiecutter', ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], process.stdout, {Volumes: { '/result': {}, '/template': {}}, HostConfig: { - Binds: [`${outDir}:/result`, `${realTemplatePath}:/template`], - }}); - return outDir; } } From e59cac61fd44d153c6940963b85ac50c3a3021e6 Mon Sep 17 00:00:00 2001 From: Ivan Shmidt Date: Tue, 23 Jun 2020 15:30:38 +0200 Subject: [PATCH 3/9] feat(scaffolder): error handling, log propagation Co-authored-by: Ben Lambert --- .../scaffolder/templater/cookiecutter.test.ts | 70 +++++++++++++++++-- .../src/scaffolder/templater/cookiecutter.ts | 22 ++++-- .../src/scaffolder/templater/index.ts | 3 + .../scaffolder-backend/src/service/router.ts | 2 +- 4 files changed, 88 insertions(+), 9 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts index 8362a759bf..bddf5a7b25 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts @@ -16,8 +16,11 @@ import { CookieCutter } from './cookiecutter'; import fs from 'fs-extra'; import os from 'os'; +import Stream, { PassThrough } from 'stream'; -const mockDocker = { run: jest.fn() }; +const mockDocker = { + run: jest.fn(() => [{ Error: null, StatusCode: 0 }]), +}; jest.mock( 'dockerode', () => @@ -32,7 +35,7 @@ describe('CookieCutter Templater', () => { const cookie = new CookieCutter(); beforeEach(() => { - jest.resetAllMocks(); + jest.clearAllMocks(); }); it('should write a cookiecutter.json file with the values from the entitiy', async () => { @@ -85,8 +88,8 @@ describe('CookieCutter Templater', () => { // to @backstage/cookiecutter in docker hub. expect(mockDocker.run).toHaveBeenCalledWith( 'backstage/cookiecutter', - ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], - process.stdout, + ['cookiecutter', '--no-input', '-o', '/result', '/template'], + expect.any(Stream), expect.objectContaining({ HostConfig: { Binds: expect.arrayContaining([ @@ -116,4 +119,63 @@ describe('CookieCutter Templater', () => { expect(path).toBe(`${realpath}/result`); }); + + it('throws a correct error if the templating fails in docker', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: new Error('Something went wrong with docker'), + StatusCode: 0, + }, + ]); + + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow( + /Something went wrong with docker/, + ); + }); + + it('uses the passed stream as a log stream', async () => { + const logStream = new PassThrough(); + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await cookie.run({ directory: tempdir, values, logStream }); + + expect(mockDocker.run).toHaveBeenCalledWith( + expect.any(String), + expect.any(Array), + logStream, + expect.any(Object), + ); + }); + + it('throws a correct error if the container returns a non-zero exit code', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: null, + StatusCode: 1, + }, + ]); + + const tempdir = os.tmpdir(); + + const values = { + component_id: 'test', + description: 'description', + }; + + await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow( + /Docker container returned a non-zero exit code \(1\)/, + ); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index e927108dc4..b612900123 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -18,7 +18,7 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; import fs from 'fs-extra'; import Docker from 'dockerode'; import { JsonValue } from '@backstage/config'; - +import { PassThrough } from 'stream'; export class CookieCutter implements TemplaterBase { private docker: Docker; constructor() { @@ -51,10 +51,12 @@ export class CookieCutter implements TemplaterBase { const realTemplatePath = await fs.promises.realpath(options.directory); const outDir = `${realTemplatePath}/result`; - await this.docker.run( + const [ + { Error: dockerError, StatusCode: containerStatusCode }, + ] = await this.docker.run( 'backstage/cookiecutter', - ['cookiecutter', '--no-input', '-o', '/result', '/template', '--verbose'], - process.stdout, + ['cookiecutter', '--no-input', '-o', '/result', '/template'], + options.logStream ?? new PassThrough(), { Volumes: { '/result': {}, '/template': {} }, HostConfig: { @@ -63,6 +65,18 @@ export class CookieCutter implements TemplaterBase { }, ); + if (dockerError) { + throw new Error( + `Docker failed to run with the following error message: ${dockerError}`, + ); + } + + if (containerStatusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${containerStatusCode})`, + ); + } + return outDir; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts index 80570a1c48..0c255354fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import type { Writable } from 'stream'; + export interface RequiredTemplateValues { component_id: string; } @@ -21,6 +23,7 @@ export interface RequiredTemplateValues { export interface TemplaterRunOptions { directory: string; values: RequiredTemplateValues & object; + logStream?: Writable; } export abstract class TemplaterBase { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index e71593b6ba..6636fc4b68 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -69,7 +69,7 @@ export async function createRouter( // Run the templater on the mock directory with values from the post body const templatedPath = await templater.run({ directory: skeletonPath, - values: { component_id: 'test', description: "Something for now" }, + values: { component_id: 'test' }, }); console.warn(templatedPath); From 0fe8e594f238702447f99c8d9654dd712cd33fbc Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 23 Jun 2020 18:07:03 +0200 Subject: [PATCH 4/9] refactor(scaffolder): Moving out the dockerish logic into something that is easy to consume for other templaters --- .../react-ssr-template/template.yaml | 1 - .../scaffolder/templater/cookiecutter.test.ts | 115 ++++----------- .../src/scaffolder/templater/cookiecutter.ts | 48 ++----- .../src/scaffolder/templater/helpers.test.ts | 131 ++++++++++++++++++ .../src/scaffolder/templater/helpers.ts | 75 ++++++++++ .../src/scaffolder/templater/templaters.ts | 38 +++++ 6 files changed, 286 insertions(+), 122 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/templater/templaters.ts diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml index 17f9ebbf7d..6834f0d1b6 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml @@ -7,4 +7,3 @@ metadata: spec: type: cookiecutter path: '.' - diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts index bddf5a7b25..bba3705a41 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts @@ -13,27 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +jest.mock('./helpers', () => ({ runDockerContainer: jest.fn() })); + import { CookieCutter } from './cookiecutter'; import fs from 'fs-extra'; import os from 'os'; -import Stream, { PassThrough } from 'stream'; - -const mockDocker = { - run: jest.fn(() => [{ Error: null, StatusCode: 0 }]), -}; -jest.mock( - 'dockerode', - () => - class { - constructor() { - return mockDocker; - } - }, -); +import { RunDockerContainerOptions } from './helpers'; +import { PassThrough } from 'stream'; describe('CookieCutter Templater', () => { const cookie = new CookieCutter(); + const { + runDockerContainer, + }: { + runDockerContainer: jest.Mock; + } = require('./helpers'); + beforeEach(() => { jest.clearAllMocks(); }); @@ -82,29 +78,14 @@ describe('CookieCutter Templater', () => { await cookie.run({ directory: tempdir, values }); - const realpath = await fs.realpath(tempdir); - - // TODO(blam): This might change when we publish our own cookiecutter image - // to @backstage/cookiecutter in docker hub. - expect(mockDocker.run).toHaveBeenCalledWith( - 'backstage/cookiecutter', - ['cookiecutter', '--no-input', '-o', '/result', '/template'], - expect.any(Stream), - expect.objectContaining({ - HostConfig: { - Binds: expect.arrayContaining([ - `${realpath}:/template`, - `${realpath}/result:/result`, - ]), - }, - Volumes: { - '/template': {}, - '/result': {}, - }, - }), - ); + expect(runDockerContainer).toHaveBeenCalledWith({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir: tempdir, + resultDir: `${tempdir}/result`, + logStream: undefined, + }); }); - it('should return the result path to the end templated folder', async () => { const tempdir = os.tmpdir(); @@ -115,18 +96,11 @@ describe('CookieCutter Templater', () => { const path = await cookie.run({ directory: tempdir, values }); - const realpath = await fs.realpath(tempdir); - - expect(path).toBe(`${realpath}/result`); + expect(path).toBe(`${tempdir}/result`); }); - it('throws a correct error if the templating fails in docker', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: new Error('Something went wrong with docker'), - StatusCode: 0, - }, - ]); + it('should pass through the streamer to the run docker helper', async () => { + const stream = new PassThrough(); const tempdir = os.tmpdir(); @@ -135,47 +109,14 @@ describe('CookieCutter Templater', () => { description: 'description', }; - await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow( - /Something went wrong with docker/, - ); - }); + await cookie.run({ directory: tempdir, values, logStream: stream }); - it('uses the passed stream as a log stream', async () => { - const logStream = new PassThrough(); - const tempdir = os.tmpdir(); - - const values = { - component_id: 'test', - description: 'description', - }; - - await cookie.run({ directory: tempdir, values, logStream }); - - expect(mockDocker.run).toHaveBeenCalledWith( - expect.any(String), - expect.any(Array), - logStream, - expect.any(Object), - ); - }); - - it('throws a correct error if the container returns a non-zero exit code', async () => { - mockDocker.run.mockResolvedValueOnce([ - { - Error: null, - StatusCode: 1, - }, - ]); - - const tempdir = os.tmpdir(); - - const values = { - component_id: 'test', - description: 'description', - }; - - await expect(cookie.run({ directory: tempdir, values })).rejects.toThrow( - /Docker container returned a non-zero exit code \(1\)/, - ); + expect(runDockerContainer).toHaveBeenCalledWith({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir: tempdir, + resultDir: `${tempdir}/result`, + logStream: stream, + }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index b612900123..70a0cc28af 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -16,15 +16,10 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; * limitations under the License. */ import fs from 'fs-extra'; -import Docker from 'dockerode'; import { JsonValue } from '@backstage/config'; -import { PassThrough } from 'stream'; -export class CookieCutter implements TemplaterBase { - private docker: Docker; - constructor() { - this.docker = new Docker(); - } +import { runDockerContainer } from './helpers'; +export class CookieCutter implements TemplaterBase { private async fetchTemplateCookieCutter( directory: string, ): Promise> { @@ -48,35 +43,20 @@ export class CookieCutter implements TemplaterBase { await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo); - const realTemplatePath = await fs.promises.realpath(options.directory); - const outDir = `${realTemplatePath}/result`; + const templateDir = options.directory; - const [ - { Error: dockerError, StatusCode: containerStatusCode }, - ] = await this.docker.run( - 'backstage/cookiecutter', - ['cookiecutter', '--no-input', '-o', '/result', '/template'], - options.logStream ?? new PassThrough(), - { - Volumes: { '/result': {}, '/template': {} }, - HostConfig: { - Binds: [`${outDir}:/result`, `${realTemplatePath}:/template`], - }, - }, - ); + // TODO(blam): This should be an entirely different directory on the host machine + // not in the template directory + const resultDir = `${templateDir}/result`; - if (dockerError) { - throw new Error( - `Docker failed to run with the following error message: ${dockerError}`, - ); - } + await runDockerContainer({ + imageName: 'backstage/cookiecutter', + args: ['cookiecutter', '--no-input', '-o', '/result', '/template'], + templateDir, + resultDir, + logStream: options.logStream, + }); - if (containerStatusCode !== 0) { - throw new Error( - `Docker container returned a non-zero exit code (${containerStatusCode})`, - ); - } - - return outDir; + return resultDir; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts new file mode 100644 index 0000000000..a5ac00165e --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Stream, { PassThrough } from 'stream'; +import os from 'os'; +import fs from 'fs'; + +const mockDocker = { + run: jest.fn(() => [{ Error: null, StatusCode: 0 }]), +}; + +jest.mock( + 'dockerode', + () => + class { + constructor() { + return mockDocker; + } + }, +); + +import { runDockerContainer } from './helpers'; + +describe('helpers', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('runDockerContainer', () => { + const imageName = 'blam/github:ben'; + const args = ['bash', '-c', 'echo lol']; + const templateDir = os.tmpdir(); + const resultDir = os.tmpdir(); + + it('should call the dockerClient run command with the correct arguments passed through', async () => { + await runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + expect.any(Stream), + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(templateDir)}:/template`, + `${await fs.promises.realpath(resultDir)}:/result`, + ]), + }, + Volumes: { + '/template': {}, + '/result': {}, + }, + }), + ); + }); + + it('throws a correct error if the templating fails in docker', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: new Error('Something went wrong with docker'), + StatusCode: 0, + }, + ]); + + await expect( + runDockerContainer({ imageName, args, templateDir, resultDir }), + ).rejects.toThrow(/Something went wrong with docker/); + }); + + it('throws a correct error when the response code of the container is non-zero', async () => { + mockDocker.run.mockResolvedValueOnce([ + { + Error: null, + StatusCode: 123, + }, + ]); + + await expect( + runDockerContainer({ imageName, args, templateDir, resultDir }), + ).rejects.toThrow( + /Docker container returned a non-zero exit code \(123\)/, + ); + }); + + it('should pass through the log stream to the docker client', async () => { + const logStream = new PassThrough(); + await runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + logStream, + }); + + expect(mockDocker.run).toHaveBeenCalledWith( + imageName, + args, + logStream, + expect.objectContaining({ + HostConfig: { + Binds: expect.arrayContaining([ + `${await fs.promises.realpath(templateDir)}:/template`, + `${await fs.promises.realpath(resultDir)}:/result`, + ]), + }, + Volumes: { + '/template': {}, + '/result': {}, + }, + }), + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts new file mode 100644 index 0000000000..f60dcfc664 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Docker from 'dockerode'; +import { Writable, PassThrough } from 'stream'; +import fs from 'fs'; + +export type RunDockerContainerOptions = { + imageName: string; + args: string[]; + logStream?: Writable; + resultDir: string; + templateDir: string; +}; + +const dockerClient = new Docker(); +/** + * + * @param options the options object + * @param options.imageName the image to run + * @param options.args the arguments to pass the container + * @param options.logStream the log streamer to capture log messages + * @param options.resultDir the /result path inside the container + * @param options.templateDir the /template path inside the container + */ +export const runDockerContainer = async ({ + imageName, + args, + logStream = new PassThrough(), + resultDir, + templateDir, +}: RunDockerContainerOptions) => { + const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( + imageName, + args, + logStream, + { + Volumes: { '/result': {}, '/template': {} }, + HostConfig: { + Binds: [ + // Need to use realpath here as Docker mounting does not like + // symlinks for binding volumes + `${await fs.promises.realpath(resultDir)}:/result`, + `${await fs.promises.realpath(templateDir)}:/template`, + ], + }, + }, + ); + + if (error) { + throw new Error( + `Docker failed to run with the following error message: ${error}`, + ); + } + + if (statusCode !== 0) { + throw new Error( + `Docker container returned a non-zero exit code (${statusCode})`, + ); + } + + return { error, statusCode }; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/templaters.ts b/plugins/scaffolder-backend/src/scaffolder/templater/templaters.ts new file mode 100644 index 0000000000..235a699a95 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/templater/templaters.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { parseLocationAnnotation } from './helpers'; + +export class Templaters implements PreparerBuilder { + private templaterMap = new Map(); + + register(protocol: RemoteProtocol, preparer: PreparerBase) { + this.templaterMap.set(protocol, preparer); + } + + get(template: TemplateEntityV1alpha1): PreparerBase { + const { protocol } = parseLocationAnnotation(template); + const preparer = this.templaterMap.get(protocol); + + if (!preparer) { + throw new Error(`No preparer registered for type: "${protocol}"`); + } + + return preparer; + } +} From 7ead8675dd130af527830d6e71049c8bc58bc2f9 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Jun 2020 10:47:43 +0200 Subject: [PATCH 5/9] chore(scaffolder): code review comments, passing down the docker client --- packages/backend/package.json | 2 + packages/backend/src/plugins/scaffolder.ts | 4 +- plugins/scaffolder-backend/src/index.ts | 1 + .../scaffolder/templater/cookiecutter.test.ts | 43 ++++++++++++++++--- .../src/scaffolder/templater/cookiecutter.ts | 5 +++ .../src/scaffolder/templater/helpers.test.ts | 40 +++++++++-------- .../src/scaffolder/templater/helpers.ts | 6 ++- .../src/scaffolder/templater/index.ts | 8 ++-- .../scaffolder-backend/src/service/router.ts | 5 ++- 9 files changed, 82 insertions(+), 32 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index a4d5f059f9..2ae211fd8e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -26,6 +26,7 @@ "@backstage/plugin-identity-backend": "^0.1.1-alpha.10", "@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.10", "@backstage/plugin-sentry-backend": "^0.1.1-alpha.10", + "dockerode": "^3.2.0", "express": "^4.17.1", "knex": "^0.21.1", "sqlite3": "^4.2.0", @@ -33,6 +34,7 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.10", + "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", "@types/express-serve-static-core": "^4.17.5", "@types/helmet": "^0.0.47" diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 95d28eacb1..1b0ee46969 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -22,15 +22,17 @@ import { Preparers, } from '@backstage/plugin-scaffolder-backend'; import type { PluginEnvironment } from '../types'; +import Docker from 'dockerode'; export default async function createPlugin({ logger }: PluginEnvironment) { const templater = new CookieCutter(); const filePreparer = new FilePreparer(); const githubPreparer = new GithubPreparer(); const preparers = new Preparers(); + const dockerClient = new Docker(); preparers.register('file', filePreparer); preparers.register('github', githubPreparer); - return await createRouter({ preparers, templater, logger }); + return await createRouter({ preparers, templater, logger, dockerClient }); } diff --git a/plugins/scaffolder-backend/src/index.ts b/plugins/scaffolder-backend/src/index.ts index c461bfede6..0a0a4cb95f 100644 --- a/plugins/scaffolder-backend/src/index.ts +++ b/plugins/scaffolder-backend/src/index.ts @@ -16,3 +16,4 @@ export * from './scaffolder'; export * from './service/router'; + diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts index bba3705a41..7c78badff1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.test.ts @@ -20,18 +20,21 @@ import fs from 'fs-extra'; import os from 'os'; import { RunDockerContainerOptions } from './helpers'; import { PassThrough } from 'stream'; +import Docker from 'dockerode'; describe('CookieCutter Templater', () => { const cookie = new CookieCutter(); - + const mockDocker = {} as Docker; const { runDockerContainer, }: { runDockerContainer: jest.Mock; } = require('./helpers'); - beforeEach(() => { + beforeEach(async () => { jest.clearAllMocks(); + + await fs.remove(`${os.tmpdir()}/cookiecutter.json`); }); it('should write a cookiecutter.json file with the values from the entitiy', async () => { @@ -42,7 +45,7 @@ describe('CookieCutter Templater', () => { description: 'description', }; - await cookie.run({ directory: tempdir, values }); + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); @@ -61,13 +64,28 @@ describe('CookieCutter Templater', () => { description: 'im something cool', }; - await cookie.run({ directory: tempdir, values }); + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`); expect(cookieCutterJson).toEqual({ ...existingJson, ...values }); }); + it('should throw an error if the cookiecutter json is malformed and not missing', async () => { + const tempdir = os.tmpdir(); + + await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'"); + + const values = { + component_id: 'hello', + description: 'im something cool', + }; + + await expect( + cookie.run({ directory: tempdir, values, dockerClient: mockDocker }), + ).rejects.toThrow(/Unexpected token ' in JSON at position 1/); + }); + it('should run the correct docker container with the correct bindings for the volumes', async () => { const tempdir = os.tmpdir(); @@ -76,7 +94,7 @@ describe('CookieCutter Templater', () => { description: 'description', }; - await cookie.run({ directory: tempdir, values }); + await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); expect(runDockerContainer).toHaveBeenCalledWith({ imageName: 'backstage/cookiecutter', @@ -84,6 +102,7 @@ describe('CookieCutter Templater', () => { templateDir: tempdir, resultDir: `${tempdir}/result`, logStream: undefined, + dockerClient: mockDocker, }); }); it('should return the result path to the end templated folder', async () => { @@ -94,7 +113,11 @@ describe('CookieCutter Templater', () => { description: 'description', }; - const path = await cookie.run({ directory: tempdir, values }); + const path = await cookie.run({ + directory: tempdir, + values, + dockerClient: mockDocker, + }); expect(path).toBe(`${tempdir}/result`); }); @@ -109,7 +132,12 @@ describe('CookieCutter Templater', () => { description: 'description', }; - await cookie.run({ directory: tempdir, values, logStream: stream }); + await cookie.run({ + directory: tempdir, + values, + logStream: stream, + dockerClient: mockDocker, + }); expect(runDockerContainer).toHaveBeenCalledWith({ imageName: 'backstage/cookiecutter', @@ -117,6 +145,7 @@ describe('CookieCutter Templater', () => { templateDir: tempdir, resultDir: `${tempdir}/result`, logStream: stream, + dockerClient: mockDocker, }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts index 70a0cc28af..a95eee7fb4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/cookiecutter.ts @@ -26,6 +26,10 @@ export class CookieCutter implements TemplaterBase { try { return await fs.readJSON(`${directory}/cookiecutter.json`); } catch (ex) { + if (ex.code !== 'ENOENT') { + throw ex; + } + return {}; } } @@ -55,6 +59,7 @@ export class CookieCutter implements TemplaterBase { templateDir, resultDir, logStream: options.logStream, + dockerClient: options.dockerClient, }); return resultDir; diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts index a5ac00165e..d27eb4f93f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.test.ts @@ -16,26 +16,16 @@ import Stream, { PassThrough } from 'stream'; import os from 'os'; import fs from 'fs'; - -const mockDocker = { - run: jest.fn(() => [{ Error: null, StatusCode: 0 }]), -}; - -jest.mock( - 'dockerode', - () => - class { - constructor() { - return mockDocker; - } - }, -); - +import Docker from 'dockerode'; import { runDockerContainer } from './helpers'; describe('helpers', () => { + const mockDocker = new Docker() as jest.Mocked; + beforeEach(() => { - jest.clearAllMocks(); + jest + .spyOn(mockDocker, 'run') + .mockResolvedValue([{ Error: null, StatusCode: 0 }]); }); describe('runDockerContainer', () => { @@ -50,6 +40,7 @@ describe('helpers', () => { args, templateDir, resultDir, + dockerClient: mockDocker, }); expect(mockDocker.run).toHaveBeenCalledWith( @@ -80,7 +71,13 @@ describe('helpers', () => { ]); await expect( - runDockerContainer({ imageName, args, templateDir, resultDir }), + runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }), ).rejects.toThrow(/Something went wrong with docker/); }); @@ -93,7 +90,13 @@ describe('helpers', () => { ]); await expect( - runDockerContainer({ imageName, args, templateDir, resultDir }), + runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }), ).rejects.toThrow( /Docker container returned a non-zero exit code \(123\)/, ); @@ -107,6 +110,7 @@ describe('helpers', () => { templateDir, resultDir, logStream, + dockerClient: mockDocker, }); expect(mockDocker.run).toHaveBeenCalledWith( diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts index f60dcfc664..2856a016e4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/helpers.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import Docker from 'dockerode'; import { Writable, PassThrough } from 'stream'; +import Docker from 'dockerode'; import fs from 'fs'; export type RunDockerContainerOptions = { @@ -23,9 +23,9 @@ export type RunDockerContainerOptions = { logStream?: Writable; resultDir: string; templateDir: string; + dockerClient: Docker; }; -const dockerClient = new Docker(); /** * * @param options the options object @@ -34,6 +34,7 @@ const dockerClient = new Docker(); * @param options.logStream the log streamer to capture log messages * @param options.resultDir the /result path inside the container * @param options.templateDir the /template path inside the container + * @param options.dockerClient the dockerClient to use */ export const runDockerContainer = async ({ imageName, @@ -41,6 +42,7 @@ export const runDockerContainer = async ({ logStream = new PassThrough(), resultDir, templateDir, + dockerClient, }: RunDockerContainerOptions) => { const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( imageName, diff --git a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts index 0c255354fe..8885e65308 100644 --- a/plugins/scaffolder-backend/src/scaffolder/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/templater/index.ts @@ -15,6 +15,7 @@ */ import type { Writable } from 'stream'; +import Docker from 'dockerode'; export interface RequiredTemplateValues { component_id: string; @@ -24,12 +25,13 @@ export interface TemplaterRunOptions { directory: string; values: RequiredTemplateValues & object; logStream?: Writable; + dockerClient: Docker; } -export abstract class TemplaterBase { +export type TemplaterBase = { // runs the templating with the values and returns the directory to push the VCS - abstract async run(opts: TemplaterRunOptions): Promise; -} + run(opts: TemplaterRunOptions): Promise; +}; export interface TemplaterConfig { templater?: TemplaterBase; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 16b119da02..ab39864e76 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -19,18 +19,20 @@ import Router from 'express-promise-router'; import express from 'express'; import { PreparerBuilder, TemplaterBase } from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import Docker from 'dockerode'; export interface RouterOptions { preparers: PreparerBuilder; templater: TemplaterBase; logger: Logger; + dockerClient: Docker; } export async function createRouter( options: RouterOptions, ): Promise { const router = Router(); - const { preparers, templater, logger: parentLogger } = options; + const { preparers, templater, logger: parentLogger, dockerClient } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); router.post('/v1/jobs', async (_, res) => { @@ -72,6 +74,7 @@ export async function createRouter( const templatedPath = await templater.run({ directory: skeletonPath, values: { component_id: 'test' }, + dockerClient, }); console.warn(templatedPath); From d581284890a653c1fbb7247cecd4effefff224e6 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Jun 2020 14:01:13 +0200 Subject: [PATCH 6/9] chore(scaffolder): fixing the type deps --- plugins/scaffolder-backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 061f6ca814..c8c40d7b67 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -25,6 +25,7 @@ "@backstage/catalog-model": "^0.1.1-alpha.12", "@backstage/config": "^0.1.1-alpha.12", "@types/express": "^4.17.6", + "@types/dockerode": "^2.5.32", "compression": "^1.7.4", "cors": "^2.8.5", "dockerode": "^3.2.0", @@ -40,7 +41,6 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.12", - "@types/dockerode": "^2.5.32", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", "@types/nodegit": "0.26.5", From f4cdbc005db6950ff72223c2f481d80a00c1d340 Mon Sep 17 00:00:00 2001 From: Bilawal Hameed Date: Thu, 25 Jun 2020 11:18:47 +0200 Subject: [PATCH 7/9] techdocs: skeleton of reader (#1438) * chore: replaced default components * feat(techdocs): added skeleton of reader * fix: added types * fix: split out into separate file * fix: add shadowDom hook test * fix: added @backstage/test-utils as dep * fix: formatting python files --- .../container/techdocs-core/src/core.py | 64 +- plugins/techdocs/package.json | 1 + .../ExampleComponent.test.tsx | 34 - .../ExampleComponent/ExampleComponent.tsx | 57 -- .../ExampleFetchComponent.tsx | 108 --- plugins/techdocs/src/plugin.ts | 8 +- .../techdocs/src/reader/components/Reader.tsx | 816 ++++++++++++++++++ .../hooks}/index.ts | 2 +- .../src/reader/hooks/shadowDom.test.tsx | 44 + .../hooks/shadowDom.ts} | 24 +- .../index.ts => reader/index.tsx} | 2 +- 11 files changed, 912 insertions(+), 248 deletions(-) delete mode 100644 plugins/techdocs/src/components/ExampleComponent/ExampleComponent.test.tsx delete mode 100644 plugins/techdocs/src/components/ExampleComponent/ExampleComponent.tsx delete mode 100644 plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx create mode 100644 plugins/techdocs/src/reader/components/Reader.tsx rename plugins/techdocs/src/{components/ExampleComponent => reader/hooks}/index.ts (92%) create mode 100644 plugins/techdocs/src/reader/hooks/shadowDom.test.tsx rename plugins/techdocs/src/{components/ExampleFetchComponent/ExampleFetchComponent.test.tsx => reader/hooks/shadowDom.ts} (56%) rename plugins/techdocs/src/{components/ExampleFetchComponent/index.ts => reader/index.tsx} (92%) diff --git a/plugins/techdocs/mkdocs/container/techdocs-core/src/core.py b/plugins/techdocs/mkdocs/container/techdocs-core/src/core.py index 18c22c40e3..e6c4a0403d 100644 --- a/plugins/techdocs/mkdocs/container/techdocs-core/src/core.py +++ b/plugins/techdocs/mkdocs/container/techdocs-core/src/core.py @@ -44,42 +44,42 @@ class TechDocsCore(BasePlugin): config["plugins"]["search"] = search_plugin # Markdown Extensions - config['markdown_extensions'].append('admonition') - config['markdown_extensions'].append('abbr') - config['markdown_extensions'].append('attr_list') - config['markdown_extensions'].append('def_list') - config['markdown_extensions'].append('codehilite') - config['mdx_configs']['codehilite'] = { - 'linenums': True, - 'guess_lang': False, - 'pygments_style': 'friendly', + config["markdown_extensions"].append("admonition") + config["markdown_extensions"].append("abbr") + config["markdown_extensions"].append("attr_list") + config["markdown_extensions"].append("def_list") + config["markdown_extensions"].append("codehilite") + config["mdx_configs"]["codehilite"] = { + "linenums": True, + "guess_lang": False, + "pygments_style": "friendly", } - config['markdown_extensions'].append('toc') - config['mdx_configs']['toc'] = { - 'permalink': True, + config["markdown_extensions"].append("toc") + config["mdx_configs"]["toc"] = { + "permalink": True, } - config['markdown_extensions'].append('footnotes') - config['markdown_extensions'].append('markdown.extensions.tables') - config['markdown_extensions'].append('pymdownx.betterem') - config['mdx_configs']['pymdownx.betterem'] = { - 'smart_enable': 'all', + config["markdown_extensions"].append("footnotes") + config["markdown_extensions"].append("markdown.extensions.tables") + config["markdown_extensions"].append("pymdownx.betterem") + config["mdx_configs"]["pymdownx.betterem"] = { + "smart_enable": "all", } - config['markdown_extensions'].append('pymdownx.caret') - config['markdown_extensions'].append('pymdownx.critic') - config['markdown_extensions'].append('pymdownx.details') - config['markdown_extensions'].append('pymdownx.emoji') - config['mdx_configs']['pymdownx.emoji'] = { - 'emoji_generator': '!!python/name:pymdownx.emoji.to_svg', + config["markdown_extensions"].append("pymdownx.caret") + config["markdown_extensions"].append("pymdownx.critic") + config["markdown_extensions"].append("pymdownx.details") + config["markdown_extensions"].append("pymdownx.emoji") + config["mdx_configs"]["pymdownx.emoji"] = { + "emoji_generator": "!!python/name:pymdownx.emoji.to_svg", } - config['markdown_extensions'].append('pymdownx.inlinehilite') - config['markdown_extensions'].append('pymdownx.magiclink') - config['markdown_extensions'].append('pymdownx.mark') - config['markdown_extensions'].append('pymdownx.smartsymbols') - config['markdown_extensions'].append('pymdownx.superfences') - config['markdown_extensions'].append('pymdownx.tasklist') - config['mdx_configs']['pymdownx.tasklist'] = { - 'custom_checkbox': True, + config["markdown_extensions"].append("pymdownx.inlinehilite") + config["markdown_extensions"].append("pymdownx.magiclink") + config["markdown_extensions"].append("pymdownx.mark") + config["markdown_extensions"].append("pymdownx.smartsymbols") + config["markdown_extensions"].append("pymdownx.superfences") + config["markdown_extensions"].append("pymdownx.tasklist") + config["mdx_configs"]["pymdownx.tasklist"] = { + "custom_checkbox": True, } - config['markdown_extensions'].append('pymdownx.tilde') + config["markdown_extensions"].append("pymdownx.tilde") return config diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 1f83d6290f..9ef290fa40 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@backstage/core": "^0.1.1-alpha.12", + "@backstage/test-utils": "^0.1.1-alpha.12", "@backstage/theme": "^0.1.1-alpha.12", "@material-ui/core": "^4.9.1", "@material-ui/icons": "^4.9.1", diff --git a/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.test.tsx b/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.test.tsx deleted file mode 100644 index e4d760526e..0000000000 --- a/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.test.tsx +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import { render } from '@testing-library/react'; -import mockFetch from 'jest-fetch-mock'; -import ExampleComponent from './ExampleComponent'; -import { ThemeProvider } from '@material-ui/core'; -import { lightTheme } from '@backstage/theme'; - -describe('ExampleComponent', () => { - it('should render', () => { - mockFetch.mockResponse(() => new Promise(() => {})); - const rendered = render( - - - , - ); - expect(rendered.getByText('Welcome to techdocs!')).toBeInTheDocument(); - }); -}); diff --git a/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.tsx b/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.tsx deleted file mode 100644 index 2ab29604ee..0000000000 --- a/plugins/techdocs/src/components/ExampleComponent/ExampleComponent.tsx +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React, { FC } from 'react'; -import { Typography, Grid } from '@material-ui/core'; -import { - InfoCard, - Header, - Page, - pageTheme, - Content, - ContentHeader, - HeaderLabel, - SupportButton, -} from '@backstage/core'; -import ExampleFetchComponent from '../ExampleFetchComponent'; - -const ExampleComponent: FC<{}> = () => ( - -
- - -
- - - A description of your plugin goes here. - - - - - - All content should be wrapped in a card like this. - - - - - - - - - -); - -export default ExampleComponent; diff --git a/plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx b/plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx deleted file mode 100644 index c2139befec..0000000000 --- a/plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React, { FC } from 'react'; -import { makeStyles } from '@material-ui/core/styles'; -import { Table, TableColumn, Progress } from '@backstage/core'; -import Alert from '@material-ui/lab/Alert'; -import { useAsync } from 'react-use'; - -const useStyles = makeStyles({ - avatar: { - height: 32, - width: 32, - borderRadius: '50%', - }, -}); - -type User = { - gender: string; // "male" - name: { - title: string; // "Mr", - first: string; // "Duane", - last: string; // "Reed" - }; - location: object; // {street: {number: 5060, name: "Hickory Creek Dr"}, city: "Albany", state: "New South Wales",…} - email: string; // "duane.reed@example.com" - login: object; // {uuid: "4b785022-9a23-4ab9-8a23-cb3fb43969a9", username: "blackdog796", password: "patch",…} - dob: object; // {date: "1983-06-22T12:30:23.016Z", age: 37} - registered: object; // {date: "2006-06-13T18:48:28.037Z", age: 14} - phone: string; // "07-2154-5651" - cell: string; // "0405-592-879" - id: { - name: string; // "TFN", - value: string; // "796260432" - }; - picture: { medium: string }; // {medium: "https://randomuser.me/api/portraits/men/95.jpg",…} - nat: string; // "AU" -}; - -type DenseTableProps = { - users: User[]; -}; - -export const DenseTable: FC = ({ users }) => { - const classes = useStyles(); - - const columns: TableColumn[] = [ - { title: 'Avatar', field: 'avatar' }, - { title: 'Name', field: 'name' }, - { title: 'Email', field: 'email' }, - { title: 'Nationality', field: 'nationality' }, - ]; - - const data = users.map(user => { - return { - avatar: ( - {user.name.first} - ), - name: `${user.name.first} ${user.name.last}`, - email: user.email, - nationality: user.nat, - }; - }); - - return ( - - ); -}; - -const ExampleFetchComponent: FC<{}> = () => { - const { value, loading, error } = useAsync(async (): Promise => { - const response = await fetch('https://randomuser.me/api/?results=20'); - const data = await response.json(); - return data.results; - }, []); - - if (loading) { - return ; - } else if (error) { - return {error.message}; - } - - return ; -}; - -export default ExampleFetchComponent; diff --git a/plugins/techdocs/src/plugin.ts b/plugins/techdocs/src/plugin.ts index 29f9f8b085..23bfa3b714 100644 --- a/plugins/techdocs/src/plugin.ts +++ b/plugins/techdocs/src/plugin.ts @@ -30,16 +30,16 @@ */ import { createPlugin, createRouteRef } from '@backstage/core'; -import ExampleComponent from './components/ExampleComponent'; +import { Reader } from './reader/components/Reader'; export const rootRouteRef = createRouteRef({ - path: '/techdocs', - title: 'techdocs', + path: '/docs', + title: 'Docs', }); export const plugin = createPlugin({ id: 'techdocs', register({ router }) { - router.addRoute(rootRouteRef, ExampleComponent); + router.addRoute(rootRouteRef, Reader); }, }); diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx new file mode 100644 index 0000000000..e306a72424 --- /dev/null +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -0,0 +1,816 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { useShadowDom } from '..'; + +const mockHtml: string = ` + + + + + + + + + + + + + + + + + + + + + + + + + + Download boilerplate - MkDocs Material Boilerplate + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+ +
+ + + + + + + + + + + + +
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+ + +
+
+
+ + +
+
+ + + + + + + + + + +

Download boilerplate

+

Git clone

+
git clone https://github.com/peaceiris/mkdocs-material-boilerplate.git
+cd mkdocs-material-boilerplate
+
+ + +

Download zip

+
wget 'https://github.com/peaceiris/mkdocs-material-boilerplate/archive/master.zip'
+unzip master.zip
+cd mkdocs-material-boilerplate-master
+
+ + +

👉 Click me to download zip

+ + + + + + + +
+
+
+
+ + + + +
+ + + + + + + + + + + +`; + +export const Reader = () => { + const shadowDomRef = useShadowDom(); + + React.useEffect(() => { + const divElement = shadowDomRef.current; + if (!divElement?.shadowRoot) { + return; + } + divElement.shadowRoot.innerHTML = mockHtml; + }, [shadowDomRef]); + + return ( + <> +

Shadow DOM should be underneath

+
+ + ); +}; diff --git a/plugins/techdocs/src/components/ExampleComponent/index.ts b/plugins/techdocs/src/reader/hooks/index.ts similarity index 92% rename from plugins/techdocs/src/components/ExampleComponent/index.ts rename to plugins/techdocs/src/reader/hooks/index.ts index e785d45082..f66c5303ed 100644 --- a/plugins/techdocs/src/components/ExampleComponent/index.ts +++ b/plugins/techdocs/src/reader/hooks/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { default } from './ExampleComponent'; +export { useShadowDom } from './shadowDom'; diff --git a/plugins/techdocs/src/reader/hooks/shadowDom.test.tsx b/plugins/techdocs/src/reader/hooks/shadowDom.test.tsx new file mode 100644 index 0000000000..1d58bf26e7 --- /dev/null +++ b/plugins/techdocs/src/reader/hooks/shadowDom.test.tsx @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { renderWithEffects } from '@backstage/test-utils'; +import { useShadowDom } from './shadowDom'; + +const ComponentWithoutHook = () => { + return
; +}; + +const ComponentWithHook = () => { + const ref = useShadowDom(); + return
; +}; + +describe('useShadowDom', () => { + it('does not create a Shadow DOM instance', async () => { + const rendered = await renderWithEffects(); + + const divElement = rendered.getByTestId('shadow-dom'); + expect(divElement.shadowRoot).not.toBeInstanceOf(ShadowRoot); + }); + + it('create a Shadow DOM instance', async () => { + const rendered = await renderWithEffects(); + + const divElement = rendered.getByTestId('shadow-dom'); + expect(divElement.shadowRoot).toBeInstanceOf(ShadowRoot); + }); +}); diff --git a/plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx b/plugins/techdocs/src/reader/hooks/shadowDom.ts similarity index 56% rename from plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx rename to plugins/techdocs/src/reader/hooks/shadowDom.ts index 7fecdc6f11..5edb8f4e4d 100644 --- a/plugins/techdocs/src/components/ExampleFetchComponent/ExampleFetchComponent.test.tsx +++ b/plugins/techdocs/src/reader/hooks/shadowDom.ts @@ -14,15 +14,17 @@ * limitations under the License. */ -import React from 'react'; -import { render } from '@testing-library/react'; -import mockFetch from 'jest-fetch-mock'; -import ExampleFetchComponent from './ExampleFetchComponent'; +import { useEffect, useRef } from 'react'; +import type { RefObject } from 'react'; -describe('ExampleFetchComponent', () => { - it('should render', async () => { - mockFetch.mockResponse(() => new Promise(() => {})); - const rendered = render(); - expect(await rendered.findByTestId('progress')).toBeInTheDocument(); - }); -}); +type IShadowDOMRefObject = RefObject; +export const useShadowDom: () => IShadowDOMRefObject = () => { + const ref: IShadowDOMRefObject = useRef(null); + + useEffect(() => { + const divElement = ref.current; + divElement?.attachShadow({ mode: 'open' }); + }, [ref]); + + return ref; +}; diff --git a/plugins/techdocs/src/components/ExampleFetchComponent/index.ts b/plugins/techdocs/src/reader/index.tsx similarity index 92% rename from plugins/techdocs/src/components/ExampleFetchComponent/index.ts rename to plugins/techdocs/src/reader/index.tsx index 28482f9fe1..72b51bd0dc 100644 --- a/plugins/techdocs/src/components/ExampleFetchComponent/index.ts +++ b/plugins/techdocs/src/reader/index.tsx @@ -14,4 +14,4 @@ * limitations under the License. */ -export { default } from './ExampleFetchComponent'; +export * from './hooks'; From 8e7cec6770995e35054462d40b555dab88c70832 Mon Sep 17 00:00:00 2001 From: danztran Date: Thu, 25 Jun 2020 16:33:00 +0700 Subject: [PATCH 8/9] plugins/catalog-backend: add gitlab reader processor --- .../src/ingestion/LocationReaders.ts | 2 + .../processors/GitlabReaderProcessor.ts | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index eee16ea0de..0fe87440fb 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -26,6 +26,7 @@ import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEn import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; +import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; import { @@ -56,6 +57,7 @@ export class LocationReaders implements LocationReader { return [ new FileReaderProcessor(), new GithubReaderProcessor(), + new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), new LocationRefProcessor(), diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts new file mode 100644 index 0000000000..211f325afe --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabReaderProcessor.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'node-fetch'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class GitlabReaderProcessor implements LocationProcessor { + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab') { + return false; + } + + try { + const url = this.buildRawUrl(location.target); + + const response = await fetch(url.toString()); + + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + throw result.notFoundError(location, message); + } + } else { + throw result.generalError(location, message); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + + return true; + } + + // Converts + // from: https://gitlab.example.com/a/b/blob/master/c.yaml + // to: https://gitlab.example.com/a/b/raw/master/c.yaml + private buildRawUrl(target: string): URL { + try { + const url = new URL(target); + + const [ + empty, + userOrOrg, + repoName, + blobKeyword, + ...restOfPath + ] = url.pathname.split('/'); + + if ( + empty !== '' || + userOrOrg === '' || + repoName === '' || + blobKeyword !== 'blob' || + !restOfPath.join('/').match(/\.yaml$/) + ) { + throw new Error('Wrong Gitlab URL'); + } + + // Replace 'blob' with 'raw' + url.pathname = [empty, userOrOrg, repoName, 'raw', ...restOfPath].join( + '/', + ); + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } +} From 0ec6200c145e6017de0d2d780881fcae9437afb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20=C3=85lund?= Date: Thu, 25 Jun 2020 12:22:13 +0200 Subject: [PATCH 9/9] Update README with catalog and new features (#1449) * Update README with catalog and new features * Update README.md --- README.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c1ca77c2cc..2f3740c36d 100644 --- a/README.md +++ b/README.md @@ -10,29 +10,39 @@ ## What is Backstage? -Backstage is an open platform for building developer portals. - -The philosophy behind Backstage is simple: Don't expose your engineers to the full complexity of your infrastructure tooling. Engineers should be shipping code — not figuring out a whole new toolset every time they want to implement the basics. Backstage allows you add "stuff" (tooling, services, features, etc.) by adding a plugin, instead of building a new tool. This saves you work and avoids the need of your team to learn how to use and support yet another tool. +[Backstage](https://backstage.io/) is an open platform for building developer portals. It’s based on the developer portal we’ve been using internally at Spotify for over four years. Backstage can be as simple as a services catalog or as powerful as the UX layer for your entire tech infrastructure. For more information go to [backstage.io](https://backstage.io) or join our [Discord chatroom](https://discord.gg/EBHEGzX). -## What problem does Backstage solve? +### Features +* Create and manage all of your organization’s software and microservices in one place +* Services catalog keeps track of all software and its ownership +* Visualizations provide information about your backend services and tooling, and help you monitor them +* A unified method for managing microservices offers both visibility and control +* Preset templates allow engineers to quickly create microservices in a standardized way ([coming soon](https://github.com/spotify/backstage/milestone/11)) +* Centralized, full-featured technical documentation with integrated tooling that makes it easy for developers to set up, publish, and maintain alongside their code ([coming soon](https://github.com/spotify/backstage/milestone/15)) -As companies grow, their infrastructure systems get messier. Backstage unifies all your infrastructure tooling, services, and documentation with a single, consistent UI. +### Benefits +* For engineering managers, it allows you to maintain standards and best practices across the organization, and can help you manage your whole tech ecosystem, from migrations to test certification. +* For end users (developers), it makes it fast and simple to build software components in a standardized way, and it provides a central place to manage all projects and documentation. +* For platform engineers, it enables extensibility and scalability by letting you easily integrate new tools and services (via plugins), as well as extending the functionality of existing ones. +* For everyone, it’s a single, consistent experience that ties all your infrastructure tooling, resources, standards, owners, contributors, and administrators together in one place. -This blog post provides more examples of how Backstage is used inside Spotify: +## Backstage Service Catalog (alpha) -https://labs.spotify.com/2020/03/17/what-the-heck-is-backstage-anyway/ +The Backstage Service Catalog — actually, a software catalog, since it includes more than just services — is a centralized system that keeps track of ownership and metadata for all the software in your ecosystem (services, websites, libraries, data pipelines, etc). The catalog is built around the concept of [metadata yaml files](https://github.com/spotify/backstage/blob/master/docs/architecture-decisions/adr002-default-catalog-file-format.md#format) stored together with the code, which are then harvested and visualized in Backstage. -https://backstage.io/demos +![servce-catalog](https://backstage.io/blog/assets/6/header.png) + +We have also found that the service catalog is a great way to organise the infrastructure tools you use to manage the software as well. This is how Backstage creates one developer portal for all your tools. Rather than asking teams to jump between different infrastructure UI’s (and incurring additional cognitive overhead each time they make a context switch), most of these tools can be organised around the entities in the catalog. ## Project roadmap We created Backstage about 4 years ago. While our internal version of Backstage has had the benefit of time to mature and evolve, the first iteration of our open source version is still nascent. We are envisioning three phases of the project and we have already begun work on various aspects of these phases: -- 🐣 **Phase 1:** Extensible frontend platform (Done ✅) - You will be able to easily create a single consistent UI layer for your internal infrastructure and tools. A set of reusable UX patterns and components help ensure a consistent experience between tools. +- 🐣 **Phase 1:** Extensible frontend platform (Done ✅) - You will be able to easily create a single consistent UI layer for your internal infrastructure and tools. A set of reusable [UX patterns and components](http://storybook.backstage.io) help ensure a consistent experience between tools. -- 🐢 **Phase 2:** Manage your stuff ([current focus](https://backstage.io/blog/2020/05/22/phase-2-service-catalog)) - Manage anything from microservices to software components to infrastructure and your service catalog. Regardless of whether you want to create a new library, view service deployment status in Kubernetes, or check the test coverage for a website -- Backstage will provide all of those tools - and many more - in a single developer portal. +- 🐢 **Phase 2:** Service Catalog ([alpha released](https://backstage.io/blog/2020/06/22/backstage-service-catalog-alpha)) - With a single catalog, Backstage makes it easy for a team to manage ten services — and makes it possible for your company to manage thousands of them. Developers can get a uniform overview of all their software and related resources, regardless of how and where they are running, as well as an easy way to onboard and manage those resources. - 🐇 **Phase 3:** Ecosystem (later) - Everyone's infrastructure stack is different. By fostering a vibrant community of contributors we hope to provide an ecosystem of Open Source plugins/integrations that allows you to pick the tools that match your stack. @@ -46,14 +56,12 @@ The Backstage platform consists of a number of different components: - **app** - Main web application that users interact with. It's built up by a number of different _Plugins_. This repo contains an example implementation of an app (located in `packages/example-app`) and you can easily get started with your own app by [creating one](docs/create-an-app.md). - [**plugins**](https://github.com/spotify/backstage/tree/master/plugins) - Each plugin is treated as a self-contained web app and can include almost any type of content. Plugins all use a common set of platform API's and reusable UI components. Plugins can fetch data either from the _backend_ or through any RESTful API exposed through the _proxy_. -- [**backend**](https://github.com/spotify/backstage/tree/master/packages/backend) - GraphQL aggregation service that holds the model of your software ecosystem, including organisational information and what team owns what software. The backend also has a Plugin model for extending its graph. +- [**service catalog**](https://github.com/spotify/backstage/tree/master/packages/backend) - Service that holds the model of your software ecosystem, including organisational information and what team owns what software. The backend also has a Plugin model for extending its graph. - **proxy** \* - Terminates HTTPS and exposes any RESTful API to Plugins. -- **identity** \* - A backend service that holds your organisation's metadata. +- **identity** - A backend service that holds your organisation's metadata. _\* not yet released_ -![overview](backstage_overview.png) - ## Getting started To run a Backstage app, you will need to have the following installed: @@ -100,6 +108,7 @@ We would love your help in building Backstage! See [CONTRIBUTING](CONTRIBUTING.m - [FAQ](docs/FAQ.md) - Frequently Asked Questions - [Code of Conduct](CODE_OF_CONDUCT.md) - This is how we roll - [Blog](https://backstage.io/blog/) - Announcements and updates +- [Newsletter](https://mailchi.mp/spotify/backstage-community) - Give us a star ⭐️ - If you are using Backstage or think it is an interesting project, we would love a star ❤️ Or, if you are an open source developer and are interested in joining our team, please reach out to [foss-opportunities@spotify.com ](mailto:foss-opportunities@spotify.com)