From c4d1cf345d0aa2b32e2f1a48d7aec7be6da1245e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 19 Feb 2021 11:03:42 +0100 Subject: [PATCH 01/66] scaffolder-backend: added test template for beta1 Co-authored-by: Johan Haals Signed-off-by: Johan Haals --- .../test-template-v2/template.yaml | 130 ++++++++++++++++++ .../{{cookiecutter.name}}/catalog-info.yaml | 8 ++ 2 files changed, 138 insertions(+) create mode 100644 plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml create mode 100644 plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml new file mode 100644 index 0000000000..f456093678 --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -0,0 +1,130 @@ +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: test-template + title: Test Template + description: Testing out the new template schema +spec: + owner: backstage/techdocs-core + type: service + + parameters: + title: Fill in some BS params + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:widget: textarea + ui:autofocus: true + ui:options: + rows: 5 + storePath: # provides {owner, repository, providerUrl} + # $ref: '#/definitions/StorePath' + title: Store Path + type: string + description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters + + steps: + - id: prepare + name: Prepare + action: legacy:prepare + parameters: + protocol: file + url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 + - id: template + name: Template + action: legacy:template + parameters: + templater: cookiecutter + values: + name: '{{ parameters.name }}' + - id: publish + name: Publish + action: legacy:publish + parameters: + values: + name: '{{ parameters.name }}' + storePath: '{{ parameters.storePath }}' + owner: kungen # not use + - id: register + name: Register + action: catalog:register + parameters: + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + + output: + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + entityRef: '{{ steps.register.entityRef }}' + + +# TODO: +# 1. [x] entity beta2 type & schema +# 2. [x] endpoint for template params + frontend refactor +# 3. [ ] implement new actions that work well with new steps format, e.g. prepare+template = fetch +# 4. [ ] external API for registering custom actions +# 5. [x] implement uiSchema split +# 6. [ ] document beta2 entity schema +# 6. [ ] user documentation for beta2 + +--- + +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: test-template-v2 + title: Test Template v2 + description: Testing out the new template schema +spec: + owner: backstage/techdocs-core + type: service + + parameters: + title: Fill in some BS params + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:widget: textarea + ui:autofocus: true + ui:options: + rows: 5 + storePath: # provides {owner, repository, providerUrl} + # $ref: '#/definitions/StorePath' + title: Store Path + type: string + description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters + + steps: + - id: fetch + name: Fetch + action: fetch + parameters: + protocol: file + url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 + templater: + type: cookiecutter + values: + name: '{{ parameters.name }}' + - id: publish + name: Publish + action: legacy:publish + parameters: + values: + name: '{{ parameters.name }}' + storePath: '{{ parameters.storePath }}' + owner: kungen # not use + - id: register + name: Register + action: catalog:register + parameters: + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + + output: + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + entityRef: '{{ steps.register.entityRef }}' diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml new file mode 100644 index 0000000000..dd1e0ebd09 --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml @@ -0,0 +1,8 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: {{cookiecutter.name | jsonify}} +spec: + type: website + lifecycle: experimental + owner: guest From 0d44305a4e165402e2da55b41e4769e0dc74dfd6 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 23 Feb 2021 10:34:29 +0100 Subject: [PATCH 02/66] chore: commit some more mock sample data Signed-off-by: Johan Haals --- .../test-template-v2/template.yaml | 33 ++++++++++--------- .../catalog-info.yaml | 0 2 files changed, 18 insertions(+), 15 deletions(-) rename plugins/scaffolder-backend/sample-templates/test-template-v2/{{{cookiecutter.name}} => template}/catalog-info.yaml (100%) diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index f456093678..0ac7493895 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -101,24 +101,27 @@ spec: description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters steps: - - id: fetch - name: Fetch - action: fetch - parameters: - protocol: file - url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 - templater: - type: cookiecutter - values: - name: '{{ parameters.name }}' - - id: publish - name: Publish - action: legacy:publish + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter parameters: + url: ./template + targetDir: blob values: name: '{{ parameters.name }}' - storePath: '{{ parameters.storePath }}' - owner: kungen # not use + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + parameters: + url: https://github.com/aeothgiaetgh/aetkijhahte/tree/master/docs/template + - id: publish + name: Publish + action: publish + parameters: + name: '{{ parameters.name }}' + # storePath: github.com?owner=spotify&repo=name + storePath: '{{ parameters.storePath }}' - id: register name: Register action: catalog:register diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template/catalog-info.yaml similarity index 100% rename from plugins/scaffolder-backend/sample-templates/test-template-v2/{{cookiecutter.name}}/catalog-info.yaml rename to plugins/scaffolder-backend/sample-templates/test-template-v2/template/catalog-info.yaml From 59769bad27b9d759bc8c346cf3a85c0550206dd0 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 23 Feb 2021 15:27:35 +0100 Subject: [PATCH 03/66] feat: starting to add the RepoUrlPicker with integrations config Signed-off-by: Johan Haals --- app-config.yaml | 3 + .../test-template-v2/template.yaml | 129 +++++++++--------- plugins/scaffolder/src/api.ts | 24 ++++ .../MultistepJsonForm/MultistepJsonForm.tsx | 6 + .../components/TemplatePage/TemplatePage.tsx | 4 + .../components/fields/RepoUrlPicker/index.tsx | 95 +++++++++++++ .../scaffolder/src/components/fields/index.ts | 16 +++ .../widgets/StorePathPicker/index.tsx | 89 ++++++++++++ plugins/scaffolder/src/plugin.ts | 11 +- 9 files changed, 312 insertions(+), 65 deletions(-) create mode 100644 plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx create mode 100644 plugins/scaffolder/src/components/fields/index.ts create mode 100644 plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx diff --git a/app-config.yaml b/app-config.yaml index f68f5a30a2..be7391707e 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -248,6 +248,9 @@ catalog: - type: file target: ../catalog-model/examples/acme-corp.yaml + - type: file + target: ../../plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml + scaffolder: github: token: diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 0ac7493895..08432e6502 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -1,63 +1,63 @@ -apiVersion: backstage.io/v1beta2 -kind: Template -metadata: - name: test-template - title: Test Template - description: Testing out the new template schema -spec: - owner: backstage/techdocs-core - type: service +# apiVersion: backstage.io/v1beta2 +# kind: Template +# metadata: +# name: test-template +# title: Test Template +# description: Testing out the new template schema +# spec: +# owner: backstage/techdocs-core +# type: service - parameters: - title: Fill in some BS params - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:widget: textarea - ui:autofocus: true - ui:options: - rows: 5 - storePath: # provides {owner, repository, providerUrl} - # $ref: '#/definitions/StorePath' - title: Store Path - type: string - description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters +# parameters: +# title: Fill in some BS params +# required: +# - name +# properties: +# name: +# title: Name +# type: string +# description: Unique name of the component +# ui:widget: textarea +# ui:autofocus: true +# ui:options: +# rows: 5 +# storePath: # provides {owner, repository, providerUrl} +# # $ref: '#/definitions/StorePath' +# title: Store Path +# type: string +# description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters - steps: - - id: prepare - name: Prepare - action: legacy:prepare - parameters: - protocol: file - url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 - - id: template - name: Template - action: legacy:template - parameters: - templater: cookiecutter - values: - name: '{{ parameters.name }}' - - id: publish - name: Publish - action: legacy:publish - parameters: - values: - name: '{{ parameters.name }}' - storePath: '{{ parameters.storePath }}' - owner: kungen # not use - - id: register - name: Register - action: catalog:register - parameters: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' +# steps: +# - id: prepare +# name: Prepare +# action: legacy:prepare +# parameters: +# protocol: file +# url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 +# - id: template +# name: Template +# action: legacy:template +# parameters: +# templater: cookiecutter +# values: +# name: '{{ parameters.name }}' +# - id: publish +# name: Publish +# action: legacy:publish +# parameters: +# values: +# name: '{{ parameters.name }}' +# storePath: '{{ parameters.storePath }}' +# owner: kungen # not use +# - id: register +# name: Register +# action: catalog:register +# parameters: +# catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' - output: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' - entityRef: '{{ steps.register.entityRef }}' +# output: +# catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' +# entityRef: '{{ steps.register.entityRef }}' # TODO: @@ -90,15 +90,17 @@ spec: title: Name type: string description: Unique name of the component - ui:widget: textarea ui:autofocus: true ui:options: rows: 5 - storePath: # provides {owner, repository, providerUrl} - # $ref: '#/definitions/StorePath' - title: Store Path + repoUrl: + title: Repository Location type: string description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com steps: - id: fetch-base @@ -115,13 +117,16 @@ spec: action: fetch:plain parameters: url: https://github.com/aeothgiaetgh/aetkijhahte/tree/master/docs/template + - id: publish name: Publish - action: publish + action: publish:github parameters: + allowedHosts: ['ghe.mycompany.com'] name: '{{ parameters.name }}' # storePath: github.com?owner=spotify&repo=name storePath: '{{ parameters.storePath }}' + - id: register name: Register action: catalog:register diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index ac165e021a..2be4caa81f 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -20,8 +20,10 @@ import { createApiRef, DiscoveryApi, Observable, + ConfigApi, IdentityApi, } from '@backstage/core'; +import { ScmIntegrations } from '@backstage/integration'; import ObservableImpl from 'zen-observable'; import { ScaffolderTask, Status } from './types'; @@ -66,6 +68,10 @@ export interface ScaffolderApi { getTask(taskId: string): Promise; + getIntegrationsList(options: { + allowedHosts: string[]; + }): Promise<{ type: string; title: string; host: string }[]>; + streamLogs({ taskId, after, @@ -77,13 +83,31 @@ export interface ScaffolderApi { export class ScaffolderClient implements ScaffolderApi { private readonly discoveryApi: DiscoveryApi; private readonly identityApi: IdentityApi; + private readonly configApi: ConfigApi; constructor(options: { discoveryApi: DiscoveryApi; identityApi: IdentityApi; + configApi: ConfigApi; }) { this.discoveryApi = options.discoveryApi; this.identityApi = options.identityApi; + this.configApi = options.configApi; + } + + async getIntegrationsList(options: { allowedHosts: string[] }) { + const integrations = ScmIntegrations.fromConfig( + this.configApi.getConfig('integrations'), + ); + + return [ + ...integrations.azure.list(), + ...integrations.bitbucket.list(), + ...integrations.github.list(), + ...integrations.gitlab.list(), + ] + .map(c => ({ type: c.type, title: c.title, host: c.config.host })) + .filter(c => options.allowedHosts.includes(c.host)); } async getTemplateParameterSchema( diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx index eb51e86d59..b33bd82ffa 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx +++ b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx @@ -45,6 +45,8 @@ type Props = { onChange: (e: IChangeEvent) => void; onReset: () => void; onFinish: () => void; + widgets?: FormProps['widgets']; + fields?: FormProps['fields']; }; export const MultistepJsonForm = ({ @@ -53,6 +55,8 @@ export const MultistepJsonForm = ({ onChange, onReset, onFinish, + fields, + widgets, }: Props) => { const [activeStep, setActiveStep] = useState(0); @@ -77,6 +81,8 @@ export const MultistepJsonForm = ({ noHtml5Validate formData={formData} onChange={onChange} + fields={fields} + widgets={widgets} onSubmit={e => { if (e.errors.length === 0) handleNext(); }} diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 1c8a662acb..282154c1e2 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -33,6 +33,7 @@ import { useAsync } from 'react-use'; import { scaffolderApiRef } from '../../api'; import { rootRouteRef } from '../../routes'; import { MultistepJsonForm } from '../MultistepJsonForm'; +import { RepoUrlPicker } from '../fields'; const useTemplateParameterSchema = (templateName: string) => { const scaffolderApi = useApi(scaffolderApiRef); @@ -53,7 +54,9 @@ const storePathValidator = ( errors: FormValidation, ) => { const { storePath } = formData; + if (!storePath) { + errors.storePath.addError('Store path is required and not present'); return errors; } @@ -131,6 +134,7 @@ export const TemplatePage = () => { { + const api = useApi(scaffolderApiRef); + const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; + + const { value: integrations, loading } = useAsync(async () => { + return await api.getIntegrationsList({ allowedHosts }); + }); + + const [hostname, setHostname] = useState(''); + const [owner, setOwner] = useState(''); + const [repo, setRepo] = useState(''); + + const updateHostname = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + setHostname(evt.target.value as string), + [setHostname], + ); + + const updateOwner = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + setOwner(evt.target.value as string), + [setOwner], + ); + + const updateRepo = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + setRepo(evt.target.value as string), + [setRepo], + ); + + useEffect(() => { + if (hostname === '' && integrations?.length) { + setHostname(integrations[0].host); + } + }, [integrations, hostname]); + + useEffect(() => { + const params = new URLSearchParams(); + params.set('owner', owner); + params.set('repo', repo); + + onChange(`${hostname}?${params.toString()}`); + }, [hostname, owner, repo, onChange]); + + if (loading) { + return ; + } + + return ( + <> + Repository Location + + {integrations! + .filter(i => allowedHosts?.includes(i.host)) + .map(({ host, title }) => ( + + {title} + + ))} + + + + + ); +}; diff --git a/plugins/scaffolder/src/components/fields/index.ts b/plugins/scaffolder/src/components/fields/index.ts new file mode 100644 index 0000000000..b0f3df61d9 --- /dev/null +++ b/plugins/scaffolder/src/components/fields/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { RepoUrlPicker } from './RepoUrlPicker'; diff --git a/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx b/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx new file mode 100644 index 0000000000..87c00dbf98 --- /dev/null +++ b/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx @@ -0,0 +1,89 @@ +/* + * Cop +yright 2021 Spotify AB + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState, useCallback, useEffect } from 'react'; +import { Field, Widget } from '@rjsf/core'; +import { useApi, Progress } from '@backstage/core'; +import { scaffolderApiRef } from '../../../api'; +import { useAsync } from 'react-use'; +import TextField from '@material-ui/core/TextField'; +import MenuItem from '@material-ui/core/MenuItem'; +import InputLabel from '@material-ui/core/InputLabel'; + +import { Typography } from '@material-ui/core'; +import { rest } from 'msw/lib/types'; + +export const StorePathPicker: Field = ({ + options, + onChange, + rawErrors, + formContext, + uiSchema +}) => { + const api = useApi(scaffolderApiRef); + console.log(uiSchema) + const allowedHosts = uiSchema['ui:Options'].allowedHosts as string[]; + + const { value: integrations, loading } = useAsync(async () => { + return await api.getIntegrationsList({ allowedHosts }); + }); + + const [hostname, setHostname] = useState(''); + const [organization, setOrganization] = useState(''); + const [repositoryName, setRepositoryName] = useState(''); + + const [organization, setOrganization] = useState(''); + const [repositoryName, setRepositoryName] = useState(''); + + const updateHostname = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + setHostname(evt.target.value as string), + [setHostname], + ); + + useEffect(() => { + if (hostname === '' && integrations?.length) { + setHostname(integrations[0].host); + } + }, [integrations, hostname]); + + if (loading) { + return ; + } + + return ( + <> + Repository Location + + {integrations! + .filter(i => allowedHosts?.includes(i.host)) + .map(({ host, title }) => ( + + {title} + () =?> Setsets + ))} + + + + + ); +}; diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index a6ba1a9899..8ba03f4682 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -19,6 +19,7 @@ import { createApiFactory, discoveryApiRef, identityApiRef, + configApiRef, createRoutableExtension, } from '@backstage/core'; import { rootRouteRef } from './routes'; @@ -29,9 +30,13 @@ export const scaffolderPlugin = createPlugin({ apis: [ createApiFactory({ api: scaffolderApiRef, - deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, - factory: ({ discoveryApi, identityApi }) => - new ScaffolderClient({ discoveryApi, identityApi }), + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + configApi: configApiRef, + }, + factory: ({ discoveryApi, identityApi, configApi }) => + new ScaffolderClient({ discoveryApi, identityApi, configApi }), }), ], routes: { From b2ce7f18c04905c1773dc7624a8c186d98a05c4e Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Feb 2021 11:23:25 +0100 Subject: [PATCH 04/66] feat: added recursive validation for the repository url picker component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johan Haals Co-authored-by: Fredrik Adelรถw Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../MultistepJsonForm/MultistepJsonForm.tsx | 59 ++++++++-------- .../TemplatePage/TemplatePage.test.tsx | 37 +++++++++- .../components/TemplatePage/TemplatePage.tsx | 68 ++++++++++++++++-- .../components/fields/RepoUrlPicker/index.tsx | 70 +++++++++++-------- 4 files changed, 170 insertions(+), 64 deletions(-) diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx index b33bd82ffa..876d6dfa66 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx +++ b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx @@ -71,34 +71,37 @@ export const MultistepJsonForm = ({ return ( <> - {steps.map(({ title, schema, ...formProps }) => ( - - - {title} - - -
{ - if (e.errors.length === 0) handleNext(); - }} - {...formProps} - {...transformSchemaToProps(schema)} - > - - -
- - - ))} + {steps.map(({ title, schema, ...formProps }) => { + return ( + + + {title} + + +
{ + if (e.errors.length === 0) handleNext(); + }} + {...formProps} + {...transformSchemaToProps(schema)} + > + + +
+ + + ); + })} {activeStep === steps.length && ( diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 1d66e07c6a..0556dd1129 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -22,7 +22,7 @@ import { act } from 'react-dom/test-utils'; import { MemoryRouter, Route } from 'react-router'; import { ScaffolderApi, scaffolderApiRef } from '../../api'; import { rootRouteRef } from '../../routes'; -import { TemplatePage } from './TemplatePage'; +import { TemplatePage, createValidator } from './TemplatePage'; jest.mock('react-router-dom', () => { return { @@ -122,3 +122,38 @@ describe('TemplatePage', () => { expect(rendered.queryByText('This is root')).toBeInTheDocument(); }); }); + +describe('createValidator', () => { + it('should validate deep schema', () => { + const validator = createValidator({ + type: 'object', + properties: { + foo: { + type: 'object', + properties: { + bar: { + type: 'string', + 'ui:field': 'RepoUrlPicker', + }, + }, + }, + }, + }); + + const errors = { foo: { bar: { addError: jest.fn() } } }; + validator({ foo: { bar: 'github.com?owner=a' } }, errors as any); + expect(errors.foo.bar.addError).toHaveBeenCalledWith( + 'Incomplete repository location provided', + ); + jest.resetAllMocks(); + + validator({ foo: { bar: 'github.com?repo=b' } }, errors as any); + expect(errors.foo.bar.addError).toHaveBeenCalledWith( + 'Incomplete repository location provided', + ); + jest.resetAllMocks(); + + validator({ foo: { bar: 'github.com?owner=a&repo=b' } }, errors as any); + expect(errors.foo.bar.addError).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 282154c1e2..5d173173ee 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -34,6 +34,7 @@ import { scaffolderApiRef } from '../../api'; import { rootRouteRef } from '../../routes'; import { MultistepJsonForm } from '../MultistepJsonForm'; import { RepoUrlPicker } from '../fields'; +import { JsonObject } from '@backstage/config'; const useTemplateParameterSchema = (templateName: string) => { const scaffolderApi = useApi(scaffolderApiRef); @@ -49,12 +50,67 @@ const useTemplateParameterSchema = (templateName: string) => { return { schema: value, loading, error }; }; +function isObject(obj: unknown): obj is JsonObject { + return typeof obj === 'object' && obj !== null && !Array.isArray(obj); +} + +export const createValidator = (rootSchema: JsonObject) => { + function validate( + schema: JsonObject, + formData: JsonObject, + errors: FormValidation, + ) { + const schemaProps = schema.properties; + if (!isObject(schemaProps)) { + return; + } + + for (const [key, propData] of Object.entries(formData)) { + const propErrors = errors[key]; + + if (isObject(propData)) { + const propSchemaProps = schemaProps[key]; + if (isObject(propSchemaProps)) { + validate( + propSchemaProps, + propData as JsonObject, + propErrors as FormValidation, + ); + } + } else { + const propSchema = schemaProps[key]; + if ( + isObject(propSchema) && + propSchema['ui:field'] === 'RepoUrlPicker' + ) { + try { + const { host, searchParams } = new URL(`https://${propData}`); + if ( + !host || + !searchParams.get('owner') || + !searchParams.get('repo') + ) { + propErrors.addError('Incomplete repository location provided'); + } + } catch { + propErrors.addError('Unable to parse the Repository URL'); + } + } + } + } + } + + return (formData: JsonObject, errors: FormValidation) => { + validate(rootSchema, formData, errors); + return errors; + }; +}; + const storePathValidator = ( formData: { storePath?: string }, errors: FormValidation, ) => { const { storePath } = formData; - if (!storePath) { errors.storePath.addError('Store path is required and not present'); return errors; @@ -139,15 +195,19 @@ export const TemplatePage = () => { onReset={handleFormReset} onFinish={handleCreate} steps={schema.steps.map(step => { - // TODO: Using this workaround to keep storePath validation, but we should replace - // it with a custom store path selection widget + // TODO: Can delete this function when the migration from v1 to v2 beta is completed + // And just have the default validator for all fields. if ((step.schema as any)?.properties?.storePath) { return { ...step, validate: (a, b) => storePathValidator(a, b), }; } - return step; + + return { + ...step, + validate: createValidator(step.schema), + }; })} /> diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx index cb7eec017d..a20f748d3a 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx @@ -14,16 +14,18 @@ * limitations under the License. */ import React, { useState, useCallback, useEffect } from 'react'; -import { Field, Widget } from '@rjsf/core'; +import { Field } from '@rjsf/core'; import { useApi, Progress } from '@backstage/core'; import { scaffolderApiRef } from '../../../api'; import { useAsync } from 'react-use'; -import TextField from '@material-ui/core/TextField'; -import MenuItem from '@material-ui/core/MenuItem'; - +import Select from '@material-ui/core/Select'; +import InputLabel from '@material-ui/core/InputLabel'; +import Input from '@material-ui/core/Input'; +import FormControl from '@material-ui/core/FormControl'; import { Typography } from '@material-ui/core'; +import { rest } from 'msw/lib/types'; -export const RepoUrlPicker: Field = ({ onChange, uiSchema }) => { +export const RepoUrlPicker: Field = ({ onChange, uiSchema, ...rest }) => { const api = useApi(scaffolderApiRef); const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; @@ -31,14 +33,16 @@ export const RepoUrlPicker: Field = ({ onChange, uiSchema }) => { return await api.getIntegrationsList({ allowedHosts }); }); - const [hostname, setHostname] = useState(''); + console.log(rest); + + const [host, setHost] = useState(''); const [owner, setOwner] = useState(''); const [repo, setRepo] = useState(''); - const updateHostname = useCallback( + const updateHost = useCallback( (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => - setHostname(evt.target.value as string), - [setHostname], + setHost(evt.target.value as string), + [setHost], ); const updateOwner = useCallback( @@ -54,18 +58,18 @@ export const RepoUrlPicker: Field = ({ onChange, uiSchema }) => { ); useEffect(() => { - if (hostname === '' && integrations?.length) { - setHostname(integrations[0].host); + if (host === '' && integrations?.length) { + setHost(integrations[0].host); } - }, [integrations, hostname]); + }, [integrations, host]); useEffect(() => { const params = new URLSearchParams(); params.set('owner', owner); params.set('repo', repo); - onChange(`${hostname}?${params.toString()}`); - }, [hostname, owner, repo, onChange]); + onChange(`${encodeURIComponent(host)}?${params.toString()}`); + }, [host, owner, repo, onChange]); if (loading) { return ; @@ -73,23 +77,27 @@ export const RepoUrlPicker: Field = ({ onChange, uiSchema }) => { return ( <> - Repository Location - - {integrations! - .filter(i => allowedHosts?.includes(i.host)) - .map(({ host, title }) => ( - - {title} - - ))} - - - + Repository Location + + Host + + + + Owner + + + + Repository + + ); }; From 459307b81d690c76f4ccfaa86d54bb6db59ecbd7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 16:22:59 +0100 Subject: [PATCH 05/66] scaffolder-backend: separate out TemplateActionRegistry and types Signed-off-by: Johan Haals --- .../src/scaffolder/stages/legacy.ts | 2 +- .../src/scaffolder/tasks/TaskWorker.ts | 2 +- .../tasks/TemplateActionRegistry.ts | 41 ++++++++++++++++++ .../src/scaffolder/tasks/TemplateConverter.ts | 42 ------------------- .../src/scaffolder/tasks/types.ts | 16 +++++++ .../scaffolder-backend/src/service/router.ts | 6 +-- 6 files changed, 61 insertions(+), 48 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index efba57a063..271618b981 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TemplateActionRegistry } from '../tasks/TemplateConverter'; +import { TemplateActionRegistry } from '../tasks/TemplateActionRegistry'; import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 9d0c0863aa..af8f85f371 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -21,7 +21,7 @@ import { JsonValue, JsonObject } from '@backstage/config'; import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; -import { TemplateActionRegistry } from './TemplateConverter'; +import { TemplateActionRegistry } from './TemplateActionRegistry'; import * as handlebars from 'handlebars'; type Options = { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts new file mode 100644 index 0000000000..9d6140fbfe --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TemplateAction } from './types'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; + +export class TemplateActionRegistry { + private readonly actions = new Map(); + + register(action: TemplateAction) { + if (this.actions.has(action.id)) { + throw new ConflictError( + `Template action with ID '${action.id}' has already been registered`, + ); + } + this.actions.set(action.id, action); + } + + get(actionId: string): TemplateAction { + const action = this.actions.get(actionId); + if (!action) { + throw new NotFoundError( + `Template action with ID '${actionId}' is not registered.`, + ); + } + return action; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index 7f76ac33d0..7d824a7d53 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -15,13 +15,8 @@ */ import { resolve as resolvePath, dirname } from 'path'; -import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { Logger } from 'winston'; -import { Writable } from 'stream'; - import { TaskSpec } from './types'; -import { ConflictError, NotFoundError } from '@backstage/backend-common'; import { getTemplaterKey, joinGitUrlPath, @@ -95,40 +90,3 @@ export function templateEntityToSpec( }, }; } - -type ActionContext = { - logger: Logger; - logStream: Writable; - - workspacePath: string; - parameters: { [name: string]: JsonValue }; - output(name: string, value: JsonValue): void; -}; - -type TemplateAction = { - id: string; - handler: (ctx: ActionContext) => Promise; -}; - -export class TemplateActionRegistry { - private readonly actions = new Map(); - - register(action: TemplateAction) { - if (this.actions.has(action.id)) { - throw new ConflictError( - `Template action with ID '${action.id}' has already been registered`, - ); - } - this.actions.set(action.id, action); - } - - get(actionId: string): TemplateAction { - const action = this.actions.get(actionId); - if (!action) { - throw new NotFoundError( - `Template action with ID '${actionId}' is not registered.`, - ); - } - return action; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 27a1b62d04..6e0ecd8600 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -15,6 +15,8 @@ */ import { JsonValue, JsonObject } from '@backstage/config'; +import { Logger } from 'winston'; +import { Writable } from 'stream'; export type Status = | 'open' @@ -112,3 +114,17 @@ export interface TaskStore { after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } + +export type ActionContext = { + logger: Logger; + logStream: Writable; + + workspacePath: string; + parameters: { [name: string]: JsonValue }; + output(name: string, value: JsonValue): void; +}; + +export type TemplateAction = { + id: string; + handler: (ctx: ActionContext) => Promise; +}; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 65b9039e27..cc483f3bd0 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -38,10 +38,8 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { - TemplateActionRegistry, - templateEntityToSpec, -} from '../scaffolder/tasks/TemplateConverter'; +import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; +import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateActionRegistry'; import { registerLegacyActions } from '../scaffolder/stages/legacy'; import { getWorkingDirectory } from './helpers'; import { From 0f64c7347429c765f508307beb6f312a93952f85 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 16:35:43 +0100 Subject: [PATCH 06/66] scaffolder-backend: added createTemporaryDirectory to action context Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/TaskWorker.ts | 15 +++++++++++++++ .../src/scaffolder/tasks/types.ts | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index af8f85f371..032fea6410 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -112,16 +112,31 @@ export class TaskWorker { const stepOutputs: { [name: string]: JsonValue } = {}; + // Keep track of all tmp dirs that are created by the action so we can remove them after + const tmpDirs = new Array(); + await action.handler({ logger: taskLogger, logStream: stream, parameters, workspacePath, + async createTemporaryDirectory() { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, output(name: string, value: JsonValue) { stepOutputs[name] = value; }, }); + // Remove all temporary directories that were created when executing the action + for (const tmpDir of tmpDirs) { + await fs.remove(tmpDir); + } + templateCtx.steps[step.id] = { output: stepOutputs }; await task.emitLog(`Finished step ${step.name}`, { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 6e0ecd8600..d7f2e6e51f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -122,6 +122,11 @@ export type ActionContext = { workspacePath: string; parameters: { [name: string]: JsonValue }; output(name: string, value: JsonValue): void; + + /** + * Creates a temporary directory for use by the action, which is then cleaned up automatically. + */ + createTemporaryDirectory(): Promise; }; export type TemplateAction = { From b90767926fd28593b7fd61a7ea349ee477d378cb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 22:53:24 +0100 Subject: [PATCH 07/66] scaffolder-backend: remove workspace after task execution is complete Signed-off-by: Johan Haals --- .../scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 032fea6410..4a09149bab 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -44,10 +44,11 @@ export class TaskWorker { } async runOneTask(task: Task) { + let workspacePath: string | undefined = undefined; try { const { actionRegistry } = this.options; - const workspacePath = path.join( + workspacePath = path.join( this.options.workingDirectory, await task.getWorkspaceName(), ); @@ -172,6 +173,10 @@ export class TaskWorker { await task.complete('failed', { error: { name: error.name, message: error.message }, }); + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } } } } From 86fc6a0360139e2d40dd5f318b80165354789750 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 22:55:57 +0100 Subject: [PATCH 08/66] scaffolder-backend: move v1 templater value injection into template converter Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/TemplateConverter.ts | 11 ++++++++++- plugins/scaffolder-backend/src/service/router.ts | 7 +------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index 7d824a7d53..f9e9abf925 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -16,6 +16,7 @@ import { resolve as resolvePath, dirname } from 'path'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import parseGitUrl from 'git-url-parse'; import { TaskSpec } from './types'; import { getTemplaterKey, @@ -26,7 +27,7 @@ import { export function templateEntityToSpec( template: TemplateEntityV1alpha1, - values: TemplaterValues, + inputValues: TemplaterValues, ): TaskSpec { const steps: TaskSpec['steps'] = []; @@ -42,6 +43,13 @@ export function templateEntityToSpec( } const templater = getTemplaterKey(template); + const values = { + ...inputValues, + destination: { + git: parseGitUrl(inputValues.storePath), + }, + } as TemplaterValues; + steps.push({ id: 'prepare', name: 'Prepare', @@ -81,6 +89,7 @@ export function templateEntityToSpec( }); return { + baseUrl: undefined, // not used by legacy actions values: {}, steps, output: { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index cc483f3bd0..db5ab28064 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -328,12 +328,7 @@ export async function createRouter( ) .post('/v2/tasks', async (req, res) => { const templateName: string = req.body.templateName; - const values: TemplaterValues = { - ...req.body.values, - destination: { - git: parseGitUrl(req.body.values.storePath), - }, - }; + const values: TemplaterValues = req.body.values; const template = await entityClient.findTemplate(templateName); let taskSpec; From c50b732b705fd6f1f6991dad06f21280b0692148 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 23:01:03 +0100 Subject: [PATCH 09/66] scaffolder-backend: add template source baseUrl to action context Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/TaskWorker.ts | 1 + .../src/scaffolder/tasks/types.ts | 6 ++++ .../scaffolder-backend/src/service/helpers.ts | 36 +++++++++++++++++++ .../scaffolder-backend/src/service/router.ts | 5 ++- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 4a09149bab..1e038ba65e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -117,6 +117,7 @@ export class TaskWorker { const tmpDirs = new Array(); await action.handler({ + baseUrl: task.spec.baseUrl, logger: taskLogger, logStream: stream, parameters, diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index d7f2e6e51f..42d8179a3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -45,6 +45,7 @@ export type DbTaskEventRow = { }; export type TaskSpec = { + baseUrl?: string; values: JsonObject; steps: Array<{ id: string; @@ -116,6 +117,11 @@ export interface TaskStore { } export type ActionContext = { + /** + * Base URL for the location of the task spec, typically the url of the source entity file. + */ + baseUrl?: string; + logger: Logger; logStream: Writable; diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index dd3d43c6d1..905d85c35d 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -18,6 +18,11 @@ import os from 'os'; import fs from 'fs-extra'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; +import { + Entity, + LOCATION_ANNOTATION, + SOURCE_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; export async function getWorkingDirectory( config: Config, @@ -42,3 +47,34 @@ export async function getWorkingDirectory( } return workingDirectory; } + +/** + * Gets the base URL of the entity location that points to the source location + * of the entity description within a repo. If there is not source location + * or if it has an invalid type, undefined will be returned instead. + * + * For file locations this will return a `file://` URL. + */ +export function getEntityBaseUrl(entity: Entity): string | undefined { + let location = entity.metadata.annotations?.[SOURCE_LOCATION_ANNOTATION]; + if (!location) { + location = entity.metadata.annotations?.[LOCATION_ANNOTATION]; + } + if (!location) { + return undefined; + } + + const [type, url] = location.split(/:(.+)/); + if (!url) { + return undefined; + } + + if (type === 'url') { + return url; + } else if (type === 'file') { + return `file://${url}`; + } + // Only url and file location are handled, as we otherwise don't know if + // what the url is pointing to makes sense to use as a baseUrl + return undefined; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index db5ab28064..79ae70d694 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -41,7 +41,7 @@ import { import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateActionRegistry'; import { registerLegacyActions } from '../scaffolder/stages/legacy'; -import { getWorkingDirectory } from './helpers'; +import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { InputError, NotFoundError, @@ -351,7 +351,10 @@ export async function createRouter( } } + const baseUrl = getEntityBaseUrl(template); + taskSpec = { + baseUrl, values, steps: template.spec.steps.map((step, index) => ({ ...step, From 2f404bd69dccbff5997ebdc916d0f57ac460f416 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 23 Feb 2021 23:11:49 +0100 Subject: [PATCH 10/66] scaffolder-backend: added built-in fetch:cookiecutter action Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/builtin.ts | 120 ++++++++++++++++++ .../scaffolder-backend/src/service/router.ts | 14 ++ 2 files changed, 134 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts new file mode 100644 index 0000000000..60e3ad5282 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -0,0 +1,120 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, isAbsolute } from 'path'; +import Docker from 'dockerode'; +import { TemplaterBuilder, TemplaterValues } from '../stages/templater'; +import { TemplateAction } from './types'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; + +export function createFetchCookiecutterAction(options: { + dockerClient: Docker; + urlReader: UrlReader; + integrations: ScmIntegrations; + templaters: TemplaterBuilder; +}): TemplateAction { + const { dockerClient, urlReader, templaters, integrations } = options; + + return { + id: 'fetch:cookiecutter', + async handler(ctx) { + ctx.logger.info('Fetching and then templating using cookiecutter'); + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + const templateContentsDir = resolvePath( + templateDir, + "{{cookiecutter and 'contents'}}", + ); + const resultDir = resolvePath(workDir, 'result'); + + const fetchUrl = ctx.parameters.url ?? '.'; + if (typeof fetchUrl !== 'string') { + throw new InputError( + `Invalid url parameter, expected string, got ${typeof fetchUrl}`, + ); + } + + // We handle both file locations and url ones + if (ctx.baseUrl?.startsWith('file://')) { + const basePath = ctx.baseUrl.slice('file://'.length); + if (isAbsolute(fetchUrl)) { + throw new InputError( + `Fetch URL may not be absolute for file locations, ${fetchUrl}`, + ); + } + const srcDir = resolvePath(basePath, '..', fetchUrl); + await fs.copy(srcDir, templateContentsDir); + } else { + let readUrl; + + if (ctx.baseUrl) { + const integration = integrations.byUrl(ctx.baseUrl); + if (!integration) { + throw new InputError( + `No integration found for location ${ctx.baseUrl}`, + ); + } + readUrl = integration.resolveUrl({ + url: fetchUrl, + base: ctx.baseUrl, + }); + } else { + // If we don't have a baseUrl, check if our provided fetch url is absolute + try { + readUrl = new URL(fetchUrl).toString(); + } catch { + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); + } + + const res = await urlReader.readTree(readUrl); + await res.dir({ targetDir: templateContentsDir }); + } + } + + const cookiecutter = templaters.get('cookiecutter'); + if (!cookiecutter) { + throw new Error('No cookiecutter templater available'); + } + + // Will execute the template in ./template and put the result in ./result + await cookiecutter.run({ + workspacePath: workDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.parameters.values as TemplaterValues, + }); + + // Finally move the template result into the task workspace + const targetPath = ctx.parameters.targetPath ?? './'; + if (typeof targetPath !== 'string') { + throw new InputError( + `Fetch action targetPath is not a string, got ${targetPath}`, + ); + } + const outputPath = resolvePath(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + await fs.copy(resultDir, outputPath); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 79ae70d694..3787d916b5 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -46,6 +46,7 @@ import { InputError, NotFoundError, PluginDatabaseManager, + UrlReader, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; import { @@ -53,6 +54,8 @@ import { TemplateEntityV1beta2, Entity, } from '@backstage/catalog-model'; +import { createFetchCookiecutterAction } from '../scaffolder/tasks/builtin'; +import { ScmIntegrations } from '@backstage/integration'; export interface RouterOptions { preparers: PreparerBuilder; @@ -61,6 +64,7 @@ export interface RouterOptions { logger: Logger; config: Config; + urlReader: UrlReader; dockerClient: Docker; database: PluginDatabaseManager; catalogClient: CatalogApi; @@ -93,6 +97,7 @@ export async function createRouter( publishers, logger: parentLogger, config, + urlReader, dockerClient, database, catalogClient, @@ -102,6 +107,7 @@ export async function createRouter( const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); const entityClient = new CatalogEntityClient(catalogClient); + const integrations = ScmIntegrations.fromConfig(config); const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), @@ -122,6 +128,14 @@ export async function createRouter( templaters, catalogClient, }); + actionRegistry.register( + createFetchCookiecutterAction({ + urlReader, + integrations, + dockerClient, + templaters, + }), + ); worker.start(); From 12771f5c3462beb7ec55e9693504808c40df25cb Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Feb 2021 14:47:03 +0100 Subject: [PATCH 11/66] feat: added some nice validation and functionality to the repo picker Co-authored-by: Patrik Oldsberg Co-authored-by: Johan Haals Signed-off-by: Johan Haals --- .../test-template-v2/template.yaml | 42 ++--- .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 160 ++++++++++++++++++ .../components/fields/RepoUrlPicker/index.ts | 1 + .../components/fields/RepoUrlPicker/index.tsx | 103 ----------- 4 files changed, 184 insertions(+), 122 deletions(-) create mode 100644 plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx create mode 100644 plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts delete mode 100644 plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 08432e6502..6b9e2142d9 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -82,25 +82,29 @@ spec: type: service parameters: - title: Fill in some BS params - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 - repoUrl: - title: Repository Location - type: string - description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - github.com + - title: Fill in some parameters for the template + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose destination + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + steps: - id: fetch-base diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx new file mode 100644 index 0000000000..49c4213251 --- /dev/null +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -0,0 +1,160 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState, useCallback, useEffect } from 'react'; +import { Field } from '@rjsf/core'; +import { useApi, Progress } from '@backstage/core'; +import { scaffolderApiRef } from '../../../api'; +import { useAsync } from 'react-use'; +import Select from '@material-ui/core/Select'; +import InputLabel from '@material-ui/core/InputLabel'; +import Input from '@material-ui/core/Input'; +import FormControl from '@material-ui/core/FormControl'; +import FormHelperText from '@material-ui/core/FormHelperText'; + +function splitFormData(url: string | undefined) { + let host = undefined; + let owner = undefined; + let repo = undefined; + + try { + if (url) { + const parsed = new URL(`https://${url}`); + host = parsed.host; + owner = parsed.searchParams.get('owner') || undefined; + repo = parsed.searchParams.get('repo') || undefined; + } + } catch { + /* ok */ + } + + return { host, owner, repo }; +} + +function serializeFormData(data: { + host?: string; + owner?: string; + repo?: string; +}) { + if (!data.host) { + return undefined; + } + const params = new URLSearchParams(); + if (data.owner) { + params.set('owner', data.owner); + } + if (data.repo) { + params.set('repo', data.repo); + } + + return `${data.host}?${params.toString()}`; +} + +export const RepoUrlPicker: Field = ({ + onChange, + uiSchema, + rawErrors, + formData, +}) => { + const api = useApi(scaffolderApiRef); + // const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; + const allowedHosts = React.useMemo( + () => ['github.com', 'gitlab.com'] as string[], + [], + ); + + const { value: integrations, loading } = useAsync(async () => { + return await api.getIntegrationsList({ allowedHosts }); + }); + + const { host, owner, repo } = splitFormData(formData); + const updateHost = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ host: evt.target.value as string, owner, repo }), + ), + [onChange, owner, repo], + ); + + const updateOwner = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ host, owner: evt.target.value as string, repo }), + ), + [onChange, host, repo], + ); + + const updateRepo = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ host, owner, repo: evt.target.value as string }), + ), + [onChange, host, owner], + ); + + useEffect(() => { + if (host === undefined && integrations?.length) { + onChange(serializeFormData({ host: integrations[0].host, owner, repo })); + } + }, [integrations, host]); + + if (loading) { + return ; + } + + return ( + <> + 0 && !host} + > + Host + + + The host where the repository will be created + + + 0 && !owner} + > + Owner + + + The organization, user or project that this repo will belong to + + + 0 && !repo} + > + Repository + + The name of the repository + + + ); +}; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts new file mode 100644 index 0000000000..1f7f9e0327 --- /dev/null +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts @@ -0,0 +1 @@ +export { RepoUrlPicker } from './RepoUrlPicker'; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx deleted file mode 100644 index a20f748d3a..0000000000 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.tsx +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React, { useState, useCallback, useEffect } from 'react'; -import { Field } from '@rjsf/core'; -import { useApi, Progress } from '@backstage/core'; -import { scaffolderApiRef } from '../../../api'; -import { useAsync } from 'react-use'; -import Select from '@material-ui/core/Select'; -import InputLabel from '@material-ui/core/InputLabel'; -import Input from '@material-ui/core/Input'; -import FormControl from '@material-ui/core/FormControl'; -import { Typography } from '@material-ui/core'; -import { rest } from 'msw/lib/types'; - -export const RepoUrlPicker: Field = ({ onChange, uiSchema, ...rest }) => { - const api = useApi(scaffolderApiRef); - const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; - - const { value: integrations, loading } = useAsync(async () => { - return await api.getIntegrationsList({ allowedHosts }); - }); - - console.log(rest); - - const [host, setHost] = useState(''); - const [owner, setOwner] = useState(''); - const [repo, setRepo] = useState(''); - - const updateHost = useCallback( - (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => - setHost(evt.target.value as string), - [setHost], - ); - - const updateOwner = useCallback( - (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => - setOwner(evt.target.value as string), - [setOwner], - ); - - const updateRepo = useCallback( - (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => - setRepo(evt.target.value as string), - [setRepo], - ); - - useEffect(() => { - if (host === '' && integrations?.length) { - setHost(integrations[0].host); - } - }, [integrations, host]); - - useEffect(() => { - const params = new URLSearchParams(); - params.set('owner', owner); - params.set('repo', repo); - - onChange(`${encodeURIComponent(host)}?${params.toString()}`); - }, [host, owner, repo, onChange]); - - if (loading) { - return ; - } - - return ( - <> - Repository Location - - Host - - - - Owner - - - - Repository - - - - ); -}; From 6706b360aac21c4dfde519aa0f872cff0abd8dbb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 24 Feb 2021 11:51:09 +0100 Subject: [PATCH 12/66] scaffolder-backend: added built-in fetch:plain action + separate out fetch logic Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/builtin.ts | 146 ++++++++++++------ .../scaffolder-backend/src/service/router.ts | 11 +- 2 files changed, 111 insertions(+), 46 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts index 60e3ad5282..1427d06880 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -21,6 +21,100 @@ import { TemplaterBuilder, TemplaterValues } from '../stages/templater'; import { TemplateAction } from './types'; import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; +import { JsonValue } from '@backstage/config'; + +async function fetchContents({ + urlReader, + integrations, + baseUrl, + fetchUrl = '.', + outputPath, +}: { + urlReader: UrlReader; + integrations: ScmIntegrations; + baseUrl?: string; + fetchUrl?: JsonValue; + outputPath: string; +}) { + if (typeof fetchUrl !== 'string') { + throw new InputError( + `Invalid url parameter, expected string, got ${typeof fetchUrl}`, + ); + } + + // We handle both file locations and url ones + if (baseUrl?.startsWith('file://')) { + const basePath = baseUrl.slice('file://'.length); + if (isAbsolute(fetchUrl)) { + throw new InputError( + `Fetch URL may not be absolute for file locations, ${fetchUrl}`, + ); + } + const srcDir = resolvePath(basePath, '..', fetchUrl); + await fs.copy(srcDir, outputPath); + } else { + let readUrl; + + if (baseUrl) { + const integration = integrations.byUrl(baseUrl); + if (!integration) { + throw new InputError(`No integration found for location ${baseUrl}`); + } + readUrl = integration.resolveUrl({ + url: fetchUrl, + base: baseUrl, + }); + } else { + // If we don't have a baseUrl, check if our provided fetch url is absolute + try { + readUrl = new URL(fetchUrl).toString(); + } catch { + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); + } + + const res = await urlReader.readTree(readUrl); + await res.dir({ targetDir: outputPath }); + } + } +} + +export function createFetchPlainAction(options: { + urlReader: UrlReader; + integrations: ScmIntegrations; +}): TemplateAction { + const { urlReader, integrations } = options; + + return { + id: 'fetch:plain', + async handler(ctx) { + ctx.logger.info('Fetching plain content from remote URL'); + + // Finally move the template result into the task workspace + const targetPath = ctx.parameters.targetPath ?? './'; + if (typeof targetPath !== 'string') { + throw new InputError( + `Fetch action targetPath is not a string, got ${targetPath}`, + ); + } + const outputPath = resolvePath(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + + await fetchContents({ + urlReader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.parameters.url, + outputPath, + }); + }, + }; +} export function createFetchCookiecutterAction(options: { dockerClient: Docker; @@ -42,51 +136,13 @@ export function createFetchCookiecutterAction(options: { ); const resultDir = resolvePath(workDir, 'result'); - const fetchUrl = ctx.parameters.url ?? '.'; - if (typeof fetchUrl !== 'string') { - throw new InputError( - `Invalid url parameter, expected string, got ${typeof fetchUrl}`, - ); - } - - // We handle both file locations and url ones - if (ctx.baseUrl?.startsWith('file://')) { - const basePath = ctx.baseUrl.slice('file://'.length); - if (isAbsolute(fetchUrl)) { - throw new InputError( - `Fetch URL may not be absolute for file locations, ${fetchUrl}`, - ); - } - const srcDir = resolvePath(basePath, '..', fetchUrl); - await fs.copy(srcDir, templateContentsDir); - } else { - let readUrl; - - if (ctx.baseUrl) { - const integration = integrations.byUrl(ctx.baseUrl); - if (!integration) { - throw new InputError( - `No integration found for location ${ctx.baseUrl}`, - ); - } - readUrl = integration.resolveUrl({ - url: fetchUrl, - base: ctx.baseUrl, - }); - } else { - // If we don't have a baseUrl, check if our provided fetch url is absolute - try { - readUrl = new URL(fetchUrl).toString(); - } catch { - throw new InputError( - `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, - ); - } - - const res = await urlReader.readTree(readUrl); - await res.dir({ targetDir: templateContentsDir }); - } - } + await fetchContents({ + urlReader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.parameters.url, + outputPath: templateContentsDir, + }); const cookiecutter = templaters.get('cookiecutter'); if (!cookiecutter) { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 3787d916b5..370924024b 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -54,7 +54,10 @@ import { TemplateEntityV1beta2, Entity, } from '@backstage/catalog-model'; -import { createFetchCookiecutterAction } from '../scaffolder/tasks/builtin'; +import { + createFetchPlainAction, + createFetchCookiecutterAction, +} from '../scaffolder/tasks/builtin'; import { ScmIntegrations } from '@backstage/integration'; export interface RouterOptions { @@ -128,6 +131,12 @@ export async function createRouter( templaters, catalogClient, }); + actionRegistry.register( + createFetchPlainAction({ + urlReader, + integrations, + }), + ); actionRegistry.register( createFetchCookiecutterAction({ urlReader, From 0d1468a09524f5b53177ead499b8459281109fdf Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Feb 2021 15:17:00 +0100 Subject: [PATCH 13/66] chore: remove the storepath picker as it has gone bye-bye Signed-off-by: Johan Haals --- .../widgets/StorePathPicker/index.tsx | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx diff --git a/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx b/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx deleted file mode 100644 index 87c00dbf98..0000000000 --- a/plugins/scaffolder/src/components/widgets/StorePathPicker/index.tsx +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Cop -yright 2021 Spotify AB - - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React, { useState, useCallback, useEffect } from 'react'; -import { Field, Widget } from '@rjsf/core'; -import { useApi, Progress } from '@backstage/core'; -import { scaffolderApiRef } from '../../../api'; -import { useAsync } from 'react-use'; -import TextField from '@material-ui/core/TextField'; -import MenuItem from '@material-ui/core/MenuItem'; -import InputLabel from '@material-ui/core/InputLabel'; - -import { Typography } from '@material-ui/core'; -import { rest } from 'msw/lib/types'; - -export const StorePathPicker: Field = ({ - options, - onChange, - rawErrors, - formContext, - uiSchema -}) => { - const api = useApi(scaffolderApiRef); - console.log(uiSchema) - const allowedHosts = uiSchema['ui:Options'].allowedHosts as string[]; - - const { value: integrations, loading } = useAsync(async () => { - return await api.getIntegrationsList({ allowedHosts }); - }); - - const [hostname, setHostname] = useState(''); - const [organization, setOrganization] = useState(''); - const [repositoryName, setRepositoryName] = useState(''); - - const [organization, setOrganization] = useState(''); - const [repositoryName, setRepositoryName] = useState(''); - - const updateHostname = useCallback( - (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => - setHostname(evt.target.value as string), - [setHostname], - ); - - useEffect(() => { - if (hostname === '' && integrations?.length) { - setHostname(integrations[0].host); - } - }, [integrations, hostname]); - - if (loading) { - return ; - } - - return ( - <> - Repository Location - - {integrations! - .filter(i => allowedHosts?.includes(i.host)) - .map(({ host, title }) => ( - - {title} - () =?> Setsets - ))} - - - - - ); -}; From a6591ef03fb97b9db2b5701917e43e73805f94b4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 24 Feb 2021 15:22:02 +0100 Subject: [PATCH 14/66] scaffolder-backend: move catalog:register action from legacy to builtin Signed-off-by: Johan Haals --- .../src/scaffolder/stages/legacy.ts | 28 +------------------ .../src/scaffolder/tasks/builtin.ts | 25 +++++++++++++++++ .../scaffolder-backend/src/service/router.ts | 3 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index 271618b981..4b3a0d8610 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -19,28 +19,19 @@ import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; import { PublisherBuilder } from './publish'; -import { CatalogApi } from '@backstage/catalog-client'; -import { getEntityName } from '@backstage/catalog-model'; type Options = { dockerClient: Docker; preparers: PreparerBuilder; templaters: TemplaterBuilder; publishers: PublisherBuilder; - catalogClient: CatalogApi; }; export function registerLegacyActions( registry: TemplateActionRegistry, options: Options, ) { - const { - dockerClient, - preparers, - templaters, - publishers, - catalogClient, - } = options; + const { dockerClient, preparers, templaters, publishers } = options; registry.register({ id: 'legacy:prepare', @@ -113,21 +104,4 @@ export function registerLegacyActions( } }, }); - - registry.register({ - id: 'catalog:register', - async handler(ctx) { - const { catalogInfoUrl } = ctx.parameters; - ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); - - const result = await catalogClient.addLocation({ - type: 'url', - target: catalogInfoUrl as string, - }); - if (result.entities.length >= 1) { - const { kind, name, namespace } = getEntityName(result.entities[0]); - ctx.output('entityRef', `${kind}:${namespace}/${name}`); - } - }, - }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts index 1427d06880..a6c39a9044 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -22,6 +22,8 @@ import { TemplateAction } from './types'; import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; import { JsonValue } from '@backstage/config'; +import { CatalogApi } from '@backstage/catalog-client'; +import { getEntityName } from '@backstage/catalog-model'; async function fetchContents({ urlReader, @@ -174,3 +176,26 @@ export function createFetchCookiecutterAction(options: { }, }; } + +export function createCatalogRegisterAction(options: { + catalogClient: CatalogApi; +}): TemplateAction { + const { catalogClient } = options; + + return { + id: 'catalog:register', + async handler(ctx) { + const { catalogInfoUrl } = ctx.parameters; + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); + + const result = await catalogClient.addLocation({ + type: 'url', + target: catalogInfoUrl as string, + }); + if (result.entities.length >= 1) { + const { kind, name, namespace } = getEntityName(result.entities[0]); + ctx.output('entityRef', `${kind}:${namespace}/${name}`); + } + }, + }; +} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 370924024b..71619de3bc 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -57,6 +57,7 @@ import { import { createFetchPlainAction, createFetchCookiecutterAction, + createCatalogRegisterAction, } from '../scaffolder/tasks/builtin'; import { ScmIntegrations } from '@backstage/integration'; @@ -129,7 +130,6 @@ export async function createRouter( preparers, publishers, templaters, - catalogClient, }); actionRegistry.register( createFetchPlainAction({ @@ -145,6 +145,7 @@ export async function createRouter( templaters, }), ); + actionRegistry.register(createCatalogRegisterAction({ catalogClient })); worker.start(); From 6fa98f941bf0b53fc22918f53862cf11f463359f Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Feb 2021 17:17:51 +0100 Subject: [PATCH 15/66] feat: new actions work nicely and publish to github as expected Signed-off-by: Johan Haals --- packages/backend/src/plugins/scaffolder.ts | 2 + .../test-template-v2/template.yaml | 27 +-- .../src/scaffolder/tasks/TaskWorker.ts | 2 + .../src/scaffolder/tasks/TemplateConverter.ts | 2 +- .../src/scaffolder/tasks/builtin.ts | 196 ++++++++++++++++-- .../scaffolder-backend/src/service/router.ts | 12 +- .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 6 +- 7 files changed, 211 insertions(+), 36 deletions(-) diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 1b97f735a0..9cab56d7f6 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -32,6 +32,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment): Promise { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -57,5 +58,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + urlReader: reader, }); } diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 6b9e2142d9..27b744664a 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -82,8 +82,8 @@ spec: type: service parameters: - - title: Fill in some parameters for the template - required: + - title: Fill in some steps + required: - name properties: name: @@ -93,7 +93,7 @@ spec: ui:autofocus: true ui:options: rows: 5 - - title: Choose destination + - title: Choose a location required: - repoUrl properties: @@ -104,7 +104,6 @@ spec: ui:options: allowedHosts: - github.com - steps: - id: fetch-base @@ -112,7 +111,6 @@ spec: action: fetch:cookiecutter parameters: url: ./template - targetDir: blob values: name: '{{ parameters.name }}' @@ -120,23 +118,26 @@ spec: name: Fetch Docs action: fetch:plain parameters: - url: https://github.com/aeothgiaetgh/aetkijhahte/tree/master/docs/template + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions - id: publish name: Publish action: publish:github parameters: - allowedHosts: ['ghe.mycompany.com'] - name: '{{ parameters.name }}' - # storePath: github.com?owner=spotify&repo=name - storePath: '{{ parameters.storePath }}' + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + # repoUrl: github.com?owner=spotify&repo=name + repoUrl: '{{ parameters.repoUrl }}' - id: register name: Register action: catalog:register parameters: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + output: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' - entityRef: '{{ steps.register.entityRef }}' + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 1e038ba65e..8fd425fbdf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -154,6 +154,7 @@ export class TaskWorker { } } + console.log('DEBUG: templateCtx =', JSON.stringify(templateCtx, 0, 2)); const output = JSON.parse( JSON.stringify(task.spec.output), (_key, value) => { @@ -168,6 +169,7 @@ export class TaskWorker { return value; }, ); + console.log('DEBUG: output =', output); await task.complete('completed', { output }); } catch (error) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index f9e9abf925..dd9a4e25da 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -94,7 +94,7 @@ export function templateEntityToSpec( steps, output: { remoteUrl: '{{ steps.publish.output.remoteUrl }}', - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', + catalogInfoUrl: '{{ steps.register.output.catalogInfoUrl }}', entityRef: '{{ steps.register.output.entityRef }}', }, }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts index a6c39a9044..214e7a8cef 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -20,10 +20,17 @@ import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from '../stages/templater'; import { TemplateAction } from './types'; import { InputError, UrlReader } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; import { JsonValue } from '@backstage/config'; import { CatalogApi } from '@backstage/catalog-client'; import { getEntityName } from '@backstage/catalog-model'; +import { PublisherBuilder } from '../stages'; +import { Octokit } from '@octokit/rest'; +import { Config } from '@backstage/config'; +import { initRepoAndPush } from '../stages/publish/helpers'; async function fetchContents({ urlReader, @@ -44,8 +51,16 @@ async function fetchContents({ ); } + let fetchUrlIsAbsolute = false; + try { + new URL(fetchUrl); + fetchUrlIsAbsolute = true; + } catch { + /* ignored */ + } + // We handle both file locations and url ones - if (baseUrl?.startsWith('file://')) { + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { const basePath = baseUrl.slice('file://'.length); if (isAbsolute(fetchUrl)) { throw new InputError( @@ -57,28 +72,27 @@ async function fetchContents({ } else { let readUrl; - if (baseUrl) { + if (fetchUrlIsAbsolute) { + readUrl = fetchUrl; + } else if (baseUrl) { const integration = integrations.byUrl(baseUrl); if (!integration) { throw new InputError(`No integration found for location ${baseUrl}`); } + readUrl = integration.resolveUrl({ url: fetchUrl, base: baseUrl, }); } else { - // If we don't have a baseUrl, check if our provided fetch url is absolute - try { - readUrl = new URL(fetchUrl).toString(); - } catch { - throw new InputError( - `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, - ); - } - - const res = await urlReader.readTree(readUrl); - await res.dir({ targetDir: outputPath }); + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); } + + const res = await urlReader.readTree(readUrl); + await fs.ensureDir(outputPath); + await res.dir({ targetDir: outputPath }); } } @@ -177,24 +191,174 @@ export function createFetchCookiecutterAction(options: { }; } +export function createPublishGithubAction(options: { + integrations: ScmIntegrations; + repoVisibility: 'private' | 'internal' | 'public'; +}): TemplateAction { + const { integrations, repoVisibility } = options; + + const credentialsProviders = new Map( + integrations.github.list().map(integration => { + const provider = GithubCredentialsProvider.create(integration.config); + return [integration.config.host, provider]; + }), + ); + + return { + id: 'publish:github', + async handler(ctx) { + const { repoUrl, description, access } = ctx.parameters; + + if (typeof repoUrl !== 'string') { + throw new Error( + `Invalid repo URL passed to publish:github, got ${typeof repoUrl}`, + ); + } + let parsed; + try { + parsed = new URL(`https://${repoUrl}`); + } catch (error) { + throw new InputError( + `Invalid repo URL passed to publish:github, got ${repoUrl}, ${error}`, + ); + } + const host = parsed.host; + const owner = parsed.searchParams.get('owner'); + if (!owner) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing owner`, + ); + } + const repo = parsed.searchParams.get('repo'); + if (!repo) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing repo`, + ); + } + + const credentialsProvider = credentialsProviders.get(host); + const integrationConfig = integrations.github.byHost(host); + + if (!credentialsProvider || !integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + + const { token } = await credentialsProvider.getCredentials({ + url: `${host}/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + }); + + if (!token) { + throw new InputError( + `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`, + ); + } + + const client = new Octokit({ + auth: token, + baseUrl: integrationConfig.config.apiBaseUrl, + }); + + const user = await client.users.getByUsername({ + username: owner, + }); + + const repoCreationPromise = + user.data.type === 'Organization' + ? client.repos.createInOrg({ + name: repo, + org: owner, + private: repoVisibility !== 'public', + visibility: repoVisibility, + description: description as string, + }) + : client.repos.createForAuthenticatedUser({ + name: repo, + private: repoVisibility === 'private', + description: description as string, + }); + + const { data } = await repoCreationPromise; + const accessString = access as string; + if (accessString?.startsWith(`${owner}/`)) { + const [, team] = accessString.split('/'); + await client.teams.addOrUpdateRepoPermissionsInOrg({ + org: owner, + team_slug: team, + owner, + repo, + permission: 'admin', + }); + // no need to add accessString if it's the person who own's the personal account + } else if (accessString && accessString !== owner) { + await client.repos.addCollaborator({ + owner, + repo, + username: accessString, + permission: 'admin', + }); + } + + const remoteUrl = data?.clone_url; + const repoContentsUrl = data?.html_url + ? `${data?.html_url}/blob/master` + : undefined; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'x-access-token', + password: token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} + export function createCatalogRegisterAction(options: { catalogClient: CatalogApi; + integrations: ScmIntegrations; }): TemplateAction { - const { catalogClient } = options; + const { catalogClient, integrations } = options; return { id: 'catalog:register', async handler(ctx) { - const { catalogInfoUrl } = ctx.parameters; + const { + repoContentsUrl, + catalogInfoPath = '/catalog-info.yaml', + } = ctx.parameters; + + let { catalogInfoUrl } = ctx.parameters; + if (!catalogInfoUrl) { + const integration = integrations.byUrl(repoContentsUrl as string); + if (!integration) { + throw new InputError('No integration found for host'); + } + + catalogInfoUrl = integration.resolveUrl({ + base: repoContentsUrl as string, + url: catalogInfoPath as string, + }); + } + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); const result = await catalogClient.addLocation({ type: 'url', target: catalogInfoUrl as string, }); + console.log('DEBUG: result.entities =', result.entities); if (result.entities.length >= 1) { const { kind, name, namespace } = getEntityName(result.entities[0]); ctx.output('entityRef', `${kind}:${namespace}/${name}`); + ctx.output('catalogInfoUrl', catalogInfoUrl); } }, }; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 71619de3bc..3cc8d01238 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -57,6 +57,7 @@ import { import { createFetchPlainAction, createFetchCookiecutterAction, + createPublishGithubAction, createCatalogRegisterAction, } from '../scaffolder/tasks/builtin'; import { ScmIntegrations } from '@backstage/integration'; @@ -145,7 +146,16 @@ export async function createRouter( templaters, }), ); - actionRegistry.register(createCatalogRegisterAction({ catalogClient })); + actionRegistry.register( + createPublishGithubAction({ + integrations, + repoVisibility: 'public', + }), + ); + + actionRegistry.register( + createCatalogRegisterAction({ catalogClient, integrations }), + ); worker.start(); diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 49c4213251..90d53fce85 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -69,11 +69,7 @@ export const RepoUrlPicker: Field = ({ formData, }) => { const api = useApi(scaffolderApiRef); - // const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; - const allowedHosts = React.useMemo( - () => ['github.com', 'gitlab.com'] as string[], - [], - ); + const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; const { value: integrations, loading } = useAsync(async () => { return await api.getIntegrationsList({ allowedHosts }); From 8b003875a8728ea0a8970346acb5a344d4f7e1e2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 24 Feb 2021 20:23:32 +0100 Subject: [PATCH 16/66] scaffolder,scaffolder-backend: fix tests, lint and type issues Signed-off-by: Johan Haals --- .../src/scaffolder/tasks/TaskWorker.ts | 2 -- .../src/scaffolder/tasks/builtin.ts | 10 +++------- .../scaffolder-backend/src/service/router.test.ts | 10 ++++++++++ plugins/scaffolder/dev/index.tsx | 12 ++++++++---- .../components/TemplatePage/TemplatePage.test.tsx | 1 + .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 10 +++++----- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 8fd425fbdf..1e038ba65e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -154,7 +154,6 @@ export class TaskWorker { } } - console.log('DEBUG: templateCtx =', JSON.stringify(templateCtx, 0, 2)); const output = JSON.parse( JSON.stringify(task.spec.output), (_key, value) => { @@ -169,7 +168,6 @@ export class TaskWorker { return value; }, ); - console.log('DEBUG: output =', output); await task.complete('completed', { output }); } catch (error) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts index 214e7a8cef..b392157ac6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -27,9 +27,7 @@ import { import { JsonValue } from '@backstage/config'; import { CatalogApi } from '@backstage/catalog-client'; import { getEntityName } from '@backstage/catalog-model'; -import { PublisherBuilder } from '../stages'; import { Octokit } from '@octokit/rest'; -import { Config } from '@backstage/config'; import { initRepoAndPush } from '../stages/publish/helpers'; async function fetchContents({ @@ -53,6 +51,7 @@ async function fetchContents({ let fetchUrlIsAbsolute = false; try { + // eslint-disable-next-line no-new new URL(fetchUrl); fetchUrlIsAbsolute = true; } catch { @@ -300,10 +299,8 @@ export function createPublishGithubAction(options: { }); } - const remoteUrl = data?.clone_url; - const repoContentsUrl = data?.html_url - ? `${data?.html_url}/blob/master` - : undefined; + const remoteUrl = data.clone_url; + const repoContentsUrl = `${data?.html_url}/blob/master`; await initRepoAndPush({ dir: ctx.workspacePath, @@ -354,7 +351,6 @@ export function createCatalogRegisterAction(options: { type: 'url', target: catalogInfoUrl as string, }); - console.log('DEBUG: result.entities =', result.entities); if (result.entities.length >= 1) { const { kind, name, namespace } = getEntityName(result.entities[0]); ctx.output('entityRef', `${kind}:${namespace}/${name}`); diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 8fd53e5fb6..b30ef135ba 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -32,6 +32,7 @@ import { SingleConnectionDatabaseManager, PluginDatabaseManager, getVoidLogger, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import express from 'express'; @@ -61,6 +62,11 @@ function createDatabase(): PluginDatabaseManager { ).forPlugin('scaffolder'); } +const mockUrlReader = UrlReaders.default({ + logger: getVoidLogger(), + config: new ConfigReader({}), +}); + describe('createRouter - working directory', () => { const mockPrepare = jest.fn(); const mockPreparers = new Preparers(); @@ -112,6 +118,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + urlReader: mockUrlReader, }), ).rejects.toThrow('access error'); }); @@ -126,6 +133,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + urlReader: mockUrlReader, }); const app = express().use(router); @@ -155,6 +163,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + urlReader: mockUrlReader, }); const app = express().use(router); @@ -228,6 +237,7 @@ describe('createRouter', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + urlReader: mockUrlReader, }); app = express().use(router); }); diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index 1501a1cd5e..a41d088329 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -16,7 +16,7 @@ import React from 'react'; import { createDevApp } from '@backstage/dev-utils'; -import { discoveryApiRef, identityApiRef } from '@backstage/core'; +import { configApiRef, discoveryApiRef, identityApiRef } from '@backstage/core'; import { CatalogClient } from '@backstage/catalog-client'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { ScaffolderPage } from '../src/plugin'; @@ -30,9 +30,13 @@ createDevApp() }) .registerApi({ api: scaffolderApiRef, - deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, - factory: ({ discoveryApi, identityApi }) => - new ScaffolderClient({ discoveryApi, identityApi }), + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + configApi: configApiRef, + }, + factory: ({ discoveryApi, identityApi, configApi }) => + new ScaffolderClient({ discoveryApi, identityApi, configApi }), }) .addPage({ path: '/create', diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 0556dd1129..1b11b704ae 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -36,6 +36,7 @@ jest.mock('react-router-dom', () => { const scaffolderApiMock: jest.Mocked = { scaffold: jest.fn(), getTemplateParameterSchema: jest.fn(), + getIntegrationsList: jest.fn(), getTask: jest.fn(), streamLogs: jest.fn(), }; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 90d53fce85..f346a65a4b 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useState, useCallback, useEffect } from 'react'; +import React, { useCallback, useEffect } from 'react'; import { Field } from '@rjsf/core'; import { useApi, Progress } from '@backstage/core'; import { scaffolderApiRef } from '../../../api'; @@ -104,7 +104,7 @@ export const RepoUrlPicker: Field = ({ if (host === undefined && integrations?.length) { onChange(serializeFormData({ host: integrations[0].host, owner, repo })); } - }, [integrations, host]); + }, [onChange, integrations, host, owner, repo]); if (loading) { return ; @@ -121,9 +121,9 @@ export const RepoUrlPicker: Field = ({ From e0c8c6c164d1abc7d841f656369e48f8cbb8ae16 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 24 Feb 2021 20:30:45 +0100 Subject: [PATCH 17/66] scaffolder-backend: move actions to separate actions dir Signed-off-by: Johan Haals --- .../TemplateActionRegistry.ts | 0 .../scaffolder/{tasks => actions}/builtin.ts | 0 .../src/scaffolder/actions/index.ts | 24 +++++++++++ .../src/scaffolder/actions/types.ts | 43 +++++++++++++++++++ .../src/scaffolder/stages/legacy.ts | 2 +- .../src/scaffolder/tasks/TaskWorker.ts | 2 +- .../src/scaffolder/tasks/types.ts | 27 +----------- .../scaffolder-backend/src/service/router.ts | 4 +- 8 files changed, 72 insertions(+), 30 deletions(-) rename plugins/scaffolder-backend/src/scaffolder/{tasks => actions}/TemplateActionRegistry.ts (100%) rename plugins/scaffolder-backend/src/scaffolder/{tasks => actions}/builtin.ts (100%) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/index.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/types.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/tasks/TemplateActionRegistry.ts rename to plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts similarity index 100% rename from plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts rename to plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts new file mode 100644 index 0000000000..3da406d45a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + createCatalogRegisterAction, + createFetchCookiecutterAction, + createFetchPlainAction, + createPublishGithubAction, +} from './builtin'; +export { TemplateActionRegistry } from './TemplateActionRegistry'; +export type { ActionContext, TemplateAction } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts new file mode 100644 index 0000000000..aed91e35f8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Writable } from 'stream'; +import { JsonValue } from '@backstage/config'; + +export type ActionContext = { + /** + * Base URL for the location of the task spec, typically the url of the source entity file. + */ + baseUrl?: string; + + logger: Logger; + logStream: Writable; + + workspacePath: string; + parameters: { [name: string]: JsonValue }; + output(name: string, value: JsonValue): void; + + /** + * Creates a temporary directory for use by the action, which is then cleaned up automatically. + */ + createTemporaryDirectory(): Promise; +}; + +export type TemplateAction = { + id: string; + handler: (ctx: ActionContext) => Promise; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index 4b3a0d8610..5f2d6a47a8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TemplateActionRegistry } from '../tasks/TemplateActionRegistry'; +import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 1e038ba65e..357e5e815c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -21,7 +21,7 @@ import { JsonValue, JsonObject } from '@backstage/config'; import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; -import { TemplateActionRegistry } from './TemplateActionRegistry'; +import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import * as handlebars from 'handlebars'; type Options = { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 42d8179a3e..3a1d207a04 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -15,8 +15,6 @@ */ import { JsonValue, JsonObject } from '@backstage/config'; -import { Logger } from 'winston'; -import { Writable } from 'stream'; export type Status = | 'open' @@ -93,6 +91,7 @@ export type TaskStoreGetEventsOptions = { taskId: string; after?: number | undefined; }; + export interface TaskStore { createTask(task: TaskSpec): Promise<{ taskId: string }>; getTask(taskId: string): Promise; @@ -115,27 +114,3 @@ export interface TaskStore { after, }: TaskStoreGetEventsOptions): Promise<{ events: DbTaskEventRow[] }>; } - -export type ActionContext = { - /** - * Base URL for the location of the task spec, typically the url of the source entity file. - */ - baseUrl?: string; - - logger: Logger; - logStream: Writable; - - workspacePath: string; - parameters: { [name: string]: JsonValue }; - output(name: string, value: JsonValue): void; - - /** - * Creates a temporary directory for use by the action, which is then cleaned up automatically. - */ - createTemporaryDirectory(): Promise; -}; - -export type TemplateAction = { - id: string; - handler: (ctx: ActionContext) => Promise; -}; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 3cc8d01238..f30cb9ce3a 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -39,7 +39,7 @@ import { TaskWorker, } from '../scaffolder/tasks'; import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; -import { TemplateActionRegistry } from '../scaffolder/tasks/TemplateActionRegistry'; +import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; import { registerLegacyActions } from '../scaffolder/stages/legacy'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { @@ -59,7 +59,7 @@ import { createFetchCookiecutterAction, createPublishGithubAction, createCatalogRegisterAction, -} from '../scaffolder/tasks/builtin'; +} from '../scaffolder/actions/builtin'; import { ScmIntegrations } from '@backstage/integration'; export interface RouterOptions { From 7e8f2552f6974b6febfdcc96af42c92180087c9d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 24 Feb 2021 20:44:37 +0100 Subject: [PATCH 18/66] scaffolder-backend: split builtin actions into multiple modules Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin.ts | 361 ------------------ .../actions/builtin/catalog/index.ts | 17 + .../actions/builtin/catalog/register.ts | 63 +++ .../actions/builtin/fetch/cookiecutter.ts | 83 ++++ .../actions/builtin/fetch/helpers.ts | 86 +++++ .../scaffolder/actions/builtin/fetch/index.ts | 18 + .../scaffolder/actions/builtin/fetch/plain.ts | 57 +++ .../src/scaffolder/actions/builtin/index.ts | 19 + .../actions/builtin/publish/github.ts | 152 ++++++++ .../actions/builtin/publish/index.ts | 17 + .../src/scaffolder/actions/index.ts | 7 +- 11 files changed, 513 insertions(+), 367 deletions(-) delete mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts deleted file mode 100644 index b392157ac6..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin.ts +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { resolve as resolvePath, isAbsolute } from 'path'; -import Docker from 'dockerode'; -import { TemplaterBuilder, TemplaterValues } from '../stages/templater'; -import { TemplateAction } from './types'; -import { InputError, UrlReader } from '@backstage/backend-common'; -import { - GithubCredentialsProvider, - ScmIntegrations, -} from '@backstage/integration'; -import { JsonValue } from '@backstage/config'; -import { CatalogApi } from '@backstage/catalog-client'; -import { getEntityName } from '@backstage/catalog-model'; -import { Octokit } from '@octokit/rest'; -import { initRepoAndPush } from '../stages/publish/helpers'; - -async function fetchContents({ - urlReader, - integrations, - baseUrl, - fetchUrl = '.', - outputPath, -}: { - urlReader: UrlReader; - integrations: ScmIntegrations; - baseUrl?: string; - fetchUrl?: JsonValue; - outputPath: string; -}) { - if (typeof fetchUrl !== 'string') { - throw new InputError( - `Invalid url parameter, expected string, got ${typeof fetchUrl}`, - ); - } - - let fetchUrlIsAbsolute = false; - try { - // eslint-disable-next-line no-new - new URL(fetchUrl); - fetchUrlIsAbsolute = true; - } catch { - /* ignored */ - } - - // We handle both file locations and url ones - if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { - const basePath = baseUrl.slice('file://'.length); - if (isAbsolute(fetchUrl)) { - throw new InputError( - `Fetch URL may not be absolute for file locations, ${fetchUrl}`, - ); - } - const srcDir = resolvePath(basePath, '..', fetchUrl); - await fs.copy(srcDir, outputPath); - } else { - let readUrl; - - if (fetchUrlIsAbsolute) { - readUrl = fetchUrl; - } else if (baseUrl) { - const integration = integrations.byUrl(baseUrl); - if (!integration) { - throw new InputError(`No integration found for location ${baseUrl}`); - } - - readUrl = integration.resolveUrl({ - url: fetchUrl, - base: baseUrl, - }); - } else { - throw new InputError( - `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, - ); - } - - const res = await urlReader.readTree(readUrl); - await fs.ensureDir(outputPath); - await res.dir({ targetDir: outputPath }); - } -} - -export function createFetchPlainAction(options: { - urlReader: UrlReader; - integrations: ScmIntegrations; -}): TemplateAction { - const { urlReader, integrations } = options; - - return { - id: 'fetch:plain', - async handler(ctx) { - ctx.logger.info('Fetching plain content from remote URL'); - - // Finally move the template result into the task workspace - const targetPath = ctx.parameters.targetPath ?? './'; - if (typeof targetPath !== 'string') { - throw new InputError( - `Fetch action targetPath is not a string, got ${targetPath}`, - ); - } - const outputPath = resolvePath(ctx.workspacePath, targetPath); - if (!outputPath.startsWith(ctx.workspacePath)) { - throw new InputError( - `Fetch action targetPath may not specify a path outside the working directory`, - ); - } - - await fetchContents({ - urlReader, - integrations, - baseUrl: ctx.baseUrl, - fetchUrl: ctx.parameters.url, - outputPath, - }); - }, - }; -} - -export function createFetchCookiecutterAction(options: { - dockerClient: Docker; - urlReader: UrlReader; - integrations: ScmIntegrations; - templaters: TemplaterBuilder; -}): TemplateAction { - const { dockerClient, urlReader, templaters, integrations } = options; - - return { - id: 'fetch:cookiecutter', - async handler(ctx) { - ctx.logger.info('Fetching and then templating using cookiecutter'); - const workDir = await ctx.createTemporaryDirectory(); - const templateDir = resolvePath(workDir, 'template'); - const templateContentsDir = resolvePath( - templateDir, - "{{cookiecutter and 'contents'}}", - ); - const resultDir = resolvePath(workDir, 'result'); - - await fetchContents({ - urlReader, - integrations, - baseUrl: ctx.baseUrl, - fetchUrl: ctx.parameters.url, - outputPath: templateContentsDir, - }); - - const cookiecutter = templaters.get('cookiecutter'); - if (!cookiecutter) { - throw new Error('No cookiecutter templater available'); - } - - // Will execute the template in ./template and put the result in ./result - await cookiecutter.run({ - workspacePath: workDir, - dockerClient, - logStream: ctx.logStream, - values: ctx.parameters.values as TemplaterValues, - }); - - // Finally move the template result into the task workspace - const targetPath = ctx.parameters.targetPath ?? './'; - if (typeof targetPath !== 'string') { - throw new InputError( - `Fetch action targetPath is not a string, got ${targetPath}`, - ); - } - const outputPath = resolvePath(ctx.workspacePath, targetPath); - if (!outputPath.startsWith(ctx.workspacePath)) { - throw new InputError( - `Fetch action targetPath may not specify a path outside the working directory`, - ); - } - await fs.copy(resultDir, outputPath); - }, - }; -} - -export function createPublishGithubAction(options: { - integrations: ScmIntegrations; - repoVisibility: 'private' | 'internal' | 'public'; -}): TemplateAction { - const { integrations, repoVisibility } = options; - - const credentialsProviders = new Map( - integrations.github.list().map(integration => { - const provider = GithubCredentialsProvider.create(integration.config); - return [integration.config.host, provider]; - }), - ); - - return { - id: 'publish:github', - async handler(ctx) { - const { repoUrl, description, access } = ctx.parameters; - - if (typeof repoUrl !== 'string') { - throw new Error( - `Invalid repo URL passed to publish:github, got ${typeof repoUrl}`, - ); - } - let parsed; - try { - parsed = new URL(`https://${repoUrl}`); - } catch (error) { - throw new InputError( - `Invalid repo URL passed to publish:github, got ${repoUrl}, ${error}`, - ); - } - const host = parsed.host; - const owner = parsed.searchParams.get('owner'); - if (!owner) { - throw new InputError( - `Invalid repo URL passed to publish:github, missing owner`, - ); - } - const repo = parsed.searchParams.get('repo'); - if (!repo) { - throw new InputError( - `Invalid repo URL passed to publish:github, missing repo`, - ); - } - - const credentialsProvider = credentialsProviders.get(host); - const integrationConfig = integrations.github.byHost(host); - - if (!credentialsProvider || !integrationConfig) { - throw new InputError( - `No matching integration configuration for host ${host}, please check your Integrations config`, - ); - } - - const { token } = await credentialsProvider.getCredentials({ - url: `${host}/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, - }); - - if (!token) { - throw new InputError( - `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`, - ); - } - - const client = new Octokit({ - auth: token, - baseUrl: integrationConfig.config.apiBaseUrl, - }); - - const user = await client.users.getByUsername({ - username: owner, - }); - - const repoCreationPromise = - user.data.type === 'Organization' - ? client.repos.createInOrg({ - name: repo, - org: owner, - private: repoVisibility !== 'public', - visibility: repoVisibility, - description: description as string, - }) - : client.repos.createForAuthenticatedUser({ - name: repo, - private: repoVisibility === 'private', - description: description as string, - }); - - const { data } = await repoCreationPromise; - const accessString = access as string; - if (accessString?.startsWith(`${owner}/`)) { - const [, team] = accessString.split('/'); - await client.teams.addOrUpdateRepoPermissionsInOrg({ - org: owner, - team_slug: team, - owner, - repo, - permission: 'admin', - }); - // no need to add accessString if it's the person who own's the personal account - } else if (accessString && accessString !== owner) { - await client.repos.addCollaborator({ - owner, - repo, - username: accessString, - permission: 'admin', - }); - } - - const remoteUrl = data.clone_url; - const repoContentsUrl = `${data?.html_url}/blob/master`; - - await initRepoAndPush({ - dir: ctx.workspacePath, - remoteUrl, - auth: { - username: 'x-access-token', - password: token, - }, - logger: ctx.logger, - }); - - ctx.output('remoteUrl', remoteUrl); - ctx.output('repoContentsUrl', repoContentsUrl); - }, - }; -} - -export function createCatalogRegisterAction(options: { - catalogClient: CatalogApi; - integrations: ScmIntegrations; -}): TemplateAction { - const { catalogClient, integrations } = options; - - return { - id: 'catalog:register', - async handler(ctx) { - const { - repoContentsUrl, - catalogInfoPath = '/catalog-info.yaml', - } = ctx.parameters; - - let { catalogInfoUrl } = ctx.parameters; - if (!catalogInfoUrl) { - const integration = integrations.byUrl(repoContentsUrl as string); - if (!integration) { - throw new InputError('No integration found for host'); - } - - catalogInfoUrl = integration.resolveUrl({ - base: repoContentsUrl as string, - url: catalogInfoPath as string, - }); - } - - ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); - - const result = await catalogClient.addLocation({ - type: 'url', - target: catalogInfoUrl as string, - }); - if (result.entities.length >= 1) { - const { kind, name, namespace } = getEntityName(result.entities[0]); - ctx.output('entityRef', `${kind}:${namespace}/${name}`); - ctx.output('catalogInfoUrl', catalogInfoUrl); - } - }, - }; -} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts new file mode 100644 index 0000000000..2e25b5592f --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createCatalogRegisterAction } from './register'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts new file mode 100644 index 0000000000..4d19f1189f --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { CatalogApi } from '@backstage/catalog-client'; +import { getEntityName } from '@backstage/catalog-model'; +import { TemplateAction } from '../../types'; + +export function createCatalogRegisterAction(options: { + catalogClient: CatalogApi; + integrations: ScmIntegrations; +}): TemplateAction { + const { catalogClient, integrations } = options; + + return { + id: 'catalog:register', + async handler(ctx) { + const { + repoContentsUrl, + catalogInfoPath = '/catalog-info.yaml', + } = ctx.parameters; + + let { catalogInfoUrl } = ctx.parameters; + if (!catalogInfoUrl) { + const integration = integrations.byUrl(repoContentsUrl as string); + if (!integration) { + throw new InputError('No integration found for host'); + } + + catalogInfoUrl = integration.resolveUrl({ + base: repoContentsUrl as string, + url: catalogInfoPath as string, + }); + } + + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); + + const result = await catalogClient.addLocation({ + type: 'url', + target: catalogInfoUrl as string, + }); + if (result.entities.length >= 1) { + const { kind, name, namespace } = getEntityName(result.entities[0]); + ctx.output('entityRef', `${kind}:${namespace}/${name}`); + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts new file mode 100644 index 0000000000..281a6e75d2 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'path'; +import Docker from 'dockerode'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater'; +import { TemplateAction } from '../../types'; +import { fetchContents } from './helpers'; + +export function createFetchCookiecutterAction(options: { + dockerClient: Docker; + urlReader: UrlReader; + integrations: ScmIntegrations; + templaters: TemplaterBuilder; +}): TemplateAction { + const { dockerClient, urlReader, templaters, integrations } = options; + + return { + id: 'fetch:cookiecutter', + async handler(ctx) { + ctx.logger.info('Fetching and then templating using cookiecutter'); + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + const templateContentsDir = resolvePath( + templateDir, + "{{cookiecutter and 'contents'}}", + ); + const resultDir = resolvePath(workDir, 'result'); + + await fetchContents({ + urlReader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.parameters.url, + outputPath: templateContentsDir, + }); + + const cookiecutter = templaters.get('cookiecutter'); + if (!cookiecutter) { + throw new Error('No cookiecutter templater available'); + } + + // Will execute the template in ./template and put the result in ./result + await cookiecutter.run({ + workspacePath: workDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.parameters.values as TemplaterValues, + }); + + // Finally move the template result into the task workspace + const targetPath = ctx.parameters.targetPath ?? './'; + if (typeof targetPath !== 'string') { + throw new InputError( + `Fetch action targetPath is not a string, got ${targetPath}`, + ); + } + const outputPath = resolvePath(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + await fs.copy(resultDir, outputPath); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts new file mode 100644 index 0000000000..a739e53e1f --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, isAbsolute } from 'path'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { JsonValue } from '@backstage/config'; + +export async function fetchContents({ + urlReader, + integrations, + baseUrl, + fetchUrl = '.', + outputPath, +}: { + urlReader: UrlReader; + integrations: ScmIntegrations; + baseUrl?: string; + fetchUrl?: JsonValue; + outputPath: string; +}) { + if (typeof fetchUrl !== 'string') { + throw new InputError( + `Invalid url parameter, expected string, got ${typeof fetchUrl}`, + ); + } + + let fetchUrlIsAbsolute = false; + try { + // eslint-disable-next-line no-new + new URL(fetchUrl); + fetchUrlIsAbsolute = true; + } catch { + /* ignored */ + } + + // We handle both file locations and url ones + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { + const basePath = baseUrl.slice('file://'.length); + if (isAbsolute(fetchUrl)) { + throw new InputError( + `Fetch URL may not be absolute for file locations, ${fetchUrl}`, + ); + } + const srcDir = resolvePath(basePath, '..', fetchUrl); + await fs.copy(srcDir, outputPath); + } else { + let readUrl; + + if (fetchUrlIsAbsolute) { + readUrl = fetchUrl; + } else if (baseUrl) { + const integration = integrations.byUrl(baseUrl); + if (!integration) { + throw new InputError(`No integration found for location ${baseUrl}`); + } + + readUrl = integration.resolveUrl({ + url: fetchUrl, + base: baseUrl, + }); + } else { + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); + } + + const res = await urlReader.readTree(readUrl); + await fs.ensureDir(outputPath); + await res.dir({ targetDir: outputPath }); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts new file mode 100644 index 0000000000..0a7239b6b7 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createFetchPlainAction } from './plain'; +export { createFetchCookiecutterAction } from './cookiecutter'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts new file mode 100644 index 0000000000..371bc35e9a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolve as resolvePath } from 'path'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../../types'; +import { fetchContents } from './helpers'; + +export function createFetchPlainAction(options: { + urlReader: UrlReader; + integrations: ScmIntegrations; +}): TemplateAction { + const { urlReader, integrations } = options; + + return { + id: 'fetch:plain', + async handler(ctx) { + ctx.logger.info('Fetching plain content from remote URL'); + + // Finally move the template result into the task workspace + const targetPath = ctx.parameters.targetPath ?? './'; + if (typeof targetPath !== 'string') { + throw new InputError( + `Fetch action targetPath is not a string, got ${targetPath}`, + ); + } + const outputPath = resolvePath(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + + await fetchContents({ + urlReader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.parameters.url, + outputPath, + }); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts new file mode 100644 index 0000000000..c85c6e33ce --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './catalog'; +export * from './fetch'; +export * from './publish'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts new file mode 100644 index 0000000000..26f2775c91 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; +import { Octokit } from '@octokit/rest'; +import { TemplateAction } from '../../types'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +export function createPublishGithubAction(options: { + integrations: ScmIntegrations; + repoVisibility: 'private' | 'internal' | 'public'; +}): TemplateAction { + const { integrations, repoVisibility } = options; + + const credentialsProviders = new Map( + integrations.github.list().map(integration => { + const provider = GithubCredentialsProvider.create(integration.config); + return [integration.config.host, provider]; + }), + ); + + return { + id: 'publish:github', + async handler(ctx) { + const { repoUrl, description, access } = ctx.parameters; + + if (typeof repoUrl !== 'string') { + throw new Error( + `Invalid repo URL passed to publish:github, got ${typeof repoUrl}`, + ); + } + let parsed; + try { + parsed = new URL(`https://${repoUrl}`); + } catch (error) { + throw new InputError( + `Invalid repo URL passed to publish:github, got ${repoUrl}, ${error}`, + ); + } + const host = parsed.host; + const owner = parsed.searchParams.get('owner'); + if (!owner) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing owner`, + ); + } + const repo = parsed.searchParams.get('repo'); + if (!repo) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing repo`, + ); + } + + const credentialsProvider = credentialsProviders.get(host); + const integrationConfig = integrations.github.byHost(host); + + if (!credentialsProvider || !integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + + const { token } = await credentialsProvider.getCredentials({ + url: `${host}/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + }); + + if (!token) { + throw new InputError( + `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`, + ); + } + + const client = new Octokit({ + auth: token, + baseUrl: integrationConfig.config.apiBaseUrl, + }); + + const user = await client.users.getByUsername({ + username: owner, + }); + + const repoCreationPromise = + user.data.type === 'Organization' + ? client.repos.createInOrg({ + name: repo, + org: owner, + private: repoVisibility !== 'public', + visibility: repoVisibility, + description: description as string, + }) + : client.repos.createForAuthenticatedUser({ + name: repo, + private: repoVisibility === 'private', + description: description as string, + }); + + const { data } = await repoCreationPromise; + const accessString = access as string; + if (accessString?.startsWith(`${owner}/`)) { + const [, team] = accessString.split('/'); + await client.teams.addOrUpdateRepoPermissionsInOrg({ + org: owner, + team_slug: team, + owner, + repo, + permission: 'admin', + }); + // no need to add accessString if it's the person who own's the personal account + } else if (accessString && accessString !== owner) { + await client.repos.addCollaborator({ + owner, + repo, + username: accessString, + permission: 'admin', + }); + } + + const remoteUrl = data.clone_url; + const repoContentsUrl = `${data?.html_url}/blob/master`; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'x-access-token', + password: token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts new file mode 100644 index 0000000000..b1ecb91f9a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createPublishGithubAction } from './github'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts index 3da406d45a..0f7931e74b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts @@ -14,11 +14,6 @@ * limitations under the License. */ -export { - createCatalogRegisterAction, - createFetchCookiecutterAction, - createFetchPlainAction, - createPublishGithubAction, -} from './builtin'; +export * from './builtin'; export { TemplateActionRegistry } from './TemplateActionRegistry'; export type { ActionContext, TemplateAction } from './types'; From 20ba3205f0ff2578c0da484963cc7af238aed155 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 25 Feb 2021 01:19:42 +0100 Subject: [PATCH 19/66] scaffolder-backend: add action parameter type and schema with validation Signed-off-by: Johan Haals --- .../actions/TemplateActionRegistry.ts | 10 ++-- .../actions/builtin/catalog/register.ts | 54 ++++++++++++++++--- .../actions/builtin/fetch/cookiecutter.ts | 31 ++++++++--- .../scaffolder/actions/builtin/fetch/plain.ts | 25 ++++++--- .../actions/builtin/publish/github.ts | 44 ++++++++++----- .../src/scaffolder/actions/types.ts | 16 ++++-- .../src/scaffolder/tasks/TaskWorker.ts | 16 ++++++ 7 files changed, 154 insertions(+), 42 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts index 9d6140fbfe..62405679d0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts @@ -14,13 +14,15 @@ * limitations under the License. */ -import { TemplateAction } from './types'; +import { ParameterBase, TemplateAction } from './types'; import { ConflictError, NotFoundError } from '@backstage/backend-common'; export class TemplateActionRegistry { - private readonly actions = new Map(); + private readonly actions = new Map>(); - register(action: TemplateAction) { + register( + action: TemplateAction, + ) { if (this.actions.has(action.id)) { throw new ConflictError( `Template action with ID '${action.id}' has already been registered`, @@ -29,7 +31,7 @@ export class TemplateActionRegistry { this.actions.set(action.id, action); } - get(actionId: string): TemplateAction { + get(actionId: string): TemplateAction { const action = this.actions.get(actionId); if (!action) { throw new NotFoundError( diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts index 4d19f1189f..275ed5ec69 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -23,19 +23,59 @@ import { TemplateAction } from '../../types'; export function createCatalogRegisterAction(options: { catalogClient: CatalogApi; integrations: ScmIntegrations; -}): TemplateAction { +}): TemplateAction< + | { catalogInfoUrl: string } + | { repoContentsUrl: string; catalogInfoPath?: string } +> { const { catalogClient, integrations } = options; return { id: 'catalog:register', + parameterSchema: { + oneOf: [ + { + type: 'object', + required: ['catalogInfoUrl'], + properties: { + catalogInfoUrl: { + title: 'Catalog Info URL', + description: + 'An absolute URL pointing to the catalog info file location', + type: 'string', + }, + }, + }, + { + type: 'object', + required: ['repoContentsUrl'], + properties: { + repoContentsUrl: { + title: 'Repository Contents URL', + description: + 'An absolute URL pointing to the root of a repository directory tree', + type: 'string', + }, + catalogInfoPath: { + title: 'Fetch URL', + description: + 'A relative path from the repo root pointing to the catalog info file, defaults to /catalog-info.yaml', + type: 'string', + }, + }, + }, + ], + }, async handler(ctx) { - const { - repoContentsUrl, - catalogInfoPath = '/catalog-info.yaml', - } = ctx.parameters; + const { parameters } = ctx; - let { catalogInfoUrl } = ctx.parameters; - if (!catalogInfoUrl) { + let catalogInfoUrl; + if ('catalogInfoUrl' in parameters) { + catalogInfoUrl = parameters.catalogInfoUrl; + } else { + const { + repoContentsUrl, + catalogInfoPath = '/catalog-info.yaml', + } = parameters; const integration = integrations.byUrl(repoContentsUrl as string); if (!integration) { throw new InputError('No integration found for host'); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 281a6e75d2..4aedb6d2af 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -19,6 +19,7 @@ import { resolve as resolvePath } from 'path'; import Docker from 'dockerode'; import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; +import { JsonObject } from '@backstage/config'; import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater'; import { TemplateAction } from '../../types'; import { fetchContents } from './helpers'; @@ -28,11 +29,34 @@ export function createFetchCookiecutterAction(options: { urlReader: UrlReader; integrations: ScmIntegrations; templaters: TemplaterBuilder; -}): TemplateAction { +}): TemplateAction<{ url: string; targetPath?: string; values: JsonObject }> { const { dockerClient, urlReader, templaters, integrations } = options; return { id: 'fetch:cookiecutter', + parameterSchema: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to cookiecutter for templating', + type: 'object', + }, + }, + }, async handler(ctx) { ctx.logger.info('Fetching and then templating using cookiecutter'); const workDir = await ctx.createTemporaryDirectory(); @@ -66,11 +90,6 @@ export function createFetchCookiecutterAction(options: { // Finally move the template result into the task workspace const targetPath = ctx.parameters.targetPath ?? './'; - if (typeof targetPath !== 'string') { - throw new InputError( - `Fetch action targetPath is not a string, got ${targetPath}`, - ); - } const outputPath = resolvePath(ctx.workspacePath, targetPath); if (!outputPath.startsWith(ctx.workspacePath)) { throw new InputError( diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index 371bc35e9a..6fd4d92611 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -23,21 +23,34 @@ import { fetchContents } from './helpers'; export function createFetchPlainAction(options: { urlReader: UrlReader; integrations: ScmIntegrations; -}): TemplateAction { +}): TemplateAction<{ url: string; targetPath?: string }> { const { urlReader, integrations } = options; return { id: 'fetch:plain', + parameterSchema: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + }, + }, async handler(ctx) { ctx.logger.info('Fetching plain content from remote URL'); // Finally move the template result into the task workspace const targetPath = ctx.parameters.targetPath ?? './'; - if (typeof targetPath !== 'string') { - throw new InputError( - `Fetch action targetPath is not a string, got ${targetPath}`, - ); - } const outputPath = resolvePath(ctx.workspacePath, targetPath); if (!outputPath.startsWith(ctx.workspacePath)) { throw new InputError( diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 26f2775c91..822854cadb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -26,7 +26,11 @@ import { initRepoAndPush } from '../../../stages/publish/helpers'; export function createPublishGithubAction(options: { integrations: ScmIntegrations; repoVisibility: 'private' | 'internal' | 'public'; -}): TemplateAction { +}): TemplateAction<{ + repoUrl: string; + description?: string; + access?: string; +}> { const { integrations, repoVisibility } = options; const credentialsProviders = new Map( @@ -38,14 +42,27 @@ export function createPublishGithubAction(options: { return { id: 'publish:github', + parameterSchema: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + access: { + title: 'Additional Repository Access', + type: 'string', + }, + }, + }, async handler(ctx) { const { repoUrl, description, access } = ctx.parameters; - if (typeof repoUrl !== 'string') { - throw new Error( - `Invalid repo URL passed to publish:github, got ${typeof repoUrl}`, - ); - } let parsed; try { parsed = new URL(`https://${repoUrl}`); @@ -103,18 +120,17 @@ export function createPublishGithubAction(options: { org: owner, private: repoVisibility !== 'public', visibility: repoVisibility, - description: description as string, + description: description, }) : client.repos.createForAuthenticatedUser({ name: repo, private: repoVisibility === 'private', - description: description as string, + description: description, }); const { data } = await repoCreationPromise; - const accessString = access as string; - if (accessString?.startsWith(`${owner}/`)) { - const [, team] = accessString.split('/'); + if (access?.startsWith(`${owner}/`)) { + const [, team] = access.split('/'); await client.teams.addOrUpdateRepoPermissionsInOrg({ org: owner, team_slug: team, @@ -122,12 +138,12 @@ export function createPublishGithubAction(options: { repo, permission: 'admin', }); - // no need to add accessString if it's the person who own's the personal account - } else if (accessString && accessString !== owner) { + // no need to add access if it's the person who own's the personal account + } else if (access && access !== owner) { await client.repos.addCollaborator({ owner, repo, - username: accessString, + username: access, permission: 'admin', }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts index aed91e35f8..bdd74e933a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts @@ -16,9 +16,14 @@ import { Logger } from 'winston'; import { Writable } from 'stream'; -import { JsonValue } from '@backstage/config'; +import { JsonValue, JsonObject } from '@backstage/config'; +import { Schema } from 'jsonschema'; -export type ActionContext = { +type PartialJsonObject = Partial; +type PartialJsonValue = PartialJsonObject | JsonValue | undefined; +export type ParameterBase = Partial<{ [name: string]: PartialJsonValue }>; + +export type ActionContext = { /** * Base URL for the location of the task spec, typically the url of the source entity file. */ @@ -28,7 +33,7 @@ export type ActionContext = { logStream: Writable; workspacePath: string; - parameters: { [name: string]: JsonValue }; + parameters: Parameters; output(name: string, value: JsonValue): void; /** @@ -37,7 +42,8 @@ export type ActionContext = { createTemporaryDirectory(): Promise; }; -export type TemplateAction = { +export type TemplateAction = { id: string; - handler: (ctx: ActionContext) => Promise; + parameterSchema?: Schema; + handler: (ctx: ActionContext) => Promise; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 357e5e815c..548083095c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -18,11 +18,13 @@ import { PassThrough } from 'stream'; import { Logger } from 'winston'; import * as winston from 'winston'; import { JsonValue, JsonObject } from '@backstage/config'; +import { validate as validateJsonSchema } from 'jsonschema'; import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import * as handlebars from 'handlebars'; +import { InputError } from '@backstage/backend-common'; type Options = { logger: Logger; @@ -111,6 +113,20 @@ export class TaskWorker { }, ); + if (action.parameterSchema) { + const validateResult = validateJsonSchema( + parameters, + action.parameterSchema, + { propertyName: 'parameters' }, + ); + if (!validateResult.valid) { + const errors = validateResult.errors.join(', '); + throw new InputError( + `Invalid parameters passed to action ${action.id}, ${errors}`, + ); + } + } + const stepOutputs: { [name: string]: JsonValue } = {}; // Keep track of all tmp dirs that are created by the action so we can remove them after From 8c1e321684237e8dc7922cef3c5ace8ec1b7855b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 25 Feb 2021 09:13:40 +0100 Subject: [PATCH 20/66] feat: porting over the other existing publishers to the new action format Signed-off-by: Johan Haals --- .../actions/builtin/publish/azure.ts | 94 ++++++++ .../actions/builtin/publish/bitbucket.ts | 217 ++++++++++++++++++ .../actions/builtin/publish/github.ts | 23 +- .../actions/builtin/publish/gitlab.ts | 102 ++++++++ .../actions/builtin/publish/index.ts | 3 + .../actions/builtin/publish/util.ts | 27 +++ 6 files changed, 445 insertions(+), 21 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts new file mode 100644 index 0000000000..0728e753b2 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../../types'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; +import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; +import { parseRepoUrl } from './util'; + +export function createPublishAzureAction(options: { + integrations: ScmIntegrations; +}): TemplateAction<{ + repoUrl: string; + description?: string; +}> { + const { integrations } = options; + + return { + id: 'publish:azure', + parameterSchema: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + }, + }, + async handler(ctx) { + const { owner, repo, host } = parseRepoUrl(ctx.parameters.repoUrl); + const integrationConfig = integrations.azure.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + if (!integrationConfig.config.token) { + throw new InputError(`No token provided for Azure Integration ${host}`); + } + const authHandler = getPersonalAccessTokenHandler( + integrationConfig.config.token, + ); + + const webApi = new WebApi(`https://${host}/${owner}`, authHandler); + const client = await webApi.getGitApi(); + const createOptions: GitRepositoryCreateOptions = { name: repo }; + const { remoteUrl } = await client.createRepository(createOptions, owner); + + if (!remoteUrl) { + throw new InputError( + 'No remote URL returned from create repository for Azure', + ); + } + + // blam: Repo contents is serialized into the path, + // so it's just the base path I think + const repoContentsUrl = remoteUrl; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'notempty', + password: integrationConfig.config.token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts new file mode 100644 index 0000000000..d530fb90bc --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -0,0 +1,217 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { + BitbucketIntegrationConfig, + ScmIntegrations, +} from '@backstage/integration'; +import { TemplateAction } from '../../types'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; + +const createBitbucketCloudRepository = async (opts: { + owner: string; + repo: string; + description: string; + repoVisibility: 'private' | 'public'; + authorization: string; +}) => { + const { owner, repo, description, repoVisibility, authorization } = opts; + + let response: Response; + + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + scm: 'git', + description: description, + is_private: repoVisibility === 'private', + }), + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + }; + try { + response = await fetch( + `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + if (response.status === 200) { + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'https') { + remoteUrl = link.href; + } + } + + // TODO use the urlReader to get the default branch + const repoContentsUrl = `${r.links.html.href}/src/master`; + return { remoteUrl, repoContentsUrl }; + } + throw new Error(`Not a valid response code ${await response.text()}`); +}; + +const createBitbucketServerRepository = async (opts: { + host: string; + owner: string; + repo: string; + description: string; + repoVisibility: 'private' | 'public'; + authorization: string; +}) => { + const { + host, + owner, + repo, + description, + authorization, + repoVisibility, + } = opts; + + let response: Response; + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + name: repo, + description: description, + is_private: repoVisibility === 'private', + }), + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + }; + + try { + response = await fetch( + `https://${host}/rest/api/1.0/projects/${owner}/repos`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + if (response.status === 201) { + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'http') { + remoteUrl = link.href; + } + } + const repoContentsUrl = `${r.links.self[0].href}`; + return { remoteUrl, repoContentsUrl }; + } + throw new Error(`Not a valid response code ${await response.text()}`); +}; + +const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { + if (config.username && config.appPassword) { + const buffer = Buffer.from( + `${config.username}:${config.appPassword}`, + 'utf8', + ); + + return `Basic ${buffer.toString('base64')}`; + } + + if (config.token) { + return `Bearer ${config.token}`; + } + + throw new Error( + `Authorization has not been provided for Bitbucket. Please add either username + appPassword or token to the Integrations config`, + ); +}; + +export function createPublishBitbucketAction(options: { + integrations: ScmIntegrations; + repoVisibility: 'private' | 'public'; +}): TemplateAction<{ + repoUrl: string; + description: string; +}> { + const { integrations, repoVisibility } = options; + + return { + id: 'publish:bitbucket', + parameterSchema: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + }, + }, + async handler(ctx) { + const { repoUrl, description } = ctx.parameters; + + const { owner, repo, host } = parseRepoUrl(repoUrl); + + const integrationConfig = integrations.bitbucket.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + + const authorization = getAuthorizationHeader(integrationConfig.config); + + const createMethod = + host === 'bitbucket.org' + ? createBitbucketCloudRepository + : createBitbucketServerRepository; + + const { remoteUrl, repoContentsUrl } = await createMethod({ + authorization, + host, + owner, + repo, + repoVisibility, + description, + }); + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: integrationConfig.config.username + ? integrationConfig.config.username + : 'x-token-auth', + password: integrationConfig.config.appPassword + ? integrationConfig.config.appPassword + : integrationConfig.config.token ?? '', + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 822854cadb..f4a7b38c50 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -22,6 +22,7 @@ import { import { Octokit } from '@octokit/rest'; import { TemplateAction } from '../../types'; import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; export function createPublishGithubAction(options: { integrations: ScmIntegrations; @@ -63,27 +64,7 @@ export function createPublishGithubAction(options: { async handler(ctx) { const { repoUrl, description, access } = ctx.parameters; - let parsed; - try { - parsed = new URL(`https://${repoUrl}`); - } catch (error) { - throw new InputError( - `Invalid repo URL passed to publish:github, got ${repoUrl}, ${error}`, - ); - } - const host = parsed.host; - const owner = parsed.searchParams.get('owner'); - if (!owner) { - throw new InputError( - `Invalid repo URL passed to publish:github, missing owner`, - ); - } - const repo = parsed.searchParams.get('repo'); - if (!repo) { - throw new InputError( - `Invalid repo URL passed to publish:github, missing repo`, - ); - } + const { owner, repo, host } = parseRepoUrl(repoUrl); const credentialsProvider = credentialsProviders.get(host); const integrationConfig = integrations.github.byHost(host); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts new file mode 100644 index 0000000000..f5af2885aa --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../../types'; +import { Gitlab } from '@gitbeaker/node'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; + +export function createPublishGitlabAction(options: { + integrations: ScmIntegrations; + repoVisibility: 'private' | 'internal' | 'public'; +}): TemplateAction<{ + repoUrl: string; +}> { + const { integrations, repoVisibility } = options; + + return { + id: 'publish:gitlab', + parameterSchema: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + }, + }, + async handler(ctx) { + const { repoUrl } = ctx.parameters; + + const { owner, repo, host } = parseRepoUrl(repoUrl); + + const integrationConfig = integrations.gitlab.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + + if (!integrationConfig.config.token) { + throw new InputError( + `No token provided for GitLab Integration ${host}`, + ); + } + + const client = new Gitlab({ + host: integrationConfig.config.baseUrl, + token: integrationConfig.config.token, + }); + + let { id: targetNamespace } = (await client.Namespaces.show(owner)) as { + id: number; + }; + + if (!targetNamespace) { + const { id } = (await client.Users.current()) as { + id: number; + }; + targetNamespace = id; + } + + const { http_url_to_repo } = await client.Projects.create({ + namespace_id: targetNamespace, + name: repo, + visibility: repoVisibility, + }); + + const remoteUrl = (http_url_to_repo as string).replace(/\.git$/, ''); + const repoContentsUrl = `${remoteUrl}/-/blob/master`; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'oauth2', + password: integrationConfig.config.token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts index b1ecb91f9a..70a7909144 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts @@ -15,3 +15,6 @@ */ export { createPublishGithubAction } from './github'; +export { createPublishAzureAction } from './azure'; +export { createPublishGitlabAction } from './gitlab'; +export { createPublishBitbucketAction } from './bitbucket'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts new file mode 100644 index 0000000000..2fbda2b0b7 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts @@ -0,0 +1,27 @@ +import { InputError } from '@backstage/backend-common'; +export const parseRepoUrl = (repoUrl: string) => { + let parsed; + try { + parsed = new URL(`https://${repoUrl}`); + } catch (error) { + throw new InputError( + `Invalid repo URL passed to publisher, got ${repoUrl}, ${error}`, + ); + } + const host = parsed.host; + const owner = parsed.searchParams.get('owner'); + + if (!owner) { + throw new InputError( + `Invalid repo URL passed to publisher: ${repoUrl}, missing owner`, + ); + } + const repo = parsed.searchParams.get('repo'); + if (!repo) { + throw new InputError( + `Invalid repo URL passed to publisher: ${repoUrl}, missing repo`, + ); + } + + return { host, owner, repo }; +}; From c19f41494d88747185c1dadd39c44dffa64cf320 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 25 Feb 2021 09:21:31 +0100 Subject: [PATCH 21/66] chore: add to the default router Signed-off-by: Johan Haals --- .../actions/builtin/publish/util.ts | 1 + .../scaffolder-backend/src/service/router.ts | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts index 2fbda2b0b7..8a67c9c5f0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts @@ -1,4 +1,5 @@ import { InputError } from '@backstage/backend-common'; + export const parseRepoUrl = (repoUrl: string) => { let parsed; try { diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index f30cb9ce3a..7987e1901d 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -58,6 +58,9 @@ import { createFetchPlainAction, createFetchCookiecutterAction, createPublishGithubAction, + createPublishBitbucketAction, + createPublishAzureAction, + createPublishGitlabAction, createCatalogRegisterAction, } from '../scaffolder/actions/builtin'; import { ScmIntegrations } from '@backstage/integration'; @@ -152,6 +155,25 @@ export async function createRouter( repoVisibility: 'public', }), ); + actionRegistry.register( + createPublishGitlabAction({ + integrations, + repoVisibility: 'public', + }), + ); + + actionRegistry.register( + createPublishBitbucketAction({ + integrations, + repoVisibility: 'public', + }), + ); + + actionRegistry.register( + createPublishAzureAction({ + integrations, + }), + ); actionRegistry.register( createCatalogRegisterAction({ catalogClient, integrations }), From b853e084dc8192ec067203b083d6fefb739d5e21 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 10:13:06 +0100 Subject: [PATCH 22/66] scaffolder: Fix providers Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../test-template-v2/template.yaml | 215 ++++++++++++++++++ .../actions/builtin/publish/azure.ts | 23 +- .../actions/builtin/publish/bitbucket.ts | 1 + .../actions/builtin/publish/gitlab.ts | 2 +- .../actions/builtin/publish/util.ts | 20 +- .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 95 ++++++-- 6 files changed, 334 insertions(+), 22 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 27b744664a..3856906454 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -138,6 +138,221 @@ spec: catalogInfoPath: '/catalog-info.yaml' + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' + +--- + +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: test-template-v2-gitlab + title: Test Template v2-gitlab + description: Testing out the new template schema +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - gitlab.com + + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + parameters: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + parameters: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:gitlab + parameters: + allowedHosts: ['gitlab.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + parameters: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' + +--- + +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: test-template-v2-bitbucket + title: Test Template v2-bitbucket + description: Testing out the new template schema +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - bitbucket.org + + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + parameters: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + parameters: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:bitbucket + parameters: + allowedHosts: ['bitbucket.org'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + parameters: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' + +--- +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: test-template-v2-azure + title: Test Template v2-azure + description: Testing out the new template schema +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - dev.azure.com + + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + parameters: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + parameters: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:azure + parameters: + allowedHosts: ['dev.azure.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + parameters: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + output: remoteUrl: '{{ steps.publish.output.remoteUrl }}' entityRef: '{{ steps.register.output.entityRef }}' diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 0728e753b2..be2f41d06c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -47,7 +47,16 @@ export function createPublishAzureAction(options: { }, }, async handler(ctx) { - const { owner, repo, host } = parseRepoUrl(ctx.parameters.repoUrl); + const { owner, repo, host, organization } = parseRepoUrl( + ctx.parameters.repoUrl, + ); + + if (!organization) { + throw new InputError( + `No Organization was included in the repo URL to create ${ctx.parameters.repoUrl}`, + ); + } + const integrationConfig = integrations.azure.byHost(host); if (!integrationConfig) { @@ -62,10 +71,18 @@ export function createPublishAzureAction(options: { integrationConfig.config.token, ); - const webApi = new WebApi(`https://${host}/${owner}`, authHandler); + const webApi = new WebApi(`https://${host}/${organization}`, authHandler); const client = await webApi.getGitApi(); const createOptions: GitRepositoryCreateOptions = { name: repo }; - const { remoteUrl } = await client.createRepository(createOptions, owner); + const returnedRepo = await client.createRepository(createOptions, owner); + + if (!returnedRepo) { + throw new InputError( + `Unable to create the repository with Organization ${organization}, Project ${owner} and Repo ${repo}. + Please make sure you that both the Org and Project are typed corrected and exist.`, + ); + } + const remoteUrl = returnedRepo.remoteUrl; if (!remoteUrl) { throw new InputError( diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index d530fb90bc..bc0c1f97c7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -22,6 +22,7 @@ import { import { TemplateAction } from '../../types'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; +import fetch from 'cross-fetch'; const createBitbucketCloudRepository = async (opts: { owner: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index f5af2885aa..67ba402354 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -87,7 +87,7 @@ export function createPublishGitlabAction(options: { await initRepoAndPush({ dir: ctx.workspacePath, - remoteUrl, + remoteUrl: http_url_to_repo as string, auth: { username: 'oauth2', password: integrationConfig.config.token, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts index 8a67c9c5f0..9349d5b5eb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts @@ -1,3 +1,19 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import { InputError } from '@backstage/backend-common'; export const parseRepoUrl = (repoUrl: string) => { @@ -24,5 +40,7 @@ export const parseRepoUrl = (repoUrl: string) => { ); } - return { host, owner, repo }; + const organization = parsed.searchParams.get('organization'); + + return { host, owner, repo, organization }; }; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index f346a65a4b..31b4c4fa9b 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -28,6 +28,7 @@ function splitFormData(url: string | undefined) { let host = undefined; let owner = undefined; let repo = undefined; + let organization = undefined; try { if (url) { @@ -35,18 +36,21 @@ function splitFormData(url: string | undefined) { host = parsed.host; owner = parsed.searchParams.get('owner') || undefined; repo = parsed.searchParams.get('repo') || undefined; + // This is azure dev ops specific. not used for any other provider. + organization = parsed.searchParams.get('organization') || undefined; } } catch { /* ok */ } - return { host, owner, repo }; + return { host, owner, repo, organization }; } function serializeFormData(data: { host?: string; owner?: string; repo?: string; + organization?: string; }) { if (!data.host) { return undefined; @@ -58,6 +62,9 @@ function serializeFormData(data: { if (data.repo) { params.set('repo', data.repo); } + if (data.organization) { + params.set('organization', data.organization); + } return `${data.host}?${params.toString()}`; } @@ -75,36 +82,71 @@ export const RepoUrlPicker: Field = ({ return await api.getIntegrationsList({ allowedHosts }); }); - const { host, owner, repo } = splitFormData(formData); + const { host, owner, repo, organization } = splitFormData(formData); const updateHost = useCallback( (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => onChange( - serializeFormData({ host: evt.target.value as string, owner, repo }), + serializeFormData({ + host: evt.target.value as string, + owner, + repo, + organization, + }), ), - [onChange, owner, repo], + [onChange, owner, repo, organization], ); const updateOwner = useCallback( (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => onChange( - serializeFormData({ host, owner: evt.target.value as string, repo }), + serializeFormData({ + host, + owner: evt.target.value as string, + repo, + organization, + }), ), - [onChange, host, repo], + [onChange, host, repo, organization], ); const updateRepo = useCallback( (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => onChange( - serializeFormData({ host, owner, repo: evt.target.value as string }), + serializeFormData({ + host, + owner, + repo: evt.target.value as string, + organization, + }), ), - [onChange, host, owner], + [onChange, host, owner, organization], + ); + + const updateOrganization = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ + host, + owner, + repo, + organization: evt.target.value as string, + }), + ), + [onChange, host, owner, repo], ); useEffect(() => { if (host === undefined && integrations?.length) { - onChange(serializeFormData({ host: integrations[0].host, owner, repo })); + onChange( + serializeFormData({ + host: integrations[0].host, + owner, + repo, + organization, + }), + ); } - }, [onChange, integrations, host, owner, repo]); + }, [onChange, integrations, host, owner, repo, organization]); if (loading) { return ; @@ -119,18 +161,37 @@ export const RepoUrlPicker: Field = ({ > Host The host where the repository will be created + {host === 'dev.azure.com' && ( + 0 && !organization} + > + Organization + + The name of the organization + + )} Date: Thu, 25 Feb 2021 11:18:17 +0100 Subject: [PATCH 23/66] Rename urlReader to reader Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- packages/backend/src/plugins/scaffolder.ts | 2 +- .../src/scaffolder/actions/builtin/fetch/cookiecutter.ts | 6 +++--- .../src/scaffolder/actions/builtin/fetch/helpers.ts | 6 +++--- .../src/scaffolder/actions/builtin/fetch/plain.ts | 6 +++--- plugins/scaffolder-backend/src/service/router.ts | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 9cab56d7f6..fca3f79270 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -58,6 +58,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, - urlReader: reader, + reader, }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 4aedb6d2af..c913c20807 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -26,11 +26,11 @@ import { fetchContents } from './helpers'; export function createFetchCookiecutterAction(options: { dockerClient: Docker; - urlReader: UrlReader; + reader: UrlReader; integrations: ScmIntegrations; templaters: TemplaterBuilder; }): TemplateAction<{ url: string; targetPath?: string; values: JsonObject }> { - const { dockerClient, urlReader, templaters, integrations } = options; + const { dockerClient, reader, templaters, integrations } = options; return { id: 'fetch:cookiecutter', @@ -68,7 +68,7 @@ export function createFetchCookiecutterAction(options: { const resultDir = resolvePath(workDir, 'result'); await fetchContents({ - urlReader, + reader, integrations, baseUrl: ctx.baseUrl, fetchUrl: ctx.parameters.url, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts index a739e53e1f..652a1a56ca 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts @@ -21,13 +21,13 @@ import { ScmIntegrations } from '@backstage/integration'; import { JsonValue } from '@backstage/config'; export async function fetchContents({ - urlReader, + reader, integrations, baseUrl, fetchUrl = '.', outputPath, }: { - urlReader: UrlReader; + reader: UrlReader; integrations: ScmIntegrations; baseUrl?: string; fetchUrl?: JsonValue; @@ -79,7 +79,7 @@ export async function fetchContents({ ); } - const res = await urlReader.readTree(readUrl); + const res = await reader.readTree(readUrl); await fs.ensureDir(outputPath); await res.dir({ targetDir: outputPath }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index 6fd4d92611..1fbc3085ee 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -21,10 +21,10 @@ import { TemplateAction } from '../../types'; import { fetchContents } from './helpers'; export function createFetchPlainAction(options: { - urlReader: UrlReader; + reader: UrlReader; integrations: ScmIntegrations; }): TemplateAction<{ url: string; targetPath?: string }> { - const { urlReader, integrations } = options; + const { reader, integrations } = options; return { id: 'fetch:plain', @@ -59,7 +59,7 @@ export function createFetchPlainAction(options: { } await fetchContents({ - urlReader, + reader, integrations, baseUrl: ctx.baseUrl, fetchUrl: ctx.parameters.url, diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 7987e1901d..a820cc6919 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -72,7 +72,7 @@ export interface RouterOptions { logger: Logger; config: Config; - urlReader: UrlReader; + reader: UrlReader; dockerClient: Docker; database: PluginDatabaseManager; catalogClient: CatalogApi; @@ -105,7 +105,7 @@ export async function createRouter( publishers, logger: parentLogger, config, - urlReader, + reader, dockerClient, database, catalogClient, @@ -137,13 +137,13 @@ export async function createRouter( }); actionRegistry.register( createFetchPlainAction({ - urlReader, + reader, integrations, }), ); actionRegistry.register( createFetchCookiecutterAction({ - urlReader, + reader, integrations, dockerClient, templaters, From 664f336c42669f043b8ab016324b829ccd7f0850 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 11:35:11 +0100 Subject: [PATCH 24/66] Update TemplateAction schema to include input and output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelรถw Signed-off-by: Johan Haals --- .../actions/builtin/fetch/cookiecutter.ts | 42 +++++++++--------- .../scaffolder/actions/builtin/fetch/plain.ts | 33 +++++++------- .../actions/builtin/publish/azure.ts | 37 +++++++++++----- .../actions/builtin/publish/bitbucket.ts | 35 ++++++++++----- .../actions/builtin/publish/github.ts | 43 +++++++++++++------ .../actions/builtin/publish/gitlab.ts | 29 ++++++++++--- .../src/scaffolder/actions/types.ts | 5 ++- .../src/scaffolder/tasks/TaskWorker.ts | 6 +-- .../src/scaffolder/tasks/types.ts | 2 +- 9 files changed, 150 insertions(+), 82 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index c913c20807..f3b2dbf559 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -34,26 +34,28 @@ export function createFetchCookiecutterAction(options: { return { id: 'fetch:cookiecutter', - parameterSchema: { - type: 'object', - required: ['url'], - properties: { - url: { - title: 'Fetch URL', - description: - 'Relative path or absolute URL pointing to the directory tree to fetch', - type: 'string', - }, - targetPath: { - title: 'Target Path', - description: - 'Target path within the working directory to download the contents to.', - type: 'string', - }, - values: { - title: 'Template Values', - description: 'Values to pass on to cookiecutter for templating', - type: 'object', + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to cookiecutter for templating', + type: 'object', + }, }, }, }, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index 1fbc3085ee..3284e1ec21 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -28,23 +28,26 @@ export function createFetchPlainAction(options: { return { id: 'fetch:plain', - parameterSchema: { - type: 'object', - required: ['url'], - properties: { - url: { - title: 'Fetch URL', - description: - 'Relative path or absolute URL pointing to the directory tree to fetch', - type: 'string', - }, - targetPath: { - title: 'Target Path', - description: - 'Target path within the working directory to download the contents to.', - type: 'string', + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, }, }, + output: {}, }, async handler(ctx) { ctx.logger.info('Fetching plain content from remote URL'); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index be2f41d06c..40f75c7a66 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -32,17 +32,32 @@ export function createPublishAzureAction(options: { return { id: 'publish:azure', - parameterSchema: { - type: 'object', - required: ['repoUrl'], - properties: { - repoUrl: { - title: 'Repository Location', - type: 'string', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, }, - description: { - title: 'Repository Description', - type: 'string', + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, }, }, }, @@ -78,7 +93,7 @@ export function createPublishAzureAction(options: { if (!returnedRepo) { throw new InputError( - `Unable to create the repository with Organization ${organization}, Project ${owner} and Repo ${repo}. + `Unable to create the repository with Organization ${organization}, Project ${owner} and Repo ${repo}. Please make sure you that both the Org and Project are typed corrected and exist.`, ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index bc0c1f97c7..bef4af4b7f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -154,17 +154,32 @@ export function createPublishBitbucketAction(options: { return { id: 'publish:bitbucket', - parameterSchema: { - type: 'object', - required: ['repoUrl'], - properties: { - repoUrl: { - title: 'Repository Location', - type: 'string', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, }, - description: { - title: 'Repository Description', - type: 'string', + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, }, }, }, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index f4a7b38c50..817250c90c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -43,21 +43,36 @@ export function createPublishGithubAction(options: { return { id: 'publish:github', - parameterSchema: { - type: 'object', - required: ['repoUrl'], - properties: { - repoUrl: { - title: 'Repository Location', - type: 'string', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + access: { + title: 'Repository Access', + type: 'string', + }, }, - description: { - title: 'Repository Description', - type: 'string', - }, - access: { - title: 'Additional Repository Access', - type: 'string', + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, }, }, }, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index 67ba402354..a0b60018b6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -31,13 +31,28 @@ export function createPublishGitlabAction(options: { return { id: 'publish:gitlab', - parameterSchema: { - type: 'object', - required: ['repoUrl'], - properties: { - repoUrl: { - title: 'Repository Location', - type: 'string', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + }, + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, }, }, }, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts index bdd74e933a..11523f98e6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts @@ -44,6 +44,9 @@ export type ActionContext = { export type TemplateAction = { id: string; - parameterSchema?: Schema; + schema?: { + input?: Schema; + output?: Schema; + }; handler: (ctx: ActionContext) => Promise; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 548083095c..08d641f9ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -99,7 +99,7 @@ export class TaskWorker { } const parameters = JSON.parse( - JSON.stringify(step.parameters), + JSON.stringify(step.input), (_key, value) => { if (typeof value === 'string') { return handlebars.compile(value, { @@ -113,10 +113,10 @@ export class TaskWorker { }, ); - if (action.parameterSchema) { + if (action.schema?.input) { const validateResult = validateJsonSchema( parameters, - action.parameterSchema, + action.schema, { propertyName: 'parameters' }, ); if (!validateResult.valid) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 3a1d207a04..6963817a1e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -49,7 +49,7 @@ export type TaskSpec = { id: string; name: string; action: string; - parameters?: JsonObject; + input?: JsonObject; }>; output: { [name: string]: string }; }; From ec5c3b82bfd007b3812e336b66db965716c904a1 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 13:15:39 +0100 Subject: [PATCH 25/66] Convert repoVisibility into template parameter Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../actions/builtin/publish/bitbucket.ts | 15 ++++++++++++--- .../scaffolder/actions/builtin/publish/github.ts | 11 ++++++++--- .../scaffolder/actions/builtin/publish/gitlab.ts | 11 ++++++++--- plugins/scaffolder-backend/src/service/router.ts | 3 --- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index bef4af4b7f..325727775f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -145,12 +145,12 @@ const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { export function createPublishBitbucketAction(options: { integrations: ScmIntegrations; - repoVisibility: 'private' | 'public'; }): TemplateAction<{ repoUrl: string; description: string; + repoVisibility: 'private' | 'public'; }> { - const { integrations, repoVisibility } = options; + const { integrations } = options; return { id: 'publish:bitbucket', @@ -167,6 +167,11 @@ export function createPublishBitbucketAction(options: { title: 'Repository Description', type: 'string', }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public'], + }, }, }, output: { @@ -184,7 +189,11 @@ export function createPublishBitbucketAction(options: { }, }, async handler(ctx) { - const { repoUrl, description } = ctx.parameters; + const { + repoUrl, + description, + repoVisibility = 'private', + } = ctx.parameters; const { owner, repo, host } = parseRepoUrl(repoUrl); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 817250c90c..251f7ed08f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -26,13 +26,13 @@ import { parseRepoUrl } from './util'; export function createPublishGithubAction(options: { integrations: ScmIntegrations; - repoVisibility: 'private' | 'internal' | 'public'; }): TemplateAction<{ repoUrl: string; description?: string; access?: string; + repoVisibility: 'private' | 'internal' | 'public'; }> { - const { integrations, repoVisibility } = options; + const { integrations } = options; const credentialsProviders = new Map( integrations.github.list().map(integration => { @@ -60,6 +60,11 @@ export function createPublishGithubAction(options: { title: 'Repository Access', type: 'string', }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public', 'internal'], + }, }, }, output: { @@ -77,7 +82,7 @@ export function createPublishGithubAction(options: { }, }, async handler(ctx) { - const { repoUrl, description, access } = ctx.parameters; + const { repoUrl, description, access, repoVisibility } = ctx.parameters; const { owner, repo, host } = parseRepoUrl(repoUrl); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index a0b60018b6..be8d8fb0be 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -23,11 +23,11 @@ import { parseRepoUrl } from './util'; export function createPublishGitlabAction(options: { integrations: ScmIntegrations; - repoVisibility: 'private' | 'internal' | 'public'; }): TemplateAction<{ repoUrl: string; + repoVisibility: 'private' | 'internal' | 'public'; }> { - const { integrations, repoVisibility } = options; + const { integrations } = options; return { id: 'publish:gitlab', @@ -40,6 +40,11 @@ export function createPublishGitlabAction(options: { title: 'Repository Location', type: 'string', }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public', 'internal'], + }, }, }, output: { @@ -57,7 +62,7 @@ export function createPublishGitlabAction(options: { }, }, async handler(ctx) { - const { repoUrl } = ctx.parameters; + const { repoUrl, repoVisibility = 'private' } = ctx.parameters; const { owner, repo, host } = parseRepoUrl(repoUrl); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a820cc6919..e9bfd0e74a 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -152,20 +152,17 @@ export async function createRouter( actionRegistry.register( createPublishGithubAction({ integrations, - repoVisibility: 'public', }), ); actionRegistry.register( createPublishGitlabAction({ integrations, - repoVisibility: 'public', }), ); actionRegistry.register( createPublishBitbucketAction({ integrations, - repoVisibility: 'public', }), ); From 3a54420fa15b627f5db9bf43b914bf5f50e8e2ef Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 13:17:12 +0100 Subject: [PATCH 26/66] chore: remove output object Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/fetch/plain.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index 3284e1ec21..b86e72746e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -47,7 +47,6 @@ export function createFetchPlainAction(options: { }, }, }, - output: {}, }, async handler(ctx) { ctx.logger.info('Fetching plain content from remote URL'); From 807cf164af712e14885de905c1d69511ad8f7814 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 13:55:37 +0100 Subject: [PATCH 27/66] Wrap actions in createTemplateAction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename paramers to input Co-authored-by: Ben Lambert Co-authored-by: Patrik Oldsberg Co-authored-by: Fredrik Adelรถw Signed-off-by: Johan Haals --- .../actions/TemplateActionRegistry.ts | 6 +- .../actions/builtin/catalog/register.ts | 98 ++++++++++--------- .../actions/builtin/fetch/cookiecutter.ts | 18 ++-- .../scaffolder/actions/builtin/fetch/plain.ts | 12 +-- .../actions/builtin/publish/azure.ts | 18 ++-- .../actions/builtin/publish/bitbucket.ts | 85 +++++++++------- .../actions/builtin/publish/github.ts | 22 ++--- .../actions/builtin/publish/gitlab.ts | 16 +-- .../actions/createTemplateAction.ts | 28 ++++++ .../src/scaffolder/actions/index.ts | 1 + .../src/scaffolder/actions/types.ts | 10 +- .../src/scaffolder/stages/legacy.ts | 8 +- .../src/scaffolder/tasks/TaskWorker.ts | 14 ++- .../src/scaffolder/tasks/TemplateConverter.ts | 8 +- .../src/service/router.test.ts | 8 +- 15 files changed, 198 insertions(+), 154 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts index 62405679d0..2a841a564b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts @@ -14,15 +14,13 @@ * limitations under the License. */ -import { ParameterBase, TemplateAction } from './types'; +import { InputBase, TemplateAction } from './types'; import { ConflictError, NotFoundError } from '@backstage/backend-common'; export class TemplateActionRegistry { private readonly actions = new Map>(); - register( - action: TemplateAction, - ) { + register(action: TemplateAction) { if (this.actions.has(action.id)) { throw new ConflictError( `Template action with ID '${action.id}' has already been registered`, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts index 275ed5ec69..b23e41b736 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -18,72 +18,76 @@ import { InputError } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; import { getEntityName } from '@backstage/catalog-model'; -import { TemplateAction } from '../../types'; +import { createTemplateAction } from '../../createTemplateAction'; export function createCatalogRegisterAction(options: { catalogClient: CatalogApi; integrations: ScmIntegrations; -}): TemplateAction< - | { catalogInfoUrl: string } - | { repoContentsUrl: string; catalogInfoPath?: string } -> { +}) { const { catalogClient, integrations } = options; - return { + return createTemplateAction< + | { catalogInfoUrl: string } + | { repoContentsUrl: string; catalogInfoPath?: string } + >({ id: 'catalog:register', - parameterSchema: { - oneOf: [ - { - type: 'object', - required: ['catalogInfoUrl'], - properties: { - catalogInfoUrl: { - title: 'Catalog Info URL', - description: - 'An absolute URL pointing to the catalog info file location', - type: 'string', + schema: { + input: { + oneOf: [ + { + type: 'object', + required: ['catalogInfoUrl'], + properties: { + catalogInfoUrl: { + title: 'Catalog Info URL', + description: + 'An absolute URL pointing to the catalog info file location', + type: 'string', + }, }, }, - }, - { - type: 'object', - required: ['repoContentsUrl'], - properties: { - repoContentsUrl: { - title: 'Repository Contents URL', - description: - 'An absolute URL pointing to the root of a repository directory tree', - type: 'string', - }, - catalogInfoPath: { - title: 'Fetch URL', - description: - 'A relative path from the repo root pointing to the catalog info file, defaults to /catalog-info.yaml', - type: 'string', + { + type: 'object', + required: ['repoContentsUrl'], + properties: { + repoContentsUrl: { + title: 'Repository Contents URL', + description: + 'An absolute URL pointing to the root of a repository directory tree', + type: 'string', + }, + catalogInfoPath: { + title: 'Fetch URL', + description: + 'A relative path from the repo root pointing to the catalog info file, defaults to /catalog-info.yaml', + type: 'string', + }, }, }, - }, - ], + ], + }, }, async handler(ctx) { - const { parameters } = ctx; + const { input } = ctx; let catalogInfoUrl; - if ('catalogInfoUrl' in parameters) { - catalogInfoUrl = parameters.catalogInfoUrl; + if ('catalogInfoUrl' in input) { + catalogInfoUrl = input.catalogInfoUrl; } else { const { repoContentsUrl, catalogInfoPath = '/catalog-info.yaml', - } = parameters; - const integration = integrations.byUrl(repoContentsUrl as string); + } = input; + const integration = integrations.byUrl(repoContentsUrl); if (!integration) { - throw new InputError('No integration found for host'); + throw new InputError( + `No integration found for host ${repoContentsUrl}`, + ); } catalogInfoUrl = integration.resolveUrl({ - base: repoContentsUrl as string, - url: catalogInfoPath as string, + base: repoContentsUrl, + url: catalogInfoPath, }); } @@ -91,13 +95,15 @@ export function createCatalogRegisterAction(options: { const result = await catalogClient.addLocation({ type: 'url', - target: catalogInfoUrl as string, + target: catalogInfoUrl, }); if (result.entities.length >= 1) { const { kind, name, namespace } = getEntityName(result.entities[0]); ctx.output('entityRef', `${kind}:${namespace}/${name}`); - ctx.output('catalogInfoUrl', catalogInfoUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } } }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index f3b2dbf559..4016fb69c5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -21,18 +21,22 @@ import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; import { JsonObject } from '@backstage/config'; import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater'; -import { TemplateAction } from '../../types'; import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; export function createFetchCookiecutterAction(options: { dockerClient: Docker; reader: UrlReader; integrations: ScmIntegrations; templaters: TemplaterBuilder; -}): TemplateAction<{ url: string; targetPath?: string; values: JsonObject }> { +}) { const { dockerClient, reader, templaters, integrations } = options; - return { + return createTemplateAction<{ + url: string; + targetPath?: string; + values: JsonObject; + }>({ id: 'fetch:cookiecutter', schema: { input: { @@ -73,7 +77,7 @@ export function createFetchCookiecutterAction(options: { reader, integrations, baseUrl: ctx.baseUrl, - fetchUrl: ctx.parameters.url, + fetchUrl: ctx.input.url, outputPath: templateContentsDir, }); @@ -87,11 +91,11 @@ export function createFetchCookiecutterAction(options: { workspacePath: workDir, dockerClient, logStream: ctx.logStream, - values: ctx.parameters.values as TemplaterValues, + values: ctx.input.values as TemplaterValues, }); // Finally move the template result into the task workspace - const targetPath = ctx.parameters.targetPath ?? './'; + const targetPath = ctx.input.targetPath ?? './'; const outputPath = resolvePath(ctx.workspacePath, targetPath); if (!outputPath.startsWith(ctx.workspacePath)) { throw new InputError( @@ -100,5 +104,5 @@ export function createFetchCookiecutterAction(options: { } await fs.copy(resultDir, outputPath); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index b86e72746e..52969fcf6b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -17,16 +17,16 @@ import { resolve as resolvePath } from 'path'; import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../../types'; import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; export function createFetchPlainAction(options: { reader: UrlReader; integrations: ScmIntegrations; -}): TemplateAction<{ url: string; targetPath?: string }> { +}) { const { reader, integrations } = options; - return { + return createTemplateAction<{ url: string; targetPath?: string }>({ id: 'fetch:plain', schema: { input: { @@ -52,7 +52,7 @@ export function createFetchPlainAction(options: { ctx.logger.info('Fetching plain content from remote URL'); // Finally move the template result into the task workspace - const targetPath = ctx.parameters.targetPath ?? './'; + const targetPath = ctx.input.targetPath ?? './'; const outputPath = resolvePath(ctx.workspacePath, targetPath); if (!outputPath.startsWith(ctx.workspacePath)) { throw new InputError( @@ -64,9 +64,9 @@ export function createFetchPlainAction(options: { reader, integrations, baseUrl: ctx.baseUrl, - fetchUrl: ctx.parameters.url, + fetchUrl: ctx.input.url, outputPath, }); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 40f75c7a66..c79048b289 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -16,21 +16,21 @@ import { InputError } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../../types'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; export function createPublishAzureAction(options: { integrations: ScmIntegrations; -}): TemplateAction<{ - repoUrl: string; - description?: string; -}> { +}) { const { integrations } = options; - return { + return createTemplateAction<{ + repoUrl: string; + description?: string; + }>({ id: 'publish:azure', schema: { input: { @@ -63,12 +63,12 @@ export function createPublishAzureAction(options: { }, async handler(ctx) { const { owner, repo, host, organization } = parseRepoUrl( - ctx.parameters.repoUrl, + ctx.input.repoUrl, ); if (!organization) { throw new InputError( - `No Organization was included in the repo URL to create ${ctx.parameters.repoUrl}`, + `No Organization was included in the repo URL to create ${ctx.input.repoUrl}`, ); } @@ -122,5 +122,5 @@ export function createPublishAzureAction(options: { ctx.output('remoteUrl', remoteUrl); ctx.output('repoContentsUrl', repoContentsUrl); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 325727775f..533595eb3c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -19,10 +19,10 @@ import { BitbucketIntegrationConfig, ScmIntegrations, } from '@backstage/integration'; -import { TemplateAction } from '../../types'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; import fetch from 'cross-fetch'; +import { createTemplateAction } from '../../createTemplateAction'; const createBitbucketCloudRepository = async (opts: { owner: string; @@ -33,8 +33,6 @@ const createBitbucketCloudRepository = async (opts: { }) => { const { owner, repo, description, repoVisibility, authorization } = opts; - let response: Response; - const options: RequestInit = { method: 'POST', body: JSON.stringify({ @@ -47,6 +45,8 @@ const createBitbucketCloudRepository = async (opts: { 'Content-Type': 'application/json', }, }; + + let response: Response; try { response = await fetch( `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`, @@ -55,20 +55,26 @@ const createBitbucketCloudRepository = async (opts: { } catch (e) { throw new Error(`Unable to create repository, ${e}`); } - if (response.status === 200) { - const r = await response.json(); - let remoteUrl = ''; - for (const link of r.links.clone) { - if (link.name === 'https') { - remoteUrl = link.href; - } - } - // TODO use the urlReader to get the default branch - const repoContentsUrl = `${r.links.html.href}/src/master`; - return { remoteUrl, repoContentsUrl }; + if (response.status !== 200) { + throw new Error( + `Unable to create repository, ${response.status} ${ + response.statusText + }, ${await response.text()}`, + ); } - throw new Error(`Not a valid response code ${await response.text()}`); + + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'https') { + remoteUrl = link.href; + } + } + + // TODO use the urlReader to get the default branch + const repoContentsUrl = `${r.links.html.href}/src/master`; + return { remoteUrl, repoContentsUrl }; }; const createBitbucketServerRepository = async (opts: { @@ -110,18 +116,25 @@ const createBitbucketServerRepository = async (opts: { } catch (e) { throw new Error(`Unable to create repository, ${e}`); } - if (response.status === 201) { - const r = await response.json(); - let remoteUrl = ''; - for (const link of r.links.clone) { - if (link.name === 'http') { - remoteUrl = link.href; - } - } - const repoContentsUrl = `${r.links.self[0].href}`; - return { remoteUrl, repoContentsUrl }; + + if (response.status !== 201) { + throw new Error( + `Unable to create repository, ${response.status} ${ + response.statusText + }, ${await response.text()}`, + ); } - throw new Error(`Not a valid response code ${await response.text()}`); + + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'http') { + remoteUrl = link.href; + } + } + + const repoContentsUrl = `${r.links.self[0].href}`; + return { remoteUrl, repoContentsUrl }; }; const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { @@ -145,14 +158,14 @@ const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { export function createPublishBitbucketAction(options: { integrations: ScmIntegrations; -}): TemplateAction<{ - repoUrl: string; - description: string; - repoVisibility: 'private' | 'public'; -}> { +}) { const { integrations } = options; - return { + return createTemplateAction<{ + repoUrl: string; + description: string; + repoVisibility: 'private' | 'public'; + }>({ id: 'publish:bitbucket', schema: { input: { @@ -189,11 +202,7 @@ export function createPublishBitbucketAction(options: { }, }, async handler(ctx) { - const { - repoUrl, - description, - repoVisibility = 'private', - } = ctx.parameters; + const { repoUrl, description, repoVisibility = 'private' } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl); @@ -238,5 +247,5 @@ export function createPublishBitbucketAction(options: { ctx.output('remoteUrl', remoteUrl); ctx.output('repoContentsUrl', repoContentsUrl); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 251f7ed08f..7f15197c09 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -20,18 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Octokit } from '@octokit/rest'; -import { TemplateAction } from '../../types'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; export function createPublishGithubAction(options: { integrations: ScmIntegrations; -}): TemplateAction<{ - repoUrl: string; - description?: string; - access?: string; - repoVisibility: 'private' | 'internal' | 'public'; -}> { +}) { const { integrations } = options; const credentialsProviders = new Map( @@ -41,7 +36,12 @@ export function createPublishGithubAction(options: { }), ); - return { + return createTemplateAction<{ + repoUrl: string; + description?: string; + access?: string; + repoVisibility: 'private' | 'internal' | 'public'; + }>({ id: 'publish:github', schema: { input: { @@ -82,7 +82,7 @@ export function createPublishGithubAction(options: { }, }, async handler(ctx) { - const { repoUrl, description, access, repoVisibility } = ctx.parameters; + const { repoUrl, description, access, repoVisibility } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl); @@ -91,7 +91,7 @@ export function createPublishGithubAction(options: { if (!credentialsProvider || !integrationConfig) { throw new InputError( - `No matching integration configuration for host ${host}, please check your Integrations config`, + `No matching integration configuration for host ${host}, please check your integrations config`, ); } @@ -165,5 +165,5 @@ export function createPublishGithubAction(options: { ctx.output('remoteUrl', remoteUrl); ctx.output('repoContentsUrl', repoContentsUrl); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index be8d8fb0be..acdce2a440 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -16,20 +16,20 @@ import { InputError } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateAction } from '../../types'; import { Gitlab } from '@gitbeaker/node'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; export function createPublishGitlabAction(options: { integrations: ScmIntegrations; -}): TemplateAction<{ - repoUrl: string; - repoVisibility: 'private' | 'internal' | 'public'; -}> { +}) { const { integrations } = options; - return { + return createTemplateAction<{ + repoUrl: string; + repoVisibility: 'private' | 'internal' | 'public'; + }>({ id: 'publish:gitlab', schema: { input: { @@ -62,7 +62,7 @@ export function createPublishGitlabAction(options: { }, }, async handler(ctx) { - const { repoUrl, repoVisibility = 'private' } = ctx.parameters; + const { repoUrl, repoVisibility = 'private' } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl); @@ -118,5 +118,5 @@ export function createPublishGitlabAction(options: { ctx.output('remoteUrl', remoteUrl); ctx.output('repoContentsUrl', repoContentsUrl); }, - }; + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts new file mode 100644 index 0000000000..f501c3f7ac --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// function createTemplateAction( +// options: TemplateAction, +// ): TemplateAction; + +import { InputBase, TemplateAction } from './types'; + +export const createTemplateAction = ( + templateAction: TemplateAction, +): TemplateAction => { + // TODO(blam): Can add some more validation here to validate the action later on + return templateAction; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts index 0f7931e74b..98ae406792 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts @@ -16,4 +16,5 @@ export * from './builtin'; export { TemplateActionRegistry } from './TemplateActionRegistry'; +export { createTemplateAction } from './createTemplateAction'; export type { ActionContext, TemplateAction } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts index 11523f98e6..20b7696d70 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts @@ -21,9 +21,9 @@ import { Schema } from 'jsonschema'; type PartialJsonObject = Partial; type PartialJsonValue = PartialJsonObject | JsonValue | undefined; -export type ParameterBase = Partial<{ [name: string]: PartialJsonValue }>; +export type InputBase = Partial<{ [name: string]: PartialJsonValue }>; -export type ActionContext = { +export type ActionContext = { /** * Base URL for the location of the task spec, typically the url of the source entity file. */ @@ -33,7 +33,7 @@ export type ActionContext = { logStream: Writable; workspacePath: string; - parameters: Parameters; + input: Input; output(name: string, value: JsonValue): void; /** @@ -42,11 +42,11 @@ export type ActionContext = { createTemporaryDirectory(): Promise; }; -export type TemplateAction = { +export type TemplateAction = { id: string; schema?: { input?: Schema; output?: Schema; }; - handler: (ctx: ActionContext) => Promise; + handler: (ctx: ActionContext) => Promise; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index 5f2d6a47a8..deebb9b1bc 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -37,7 +37,7 @@ export function registerLegacyActions( id: 'legacy:prepare', async handler(ctx) { ctx.logger.info('Preparing the skeleton'); - const { protocol, url } = ctx.parameters; + const { protocol, url } = ctx.input; const preparer = protocol === 'file' ? new FilePreparer() : preparers.get(url as string); @@ -53,12 +53,12 @@ export function registerLegacyActions( id: 'legacy:template', async handler(ctx) { ctx.logger.info('Running the templater'); - const templater = templaters.get(ctx.parameters.templater as string); + const templater = templaters.get(ctx.input.templater as string); await templater.run({ workspacePath: ctx.workspacePath, dockerClient, logStream: ctx.logStream, - values: ctx.parameters.values as TemplaterValues, + values: ctx.input.values as TemplaterValues, }); }, }); @@ -66,7 +66,7 @@ export function registerLegacyActions( registry.register({ id: 'legacy:publish', async handler(ctx) { - const { values } = ctx.parameters; + const { values } = ctx.input; if ( typeof values !== 'object' || values === null || diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 08d641f9ea..4d8f652ae0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -98,7 +98,7 @@ export class TaskWorker { throw new Error(`Action '${step.action}' does not exist`); } - const parameters = JSON.parse( + const input = JSON.parse( JSON.stringify(step.input), (_key, value) => { if (typeof value === 'string') { @@ -114,15 +114,13 @@ export class TaskWorker { ); if (action.schema?.input) { - const validateResult = validateJsonSchema( - parameters, - action.schema, - { propertyName: 'parameters' }, - ); + const validateResult = validateJsonSchema(input, action.schema, { + propertyName: 'input', + }); if (!validateResult.valid) { const errors = validateResult.errors.join(', '); throw new InputError( - `Invalid parameters passed to action ${action.id}, ${errors}`, + `Invalid input passed to action ${action.id}, ${errors}`, ); } } @@ -136,7 +134,7 @@ export class TaskWorker { baseUrl: task.spec.baseUrl, logger: taskLogger, logStream: stream, - parameters, + input, workspacePath, async createTemporaryDirectory() { const tmpDir = await fs.mkdtemp( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index dd9a4e25da..03f95b2aae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -54,7 +54,7 @@ export function templateEntityToSpec( id: 'prepare', name: 'Prepare', action: 'legacy:prepare', - parameters: { + input: { protocol, url, }, @@ -64,7 +64,7 @@ export function templateEntityToSpec( id: 'template', name: 'Template', action: 'legacy:template', - parameters: { + input: { templater, values, }, @@ -74,7 +74,7 @@ export function templateEntityToSpec( id: 'publish', name: 'Publish', action: 'legacy:publish', - parameters: { + input: { values, }, }); @@ -83,7 +83,7 @@ export function templateEntityToSpec( id: 'register', name: 'Register', action: 'catalog:register', - parameters: { + input: { catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', }, }); diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index b30ef135ba..b7c2582c81 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -118,7 +118,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), - urlReader: mockUrlReader, + reader: mockUrlReader, }), ).rejects.toThrow('access error'); }); @@ -133,7 +133,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), - urlReader: mockUrlReader, + reader: mockUrlReader, }); const app = express().use(router); @@ -163,7 +163,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), - urlReader: mockUrlReader, + reader: mockUrlReader, }); const app = express().use(router); @@ -237,7 +237,7 @@ describe('createRouter', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), - urlReader: mockUrlReader, + reader: mockUrlReader, }); app = express().use(router); }); From 3efba84528a3c28bdac22da03db53358cfd6549e Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 15:38:02 +0100 Subject: [PATCH 28/66] Add support for passing actions into router Co-authored-by: blam Signed-off-by: Johan Haals --- .../actions/builtin/createBuiltinActions.ts | 71 +++++++++ .../src/scaffolder/actions/builtin/index.ts | 1 + .../src/scaffolder/index.ts | 1 + .../src/scaffolder/stages/index.ts | 2 + .../src/scaffolder/stages/legacy.ts | 149 +++++++++--------- .../scaffolder-backend/src/service/router.ts | 78 +++------ 6 files changed, 173 insertions(+), 129 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts new file mode 100644 index 0000000000..0829a101f2 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { ScmIntegrations } from '@backstage/integration'; +import { createCatalogRegisterAction } from './catalog'; +import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch'; +import { + createPublishAzureAction, + createPublishBitbucketAction, + createPublishGithubAction, + createPublishGitlabAction, +} from './publish'; +import Docker from 'dockerode'; +import { TemplaterBuilder } from '../../stages'; + +export const createBuiltinActions = (options: { + reader: UrlReader; + integrations: ScmIntegrations; + dockerClient: Docker; + catalogClient: CatalogApi; + templaters: TemplaterBuilder; +}) => { + const { + reader, + integrations, + dockerClient, + templaters, + catalogClient, + } = options; + + return [ + createFetchPlainAction({ + reader, + integrations, + }), + createFetchCookiecutterAction({ + reader, + integrations, + dockerClient, + templaters, + }), + createPublishGithubAction({ + integrations, + }), + createPublishGitlabAction({ + integrations, + }), + createPublishBitbucketAction({ + integrations, + }), + createPublishAzureAction({ + integrations, + }), + createCatalogRegisterAction({ catalogClient, integrations }), + ]; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts index c85c6e33ce..e4281c172b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts @@ -17,3 +17,4 @@ export * from './catalog'; export * from './fetch'; export * from './publish'; +export { createBuiltinActions } from './createBuiltinActions'; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 470bc230d6..9de10ca1eb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -15,3 +15,4 @@ */ export * from './stages'; export * from './jobs'; +export * from './actions'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts index dfe49a8aa5..ef23af1b3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts @@ -17,3 +17,5 @@ export * from './prepare'; export * from './publish'; export * from './templater'; export * from './helpers'; + +export { createLegacyActions } from './legacy'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index deebb9b1bc..c7c52fa320 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,11 +14,11 @@ * limitations under the License. */ -import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; import { PublisherBuilder } from './publish'; +import { createTemplateAction } from '../actions'; type Options = { dockerClient: Docker; @@ -27,81 +27,82 @@ type Options = { publishers: PublisherBuilder; }; -export function registerLegacyActions( - registry: TemplateActionRegistry, - options: Options, -) { +export function createLegacyActions(options: Options) { const { dockerClient, preparers, templaters, publishers } = options; - registry.register({ - id: 'legacy:prepare', - async handler(ctx) { - ctx.logger.info('Preparing the skeleton'); - const { protocol, url } = ctx.input; - const preparer = - protocol === 'file' ? new FilePreparer() : preparers.get(url as string); + return [ + createTemplateAction({ + id: 'legacy:prepare', + async handler(ctx) { + ctx.logger.info('Preparing the skeleton'); + const { protocol, url } = ctx.input; + const preparer = + protocol === 'file' + ? new FilePreparer() + : preparers.get(url as string); - await preparer.prepare({ - url: url as string, - logger: ctx.logger, - workspacePath: ctx.workspacePath, - }); - }, - }); + await preparer.prepare({ + url: url as string, + logger: ctx.logger, + workspacePath: ctx.workspacePath, + }); + }, + }), + createTemplateAction({ + id: 'legacy:template', + async handler(ctx) { + ctx.logger.info('Running the templater'); + const templater = templaters.get(ctx.input.templater as string); + await templater.run({ + workspacePath: ctx.workspacePath, + dockerClient, + logStream: ctx.logStream, + values: ctx.input.values as TemplaterValues, + }); + }, + }), + createTemplateAction({ + id: 'legacy:publish', + async handler(ctx) { + const { values } = ctx.input; + if ( + typeof values !== 'object' || + values === null || + Array.isArray(values) + ) { + throw new Error( + `Invalid values passed to publish, got ${typeof values}`, + ); + } + const storePath = values.storePath as unknown; + if (typeof storePath !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof storePath}`, + ); + } + const owner = values.owner as unknown; + if (typeof owner !== 'string') { + throw new Error( + `Invalid owner passed to publish, got ${typeof owner}`, + ); + } - registry.register({ - id: 'legacy:template', - async handler(ctx) { - ctx.logger.info('Running the templater'); - const templater = templaters.get(ctx.input.templater as string); - await templater.run({ - workspacePath: ctx.workspacePath, - dockerClient, - logStream: ctx.logStream, - values: ctx.input.values as TemplaterValues, - }); - }, - }); - - registry.register({ - id: 'legacy:publish', - async handler(ctx) { - const { values } = ctx.input; - if ( - typeof values !== 'object' || - values === null || - Array.isArray(values) - ) { - throw new Error( - `Invalid values passed to publish, got ${typeof values}`, - ); - } - const storePath = values.storePath as unknown; - if (typeof storePath !== 'string') { - throw new Error( - `Invalid store path passed to publish, got ${typeof storePath}`, - ); - } - const owner = values.owner as unknown; - if (typeof owner !== 'string') { - throw new Error(`Invalid owner passed to publish, got ${typeof owner}`); - } - - const publisher = publishers.get(storePath); - ctx.logger.info('Will now store the template'); - const { remoteUrl, catalogInfoUrl } = await publisher.publish({ - values: { - ...values, - owner, - storePath, - }, - workspacePath: ctx.workspacePath, - logger: ctx.logger, - }); - ctx.output('remoteUrl', remoteUrl); - if (catalogInfoUrl) { - ctx.output('catalogInfoUrl', catalogInfoUrl); - } - }, - }); + const publisher = publishers.get(storePath); + ctx.logger.info('Will now store the template'); + const { remoteUrl, catalogInfoUrl } = await publisher.publish({ + values: { + ...values, + owner, + storePath, + }, + workspacePath: ctx.workspacePath, + logger: ctx.logger, + }); + ctx.output('remoteUrl', remoteUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }), + ]; } diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index e9bfd0e74a..1ed0c1af7a 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -40,7 +40,7 @@ import { } from '../scaffolder/tasks'; import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; -import { registerLegacyActions } from '../scaffolder/stages/legacy'; +import { createLegacyActions } from '../scaffolder/stages/legacy'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { InputError, @@ -54,16 +54,9 @@ import { TemplateEntityV1beta2, Entity, } from '@backstage/catalog-model'; -import { - createFetchPlainAction, - createFetchCookiecutterAction, - createPublishGithubAction, - createPublishBitbucketAction, - createPublishAzureAction, - createPublishGitlabAction, - createCatalogRegisterAction, -} from '../scaffolder/actions/builtin'; import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../scaffolder/actions'; +import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; export interface RouterOptions { preparers: PreparerBuilder; @@ -76,6 +69,7 @@ export interface RouterOptions { dockerClient: Docker; database: PluginDatabaseManager; catalogClient: CatalogApi; + actions?: TemplateAction[]; } function isAlpha1Template( @@ -109,6 +103,7 @@ export async function createRouter( dockerClient, database, catalogClient, + actions, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); @@ -129,52 +124,25 @@ export async function createRouter( workingDirectory, }); - registerLegacyActions(actionRegistry, { - dockerClient, - preparers, - publishers, - templaters, - }); - actionRegistry.register( - createFetchPlainAction({ - reader, - integrations, - }), - ); - actionRegistry.register( - createFetchCookiecutterAction({ - reader, - integrations, - dockerClient, - templaters, - }), - ); - actionRegistry.register( - createPublishGithubAction({ - integrations, - }), - ); - actionRegistry.register( - createPublishGitlabAction({ - integrations, - }), - ); + const actionsToRegister = Array.isArray(actions) + ? actions + : [ + ...createLegacyActions({ + dockerClient, + preparers, + publishers, + templaters, + }), + ...createBuiltinActions({ + dockerClient, + integrations, + catalogClient, + templaters, + reader, + }), + ]; - actionRegistry.register( - createPublishBitbucketAction({ - integrations, - }), - ); - - actionRegistry.register( - createPublishAzureAction({ - integrations, - }), - ); - - actionRegistry.register( - createCatalogRegisterAction({ catalogClient, integrations }), - ); + actionsToRegister.forEach(action => actionRegistry.register(action)); worker.start(); From e8867ba6aa37fe5fd34ebaaba80885d244e38e2b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 15:39:34 +0100 Subject: [PATCH 29/66] chore: fix input errors Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/publish/azure.ts | 2 +- .../src/scaffolder/actions/builtin/publish/gitlab.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index c79048b289..04598f31d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -76,7 +76,7 @@ export function createPublishAzureAction(options: { if (!integrationConfig) { throw new InputError( - `No matching integration configuration for host ${host}, please check your Integrations config`, + `No matching integration configuration for host ${host}, please check your integrations config`, ); } if (!integrationConfig.config.token) { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index acdce2a440..2b89136dda 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -70,7 +70,7 @@ export function createPublishGitlabAction(options: { if (!integrationConfig) { throw new InputError( - `No matching integration configuration for host ${host}, please check your Integrations config`, + `No matching integration configuration for host ${host}, please check your integrations config`, ); } From b92e43c3bb49e610e8b976662707215f15244e35 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 15:40:31 +0100 Subject: [PATCH 30/66] chore: use path.resolve instead of resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelรถw Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/fetch/plain.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts index 52969fcf6b..1eb1a6dc82 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { resolve as resolvePath } from 'path'; +import path from 'path'; import { InputError, UrlReader } from '@backstage/backend-common'; import { ScmIntegrations } from '@backstage/integration'; import { fetchContents } from './helpers'; @@ -53,7 +53,7 @@ export function createFetchPlainAction(options: { // Finally move the template result into the task workspace const targetPath = ctx.input.targetPath ?? './'; - const outputPath = resolvePath(ctx.workspacePath, targetPath); + const outputPath = path.resolve(ctx.workspacePath, targetPath); if (!outputPath.startsWith(ctx.workspacePath)) { throw new InputError( `Fetch action targetPath may not specify a path outside the working directory`, From db471b6182163119f2d139f31d0960a06303df5b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Thu, 25 Feb 2021 16:10:39 +0100 Subject: [PATCH 31/66] chore: lowercase 'integration' Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/publish/bitbucket.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 533595eb3c..1cc77cbf81 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -210,7 +210,7 @@ export function createPublishBitbucketAction(options: { if (!integrationConfig) { throw new InputError( - `No matching integration configuration for host ${host}, please check your Integrations config`, + `No matching integration configuration for host ${host}, please check your integrations config`, ); } From ebab16aa303f379fbc429b835c2b40004c49133a Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 25 Feb 2021 16:26:30 +0100 Subject: [PATCH 32/66] chore: added some tests for publish:azure action Signed-off-by: Johan Haals --- .../actions/builtin/publish/azure.test.ts | 186 ++++++++++++++++++ .../actions/builtin/publish/azure.ts | 2 +- 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts new file mode 100644 index 0000000000..fe292abfe9 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts @@ -0,0 +1,186 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('azure-devops-node-api', () => ({ + WebApi: jest.fn(), + getPersonalAccessTokenHandler: jest.fn().mockReturnValue(() => {}), +})); + +import { createPublishAzureAction } from './azure'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { WebApi } from 'azure-devops-node-api'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('Azure Publish Action', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + const action = createPublishAzureAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'dev.azure.com?repo=repo&owner=owner&organization=org', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const mockGitClient = { + createRepository: jest.fn(), + }; + const mockGitApi = { + getGitApi: jest.fn().mockReturnValue(mockGitClient), + }; + + ((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi); + + beforeEach(() => { + jest.restoreAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?owner=owner&repo=repo' }, + }), + ).rejects.toThrow(/missing organization/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?repo=bob&owner=owner&organization=org' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: + 'myazurehostnotoken.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/No token provided for Azure Integration/); + }); + + it('should throw when no repo is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'dev.azure.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/Unable to create the repository/); + }); + + it('should throw if there is no remoteUrl returned', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: null, + })); + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'dev.azure.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/No remote URL returned/); + }); + + it('should call the azureApis with the correct values', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'http://google.com', + })); + + await action.handler(mockContext); + + expect(WebApi).toHaveBeenCalledWith( + 'https://dev.azure.com/org', + expect.any(Function), + ); + + expect(mockGitClient.createRepository).toHaveBeenCalledWith( + { + name: 'bob', + }, + 'owner', + ); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + })); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + auth: { username: 'notempty', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call output with the remoteUrl and the repoContentsUrl', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + })); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://dev.azure.com/organization/project/_git/repo', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://dev.azure.com/organization/project/_git/repo', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 04598f31d4..e459555adb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -68,7 +68,7 @@ export function createPublishAzureAction(options: { if (!organization) { throw new InputError( - `No Organization was included in the repo URL to create ${ctx.input.repoUrl}`, + `Invalid URL provider was included in the repo URL to create ${ctx.input.repoUrl}, missing organization`, ); } From c6304015727b3fc53c71fd947b9f3521d2bef7dc Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 25 Feb 2021 16:58:32 +0100 Subject: [PATCH 33/66] tests: added some more tests for the github publish action Signed-off-by: Johan Haals --- .../actions/builtin/publish/github.test.ts | 204 ++++++++++++++++++ .../actions/builtin/publish/github.ts | 9 +- 2 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts new file mode 100644 index 0000000000..9c8025423b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -0,0 +1,204 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('@octokit/rest'); + +import { createPublishGithubAction } from './github'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('Github Publish Action', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'tokenlols' }, + { host: 'ghe.github.com' }, + ], + }, + }), + ); + const action = createPublishGithubAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'github.com?repo=repo&owner=owner', + description: 'description', + repoVisibility: 'private', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const { mockGithubClient } = require('@octokit/rest'); + + beforeEach(() => { + jest.restoreAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'github.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'github.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'ghe.github.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/No token available for host/); + }); + + it('should call the githubApis with the correct values for createInOrg', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'Organization' }, + }); + + mockGithubClient.repos.createInOrg.mockResolvedValue({ data: {} }); + + await action.handler(mockContext); + expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + org: 'owner', + private: true, + visibility: 'private', + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoVisibility: 'public', + }, + }); + expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + org: 'owner', + private: false, + visibility: 'public', + }); + }); + + it('should call the githubApis with the correct values for createForAuthenticatedUser', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: {}, + }); + + await action.handler(mockContext); + expect( + mockGithubClient.repos.createForAuthenticatedUser, + ).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + private: true, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoVisibility: 'public', + }, + }); + expect( + mockGithubClient.repos.createForAuthenticatedUser, + ).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + private: false, + }); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://github.com/clone/url.git', + auth: { username: 'x-access-token', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call output with the remoteUrl and the repoContentsUrl', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://github.com/clone/url.git', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://github.com/html/url/blob/master', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 7f15197c09..d449ee135c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -82,7 +82,12 @@ export function createPublishGithubAction(options: { }, }, async handler(ctx) { - const { repoUrl, description, access, repoVisibility } = ctx.input; + const { + repoUrl, + description, + access, + repoVisibility = 'private', + } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl); @@ -150,7 +155,7 @@ export function createPublishGithubAction(options: { } const remoteUrl = data.clone_url; - const repoContentsUrl = `${data?.html_url}/blob/master`; + const repoContentsUrl = `${data.html_url}/blob/master`; await initRepoAndPush({ dir: ctx.workspacePath, From e787b088b846fce3a5f175a21bb4df2affbee222 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 26 Feb 2021 09:22:02 +0100 Subject: [PATCH 34/66] tests: added test suite for gitlab publisher action Signed-off-by: Johan Haals --- .../actions/builtin/publish/gitab.test.ts | 165 ++++++++++++++++++ .../actions/builtin/publish/github.test.ts | 56 +++++- .../actions/builtin/publish/gitlab.ts | 4 +- 3 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts new file mode 100644 index 0000000000..8757f7865a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts @@ -0,0 +1,165 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('@gitbeaker/node'); + +import { createPublishGitlabAction } from './gitlab'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('Gitlab Publish Action', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + token: 'tokenlols', + apiBaseUrl: 'https://api.gitlab.com', + }, + { + host: 'hosted.gitlab.com', + apiBaseUrl: 'https://api.hosted.gitlab.com', + }, + ], + }, + }), + ); + const action = createPublishGitlabAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'gitlab.com?repo=repo&owner=owner', + repoVisibility: 'private', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const { mockGitlabClient } = require('@gitbeaker/node'); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'gitlab.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'gitlab.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'hosted.gitlab.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/No token available for host/); + }); + + it('should call the correct Gitlab APIs when the owner is an organization', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ + namespace_id: 1234, + name: 'repo', + visibility: 'private', + }); + }); + + it('should call the correct Gitlab APIs when the owner is not an organization', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: null }); + mockGitlabClient.Users.current.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ + namespace_id: 12345, + name: 'repo', + visibility: 'private', + }); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'http://mockurl.git', + auth: { username: 'oauth2', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call output with the remoteUrl and repoContentsUrl', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'http://mockurl', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'http://mockurl/-/blob/master', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 9c8025423b..fe6e78b445 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -40,6 +40,7 @@ describe('Github Publish Action', () => { repoUrl: 'github.com?repo=repo&owner=owner', description: 'description', repoVisibility: 'private', + access: 'owner/blam', }, workspacePath: 'lol', logger: getVoidLogger(), @@ -51,7 +52,7 @@ describe('Github Publish Action', () => { const { mockGithubClient } = require('@octokit/rest'); beforeEach(() => { - jest.restoreAllMocks(); + jest.resetAllMocks(); }); it('should throw an error when the repoUrl is not well formed', async () => { @@ -178,6 +179,59 @@ describe('Github Publish Action', () => { }); }); + it('should add access for the team when it starts with the owner', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect( + mockGithubClient.teams.addOrUpdateRepoPermissionsInOrg, + ).toHaveBeenCalledWith({ + org: 'owner', + team_slug: 'blam', + owner: 'owner', + repo: 'repo', + permission: 'admin', + }); + }); + + it('should add outside collaborators when provided', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + access: 'outsidecollaborator', + }, + }); + + expect(mockGithubClient.repos.addCollaborator).toHaveBeenCalledWith({ + username: 'outsidecollaborator', + owner: 'owner', + repo: 'repo', + permission: 'admin', + }); + }); + it('should call output with the remoteUrl and the repoContentsUrl', async () => { mockGithubClient.users.getByUsername.mockResolvedValue({ data: { type: 'User' }, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index 2b89136dda..603ca84f05 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -75,9 +75,7 @@ export function createPublishGitlabAction(options: { } if (!integrationConfig.config.token) { - throw new InputError( - `No token provided for GitLab Integration ${host}`, - ); + throw new InputError(`No token available for host ${host}`); } const client = new Gitlab({ From ed4941accbd2a968b4346d339919493ac29f378f Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 26 Feb 2021 10:54:05 +0100 Subject: [PATCH 35/66] test: added the tests for bitbucket publisher Signed-off-by: Johan Haals --- .../actions/builtin/publish/bitbucket.test.ts | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts new file mode 100644 index 0000000000..04e3b9b0bb --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); + +import { createPublishBitbucketAction } from './bitbucket'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('Bitbucket Publish Action', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + token: 'tokenlols', + }, + { + host: 'hosted.bitbucket.com', + token: 'thing', + }, + { + host: 'notoken.bitbucket.com', + }, + ], + }, + }), + ); + const action = createPublishBitbucketAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'bitbucket.org?repo=repo&owner=owner', + repoVisibility: 'private', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + const server = setupServer(); + msw.setupDefaultHandlers(server); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'bitbucket.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'bitbucket.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'notoken.bitbucket.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/Authorization has not been provided for Bitbucket/); + }); + + it('should call the correct APIs when the host is bitbucket cloud', async () => { + expect.assertions(2); + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Bearer tokenlols'); + expect(req.body).toEqual({ is_private: true, scm: 'git' }); + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/repo', + }, + ], + }, + }), + ); + }, + ), + ); + + await action.handler(mockContext); + }); + + it('should call the correct APIs when the host is hosted bitbucket', async () => { + expect.assertions(2); + server.use( + rest.post( + 'https://hosted.bitbucket.com/rest/api/1.0/projects/owner/repos', + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Bearer thing'); + expect(req.body).toEqual({ is_private: true, name: 'repo' }); + return res( + ctx.status(201), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/project/repos/repo', + }, + ], + clone: [ + { + name: 'http', + href: 'https://bitbucket.mycompany.com/scm/project/repo', + }, + ], + }, + }), + ); + }, + ), + ); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoUrl: 'hosted.bitbucket.com?owner=owner&repo=repo', + }, + }); + }); + + it('should call initAndPush with the correct values', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/cloneurl', + }, + ], + }, + }), + ), + ), + ); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://bitbucket.org/owner/cloneurl', + auth: { username: 'x-token-auth', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call outputs with the correct urls', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/cloneurl', + }, + ], + }, + }), + ), + ), + ); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://bitbucket.org/owner/cloneurl', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://bitbucket.org/owner/repo/src/master', + ); + }); +}); From edcbcff1060a2ae09584c5176796de378d08014c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Fri, 26 Feb 2021 13:25:10 +0100 Subject: [PATCH 36/66] make the template card grid even and nice Signed-off-by: Johan Haals --- .../ScaffolderPage/ScaffolderPage.tsx | 28 ++++++++----------- .../components/TemplateCard/TemplateCard.tsx | 15 ++++++++-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 3f6ceb64c7..ebce35a91f 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -30,7 +30,7 @@ import { WarningPanel, } from '@backstage/core'; import { useStarredEntities } from '@backstage/plugin-catalog-react'; -import { Button, Grid, Link, makeStyles, Typography } from '@material-ui/core'; +import { Box, Button, Link, makeStyles, Typography } from '@material-ui/core'; import StarIcon from '@material-ui/icons/Star'; import { EntityFilterGroupsProvider, useFilteredEntities } from '../../filter'; import { TemplateCard, TemplateCardProps } from '../TemplateCard'; @@ -46,6 +46,12 @@ const useStyles = makeStyles(theme => ({ gridTemplateColumns: '250px 1fr', gridColumnGap: theme.spacing(2), }, + templateGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fill, minmax(22em, 1fr))', + gridAutoRows: '1fr', + gridGap: theme.spacing(2), + }, })); const getTemplateCardProps = ( @@ -177,23 +183,13 @@ export const ScaffolderPageContents = () => { )} - + {matchingEntities && matchingEntities?.length > 0 && - matchingEntities.map(template => { - return ( - - - - ); - })} - + matchingEntities.map(template => ( + + ))} + diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index c38f4e09a5..abff0538ab 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -37,9 +37,18 @@ const useStyles = makeStyles(theme => ({ backgroundPosition: 0, }, description: { - height: 175, overflow: 'hidden', textOverflow: 'ellipsis', + display: '-webkit-box', + '-webkit-line-clamp': 10, + '-webkit-box-orient': 'vertical', + }, + card: { + display: 'flex', + flexDirection: 'column', + }, + cardContent: { + flexGrow: 1, }, })); @@ -69,12 +78,12 @@ export const TemplateCard = ({ }); return ( - +
{type} {title}
- + {tags?.map(tag => ( ))} From ab6d9de923f0de9c4392cd4173ccfd4e349433b2 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 13:29:36 +0100 Subject: [PATCH 37/66] Add action tests Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../actions/builtin/catalog/register.test.ts | 101 ++++++++++++++ .../builtin/fetch/cookiecutter.test.ts | 132 ++++++++++++++++++ .../actions/builtin/fetch/cookiecutter.ts | 3 - .../actions/builtin/fetch/plain.test.ts | 85 +++++++++++ .../actions/builtin/publish/azure.test.ts | 2 +- .../actions/builtin/publish/bitbucket.test.ts | 2 +- .../actions/builtin/publish/gitab.test.ts | 2 +- .../actions/builtin/publish/github.test.ts | 2 +- 8 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts new file mode 100644 index 0000000000..9fa96132d6 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PassThrough } from 'stream'; +import os from 'os'; +import { getVoidLogger } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { createCatalogRegisterAction } from './register'; +import { Entity } from '@backstage/catalog-model'; + +describe('catalog:register', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const addLocation = jest.fn(); + const catalogClient = { + addLocation: addLocation, + }; + + const action = createCatalogRegisterAction({ + integrations, + catalogClient: (catalogClient as unknown) as CatalogApi, + }); + + const mockContext = { + workspacePath: os.tmpdir(), + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should reject registrations for locations that does not match any integration', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoContentsUrl: 'https://google.com/foo/bar', + }, + }), + ).rejects.toThrow( + /No integration found for host https:\/\/google.com\/foo\/bar/, + ); + }); + + it('should register location in catalog', async () => { + addLocation.mockResolvedValue({ + entities: [ + { + metadata: { + namespace: 'default', + name: 'test', + }, + kind: 'service', + } as Entity, + ], + }); + await action.handler({ + ...mockContext, + input: { + catalogInfoUrl: 'http://foo/var', + }, + }); + expect(addLocation).toBeCalledWith({ + type: 'url', + target: 'http://foo/var', + }); + + expect(mockContext.output).toBeCalledWith( + 'entityRef', + 'service:default/test', + ); + expect(mockContext.output).toBeCalledWith( + 'catalogInfoUrl', + 'http://foo/var', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts new file mode 100644 index 0000000000..923e22d0a5 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('./helpers'); + +import os from 'os'; +import { createFetchCookiecutterAction } from './cookiecutter'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { Templaters } from '../../../stages/templater'; +import { PassThrough } from 'stream'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { fetchContents } from './helpers'; +import mock from 'mock-fs'; + +describe('fetch:cookiecutter', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + + const templaters = new Templaters(); + const cookiecutterTemplater = { run: jest.fn() }; + const mockDockerClient = {}; + const mockTmpDir = os.tmpdir(); + const mockContext = { + input: { + url: 'https://google.com/cookie/cutter', + targetPath: 'something', + values: { + help: 'me', + }, + }, + baseUrl: 'somebase', + workspacePath: mockTmpDir, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), + }; + + mock({ [`${mockContext.workspacePath}/result`]: {} }); + + const mockReader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + const action = createFetchCookiecutterAction({ + integrations, + templaters, + dockerClient: mockDockerClient as any, + reader: mockReader, + }); + + templaters.register('cookiecutter', cookiecutterTemplater); + + beforeEach(() => { + jest.restoreAllMocks(); + }); + + it('should call fetchContents with the correct values', async () => { + await action.handler(mockContext); + + expect(fetchContents).toHaveBeenCalledWith({ + reader: mockReader, + integrations, + baseUrl: mockContext.baseUrl, + fetchUrl: mockContext.input.url, + outputPath: `${mockContext.workspacePath}/template/{{cookiecutter and 'contents'}}`, + }); + }); + + it('should execute the cookiecutter templater with the correct values', async () => { + await action.handler(mockContext); + + expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ + workspacePath: mockTmpDir, + dockerClient: mockDockerClient, + logStream: mockContext.logStream, + values: mockContext.input.values, + }); + }); + + it('should throw if there is no cookiecutter templater initialized', async () => { + const templatersWithoutCookiecutter = new Templaters(); + + const newAction = createFetchCookiecutterAction({ + integrations, + templaters: templatersWithoutCookiecutter, + dockerClient: mockDockerClient as any, + reader: mockReader, + }); + + await expect(newAction.handler(mockContext)).rejects.toThrow( + /No templater registered/, + ); + }); + + it('should throw if the target directory is outside of the workspace path', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + ...mockContext.input, + targetPath: '/foo', + }, + }), + ).rejects.toThrow( + /targetPath may not specify a path outside the working directory/, + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 4016fb69c5..2a04ed301d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -82,9 +82,6 @@ export function createFetchCookiecutterAction(options: { }); const cookiecutter = templaters.get('cookiecutter'); - if (!cookiecutter) { - throw new Error('No cookiecutter templater available'); - } // Will execute the template in ./template and put the result in ./result await cookiecutter.run({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts new file mode 100644 index 0000000000..88eeb5997b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('./helpers'); + +import os from 'os'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { createFetchPlainAction } from './plain'; +import { PassThrough } from 'stream'; +import { fetchContents } from './helpers'; + +describe('fetch:plain', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + const reader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + const action = createFetchPlainAction({ integrations, reader }); + const mockContext = { + workspacePath: os.tmpdir(), + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + it('should disallow a target path outside working directory', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + url: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + targetPath: '/foobar', + }, + }), + ).rejects.toThrow( + /Fetch action targetPath may not specify a path outside the working directory/, + ); + }); + + it('should fetch plain', async () => { + await action.handler({ + ...mockContext, + input: { + url: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + targetPath: 'lol', + }, + }); + expect(fetchContents).toBeCalledWith( + expect.objectContaining({ + outputPath: `${mockContext.workspacePath}/lol`, + fetchUrl: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + }), + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts index fe292abfe9..4e2d721af9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts @@ -27,7 +27,7 @@ import { WebApi } from 'azure-devops-node-api'; import { PassThrough } from 'stream'; import { initRepoAndPush } from '../../../stages/publish/helpers'; -describe('Azure Publish Action', () => { +describe('publish:azure', () => { const integrations = ScmIntegrations.fromConfig( new ConfigReader({ integrations: { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts index 04e3b9b0bb..40e7f613f2 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts @@ -25,7 +25,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; import { initRepoAndPush } from '../../../stages/publish/helpers'; -describe('Bitbucket Publish Action', () => { +describe('publish:bitbucket', () => { const integrations = ScmIntegrations.fromConfig( new ConfigReader({ integrations: { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts index 8757f7865a..6f789a7b76 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts @@ -23,7 +23,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; import { initRepoAndPush } from '../../../stages/publish/helpers'; -describe('Gitlab Publish Action', () => { +describe('publish:gitlab', () => { const integrations = ScmIntegrations.fromConfig( new ConfigReader({ integrations: { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index fe6e78b445..9880dda621 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -23,7 +23,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; import { initRepoAndPush } from '../../../stages/publish/helpers'; -describe('Github Publish Action', () => { +describe('publish:github', () => { const integrations = ScmIntegrations.fromConfig( new ConfigReader({ integrations: { From e39a1d5d37cd75968e290655bef72dbab77a2127 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 14:06:38 +0100 Subject: [PATCH 38/66] Add fetch helper tests Signed-off-by: Johan Haals --- .../actions/builtin/fetch/helpers.test.ts | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts new file mode 100644 index 0000000000..e475764a0b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('fs-extra'); +import fs from 'fs-extra'; + +import { UrlReader } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchContents } from './helpers'; +import os from 'os'; + +describe('fetchContent helper', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const readTree = jest.fn(); + const reader: UrlReader = { + read: jest.fn(), + readTree, + search: jest.fn(), + }; + + const options = { + reader, + integrations, + outputPath: os.tmpdir(), + }; + + it('should reject non string fetchUrls', async () => { + await expect( + fetchContents({ + ...options, + fetchUrl: false, + }), + ).rejects.toThrow('Invalid url parameter, expected string, got boolean'); + }); + + it('should reject absolute file locations', async () => { + await expect( + fetchContents({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: '/etc/passwd', + }), + ).rejects.toThrow( + 'Fetch URL may not be absolute for file locations, /etc/passwd', + ); + }); + + it('should copy file to outputpath', async () => { + await fetchContents({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: 'foo', + outputPath: 'somepath', + }); + expect(fs.copy).toBeCalledWith('/some/foo', 'somepath'); + }); + + it('should reject if no integration matches location', async () => { + await expect( + fetchContents({ + ...options, + baseUrl: 'http://example.com/some/folder', + }), + ).rejects.toThrow( + 'No integration found for location http://example.com/some/folder', + ); + }); + + it('should reject if fetch url is relative and no base url is specified', async () => { + await expect( + fetchContents({ + ...options, + fetchUrl: 'foo', + }), + ).rejects.toThrow( + 'Failed to fetch, template location could not be determined and the fetch URL is relative, foo', + ); + }); + + it('should fetch url contents', async () => { + const dirFunction = jest.fn(); + readTree.mockResolvedValue({ + dir: dirFunction, + }); + await fetchContents({ + ...options, + outputPath: 'foo', + fetchUrl: 'https://github.com/backstage/foo', + }); + expect(fs.ensureDir).toBeCalled(); + expect(dirFunction).toBeCalledWith({ targetDir: 'foo' }); + }); +}); From d86db6827802f390394bc2b688986d041dced696 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 14:18:54 +0100 Subject: [PATCH 39/66] chore: Add notice header Signed-off-by: Johan Haals --- .../src/components/fields/RepoUrlPicker/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts index 1f7f9e0327..b0f3df61d9 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts @@ -1 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ export { RepoUrlPicker } from './RepoUrlPicker'; From 78dfa455001608b7bc3c56827f40e33bd19b5557 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 14:22:00 +0100 Subject: [PATCH 40/66] scaffolder: add backstage integration dependency Signed-off-by: Johan Haals --- plugins/scaffolder/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 7b288a25f1..f558f462ad 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -34,6 +34,7 @@ "@backstage/catalog-model": "^0.7.2", "@backstage/config": "^0.1.3", "@backstage/core": "^0.6.3", + "@backstage/integration": "^0.5.0", "@backstage/plugin-catalog-react": "^0.1.0", "@backstage/theme": "^0.2.3", "@material-ui/core": "^4.11.0", From 4b0b8153720bddf8c0e81e9586f2b97d174f7a6a Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 14:44:08 +0100 Subject: [PATCH 41/66] cleanup test template Signed-off-by: Johan Haals --- .../test-template-v2/template.yaml | 290 ------------------ 1 file changed, 290 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 3856906454..4570b911f3 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -1,76 +1,3 @@ -# apiVersion: backstage.io/v1beta2 -# kind: Template -# metadata: -# name: test-template -# title: Test Template -# description: Testing out the new template schema -# spec: -# owner: backstage/techdocs-core -# type: service - -# parameters: -# title: Fill in some BS params -# required: -# - name -# properties: -# name: -# title: Name -# type: string -# description: Unique name of the component -# ui:widget: textarea -# ui:autofocus: true -# ui:options: -# rows: 5 -# storePath: # provides {owner, repository, providerUrl} -# # $ref: '#/definitions/StorePath' -# title: Store Path -# type: string -# description: Copy https://github.com/aeothgiaetgh/aetkijhahte and hammer in some more letters - -# steps: -# - id: prepare -# name: Prepare -# action: legacy:prepare -# parameters: -# protocol: file -# url: file:///Users/patriko/dev/backstage/plugins/scaffolder-backend/sample-templates/test-template-v2 -# - id: template -# name: Template -# action: legacy:template -# parameters: -# templater: cookiecutter -# values: -# name: '{{ parameters.name }}' -# - id: publish -# name: Publish -# action: legacy:publish -# parameters: -# values: -# name: '{{ parameters.name }}' -# storePath: '{{ parameters.storePath }}' -# owner: kungen # not use -# - id: register -# name: Register -# action: catalog:register -# parameters: -# catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' - -# output: -# catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' -# entityRef: '{{ steps.register.entityRef }}' - - -# TODO: -# 1. [x] entity beta2 type & schema -# 2. [x] endpoint for template params + frontend refactor -# 3. [ ] implement new actions that work well with new steps format, e.g. prepare+template = fetch -# 4. [ ] external API for registering custom actions -# 5. [x] implement uiSchema split -# 6. [ ] document beta2 entity schema -# 6. [ ] user documentation for beta2 - ---- - apiVersion: backstage.io/v1beta2 kind: Template metadata: @@ -127,7 +54,6 @@ spec: parameters: allowedHosts: ['github.com'] description: 'This is {{ parameters.name }}' - # repoUrl: github.com?owner=spotify&repo=name repoUrl: '{{ parameters.repoUrl }}' - id: register @@ -136,222 +62,6 @@ spec: parameters: repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' catalogInfoPath: '/catalog-info.yaml' - - - output: - remoteUrl: '{{ steps.publish.output.remoteUrl }}' - entityRef: '{{ steps.register.output.entityRef }}' - ---- - -apiVersion: backstage.io/v1beta2 -kind: Template -metadata: - name: test-template-v2-gitlab - title: Test Template v2-gitlab - description: Testing out the new template schema -spec: - owner: backstage/techdocs-core - type: service - - parameters: - - title: Fill in some steps - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 - - title: Choose a location - required: - - repoUrl - properties: - repoUrl: - title: Repository Location - type: string - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - gitlab.com - - steps: - - id: fetch-base - name: Fetch Base - action: fetch:cookiecutter - parameters: - url: ./template - values: - name: '{{ parameters.name }}' - - - id: fetch-docs - name: Fetch Docs - action: fetch:plain - parameters: - targetPath: ./community - url: https://github.com/backstage/community/tree/main/backstage-community-sessions - - - id: publish - name: Publish - action: publish:gitlab - parameters: - allowedHosts: ['gitlab.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' - - - id: register - name: Register - action: catalog:register - parameters: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' - catalogInfoPath: '/catalog-info.yaml' - - - output: - remoteUrl: '{{ steps.publish.output.remoteUrl }}' - entityRef: '{{ steps.register.output.entityRef }}' - ---- - -apiVersion: backstage.io/v1beta2 -kind: Template -metadata: - name: test-template-v2-bitbucket - title: Test Template v2-bitbucket - description: Testing out the new template schema -spec: - owner: backstage/techdocs-core - type: service - - parameters: - - title: Fill in some steps - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 - - title: Choose a location - required: - - repoUrl - properties: - repoUrl: - title: Repository Location - type: string - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - bitbucket.org - - steps: - - id: fetch-base - name: Fetch Base - action: fetch:cookiecutter - parameters: - url: ./template - values: - name: '{{ parameters.name }}' - - - id: fetch-docs - name: Fetch Docs - action: fetch:plain - parameters: - targetPath: ./community - url: https://github.com/backstage/community/tree/main/backstage-community-sessions - - - id: publish - name: Publish - action: publish:bitbucket - parameters: - allowedHosts: ['bitbucket.org'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' - - - id: register - name: Register - action: catalog:register - parameters: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' - catalogInfoPath: '/catalog-info.yaml' - - - output: - remoteUrl: '{{ steps.publish.output.remoteUrl }}' - entityRef: '{{ steps.register.output.entityRef }}' - ---- -apiVersion: backstage.io/v1beta2 -kind: Template -metadata: - name: test-template-v2-azure - title: Test Template v2-azure - description: Testing out the new template schema -spec: - owner: backstage/techdocs-core - type: service - - parameters: - - title: Fill in some steps - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 - - title: Choose a location - required: - - repoUrl - properties: - repoUrl: - title: Repository Location - type: string - ui:field: RepoUrlPicker - ui:options: - allowedHosts: - - dev.azure.com - - steps: - - id: fetch-base - name: Fetch Base - action: fetch:cookiecutter - parameters: - url: ./template - values: - name: '{{ parameters.name }}' - - - id: fetch-docs - name: Fetch Docs - action: fetch:plain - parameters: - targetPath: ./community - url: https://github.com/backstage/community/tree/main/backstage-community-sessions - - - id: publish - name: Publish - action: publish:azure - parameters: - allowedHosts: ['dev.azure.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' - - - id: register - name: Register - action: catalog:register - parameters: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' - catalogInfoPath: '/catalog-info.yaml' - output: remoteUrl: '{{ steps.publish.output.remoteUrl }}' From 43ff002754c02ed220a192e9c981aef709855ffd Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 15:02:48 +0100 Subject: [PATCH 42/66] Restructure demo template Signed-off-by: Johan Haals --- app-config.yaml | 3 --- .../sample-templates/local-templates.yaml | 1 + .../{test-template-v2 => v1beta2-demo}/template.yaml | 6 +++--- .../template/catalog-info.yaml | 0 4 files changed, 4 insertions(+), 6 deletions(-) rename plugins/scaffolder-backend/sample-templates/{test-template-v2 => v1beta2-demo}/template.yaml (93%) rename plugins/scaffolder-backend/sample-templates/{test-template-v2 => v1beta2-demo}/template/catalog-info.yaml (100%) diff --git a/app-config.yaml b/app-config.yaml index be7391707e..f68f5a30a2 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -248,9 +248,6 @@ catalog: - type: file target: ../catalog-model/examples/acme-corp.yaml - - type: file - target: ../../plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml - scaffolder: github: token: diff --git a/plugins/scaffolder-backend/sample-templates/local-templates.yaml b/plugins/scaffolder-backend/sample-templates/local-templates.yaml index 468fb72ece..b0e16f9dea 100644 --- a/plugins/scaffolder-backend/sample-templates/local-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/local-templates.yaml @@ -9,3 +9,4 @@ spec: - ./docs-template/template.yaml - ./react-ssr-template/template.yaml - ./springboot-grpc-template/template.yaml + - ./v1beta2-demo/template.yaml diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml similarity index 93% rename from plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml rename to plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml index 4570b911f3..b1a4b0d91d 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml @@ -1,9 +1,9 @@ apiVersion: backstage.io/v1beta2 kind: Template metadata: - name: test-template-v2 - title: Test Template v2 - description: Testing out the new template schema + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo publishing to github spec: owner: backstage/techdocs-core type: service diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml similarity index 100% rename from plugins/scaffolder-backend/sample-templates/test-template-v2/template/catalog-info.yaml rename to plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml From 296bba9910b240cdfd326686fa098c0a1868ce4f Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 21:40:29 +0100 Subject: [PATCH 43/66] Delete old scaffolder docs Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../extending/create-your-own-preparer.md | 101 ------------ .../extending/create-your-own-publisher.md | 96 ----------- .../extending/create-your-own-templater.md | 151 ------------------ .../software-templates/extending/index.md | 90 ----------- 4 files changed, 438 deletions(-) delete mode 100644 docs/features/software-templates/extending/create-your-own-preparer.md delete mode 100644 docs/features/software-templates/extending/create-your-own-publisher.md delete mode 100644 docs/features/software-templates/extending/create-your-own-templater.md delete mode 100644 docs/features/software-templates/extending/index.md diff --git a/docs/features/software-templates/extending/create-your-own-preparer.md b/docs/features/software-templates/extending/create-your-own-preparer.md deleted file mode 100644 index 5419748c3a..0000000000 --- a/docs/features/software-templates/extending/create-your-own-preparer.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -id: extending-preparer -title: Create your own Preparer -description: Documentation on Creating your own Preparer ---- - -Preparers are responsible for reading the location of the definition of a -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) and -making a temporary folder with the contents of the selected skeleton. - -Currently, we provide two different providers that can parse two different -location protocols: - -- `file://` -- `github://` - -These two are added to the `PreparersBuilder` and then passed into the -`createRouter` function of the `@backstage/plugin-scaffolder-backend`. - -A full example backend can be found in -[`scaffolder.ts`](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - createRouter, - FilePreparer, - GithubPreparer, - Preparers, -} from '@backstage/plugin-scaffolder-backend'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const preparers = new Preparers(); - - const filePreparer = new FilePreparer(); - const githubPreparer = new GithubPreparer(); - - preparers.register('file', filePreparer); - preparers.register('github', githubPreparer); - - return await createRouter({ - preparers, - templaters, - logger, - }); -} -``` - -As you can see in the above code, a `PreparerBuilder` is created, and then two -of the `preparers` are registered with the different protocols that they accept. - -The `protocol` is set on the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -when added to the service catalog. You can see more about this `PreparerKey` -here in [Register your own template](../adding-templates.md) - -**note:** Currently the catalog supports loading definitions from GitHub + Local -Files, which translate into the two `PreparerKeys`: `file` and `github`. To load -from other places, not only will there need to be another preparer, but the -support to load the location will also need to be added to the Catalog. - -### Creating your own Preparer to add to the `PreparerBuilder` - -All preparers need to implement the `PreparerBase` type. - -That type looks like the following: - -```ts -export type PreparerBase = { - prepare( - template: TemplateEntityV1alpha1, - opts: { logger: Logger }, - ): Promise; -}; -``` - -The `prepare` function will be given the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -along with the source of where the `template.yaml` was loaded from under the -`metedata.annotations.managed-by-location` property. - -Now it's up to you to implement a function which can go and fetch the skeleton -and put the contents into a temporary directory and return that directory path. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts - -### Registering your own Preparer - -You can register the preparer that you have created with the `PreparerBuilder` -by using the `PreparerKey` from the Catalog, for example like this: - -```ts -const preparers = new Preparers(); -preparers.register('gcs', new GoogleCloudStoragePreparer()); -``` - -And then pass this into the `createRouter` function. diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md deleted file mode 100644 index 5e2809e006..0000000000 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -id: extending-publisher -title: Create your own Publisher -description: Documentation on Creating your own Publisher ---- - -Publishers are responsible for pushing and storing the templated skeleton after -the values have been templated by the `Templater`. See -[Create your own templater](./create-your-own-templater.md) for more info. - -They receive a directory or location where the templater has successfully run -and is now ready to store somewhere. They also are given some other options -which are sent from the frontend, such as the `storePath` which is a string of -where the frontend thinks we should save this templated folder. - -Currently we provide the following `publishers`: - -- `github` - -This publisher is passed through to the `createRouter` function of the -`@backstage/plugin-scaffolder-backend`. Currently, only one publisher is -supported, but PR's are always welcome. - -An full example backend can be found -[here](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - createRouter, - GithubPublisher, -} from '@backstage/plugin-scaffolder-backend'; -import { Octokit } from '@octokit/rest'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const githubClient = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const publisher = new GithubPublisher({ client: githubClient }); - - return await createRouter({ - publisher, - logger, - }); -} -``` - -The publisher will always be called with the location from the selected -`Preparer`. - -### Create your own Publisher and register it with the Scaffolder - -All `publishers` need to implement the `PublisherBase` type. - -That type looks like the following: - -```ts -export type PublisherBase = { - publish(opts: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - directory: string; - }): Promise<{ remoteUrl: string }>; -}; -``` - -The `publisher` function will be called with an `options` object which contains -the following: - -- `entity` - the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is currently being scaffolded -- `values` - a json object which will resemble the `spec.schema` from the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is defined here under spec.schema`. More info can be found here - [Register your own template](../adding-templates.md#adding-form-values-in-the-scaffolder-wizard) -- `directory` - a string containing the returned path from the `templater`. See - more information here in - [Create your own templater](./create-your-own-templater.md) - -Now it's up to you to implement the `publish` function and return -`{ remoteUrl: string }` which can be used to identify the finished product. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts - -### Registering your own Publisher - -Currently, we only support one `publisher` (PR's welcome), but you can register -any single `publisher` with the `createRouter`. - -```ts -return await createRouter({ - publisher: new MyGitlabPublisher(), -}); -``` diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md deleted file mode 100644 index 28769d77e1..0000000000 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -id: extending-templater -title: Creating your own Templater -description: Documentation on Creating your own Templater ---- - -Templaters are responsible for taking the directory path for the skeleton -returned by the preparers, and then executing the templating command on top of -the file and returning the completed template path. This may or may not be the -same directory as the input directory. - -They also receive additional values from the frontend, which can be used to -interpolate into the skeleton files. - -Currently we provide the following templaters: - -- `cookiecutter` - -This templater is added to the `TemplaterBuilder` and then passed into the -`createRouter` function of the `@backstage/plugin-scaffolder-backend` - -An full example backend can be found -[here](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - CookieCutter, - createRouter, - Templaters, -} from '@backstage/plugin-scaffolder-backend'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const templaters = new Templaters(); - const cookiecutterTemplater = new CookieCutter(); - templaters.register('cookiecutter', cookiecutterTemplater); - - return await createRouter({ - templaters, - }); -} -``` - -As you can see in the above code a `TemplaterBuilder` is created and the default -`cookiecutter` `templater` is registered under the key `cookicutter`. - -This `TemplaterKey` is used to select the correct templater from the -`spec.templater` in the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template). - -If you wish to add a new templater, you'll need to register it with the -`TemplaterBuilder`. - -### Creating your own Templater to add to the `TemplaterBuilder` - -All templaters need to implement the `TemplaterBase` type. - -That type looks like the following: - -```ts -export type TemplaterRunOptions = { - directory: string; - values: TemplaterValues; - logStream?: Writable; - dockerClient: Docker; -}; - -export type TemplaterBase = { - run(opts: TemplaterRunOptions): Promise; -}; -``` - -The `run` function will be given a `TemplaterRunOptions` object which is as -follows: - -- `directory`- the skeleton directory returned from the `Preparer`, more info at - [Create your own preparer](./create-your-own-preparer.md). -- `values` - a json object which will resemble the `spec.schema` from the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is defined here under spec.schema`. More info can be found here - [Register your own template](../adding-templates.md#adding-form-values-in-the-scaffolder-wizard) -- `logStream` - a stream that you can write to for displaying in the frontend. -- `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to - be able to run docker containers. - -_note_ Currently the templaters that we provide are basically Docker action -containers that are run on top of the skeleton folder. This keeps dependencies -to a minimum for running Backstage scaffolder, but you don't _have_ to use -Docker. You can `pip install cookiecutter` to run it locally in your backend. -You could create your own templater that spins up an EC2 instance and downloads -the folder and does everything using an AMI if you want. It's entirely up to -you! - -Now it's up to you to implement the `run` function, and then return a -`TemplaterRunResult` which is `{ resultDir: string }`. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts - -### Registering your own Templater - -If you try to process a -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -with a new `spec.templater` value, you'll need to register that with the -`TemplaterBuilder`. - -For example let's say you have the following -[Template Entity](../../software-catalog/descriptor-format.md#kind-template): - -```yaml -apiVersion: backstage.io/v1alpha1 -kind: Template -metadata: - name: react-ssr-template - title: React SSR Template - description: - Next.js application skeleton for creating isomorphic web applications. - tags: - - recommended - - react -spec: - owner: web@example.com - templater: handlebars - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component -``` - -You see that the `spec.templater` is set as `handlebars`, so you'll need to -register this with the `TemplaterBuilder` like so: - -```ts -const templaters = new Templaters(); -templaters.register('handlebars', new HandlebarsTemplater()); -``` - -And then pass this into the `createRouter` function. diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md deleted file mode 100644 index aaf42d6b12..0000000000 --- a/docs/features/software-templates/extending/index.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -id: extending-index -title: Extending the Scaffolder ---- - -Welcome. Take a seat. You're at the Scaffolder Documentation. - -So, you want to create stuff inside your company from some prebaked templates? -You're at the right place. - -This guide is going to take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through what's going on in the backend to be -able to create these templates. There's also more guides that you might find -useful at the bottom of this document. At its core, there are 3 simple stages. - -1. Pick a skeleton -2. Template some variables into the skeleton -3. Send the templated skeleton somewhere - -These three steps are translated to the following stages under the hood in the -scaffolder that you will need to know: - -1. Prepare -2. Template -3. Publish - -Each of these steps can be configured for your own use case, but we provide some -sensible defaults, too. - -Let's dive a little deeper into these phases. - -### Glossary and Jargon - -**Preparer** - The preparer is responsible for fetching the skeleton code and -placing it into a directory and then will return that directory. It is -registered with the `Preparers` with a particular type, which is then used in -the router to pick the correct `Preparer` to run for the `Template` entity. - -**Templater** - The templater is responsible for actually running the chosen -templater on top of the previously returned temporary directory from the -**Preparer**. We advise making these Docker containers as it can keep all -dependencies--for example Cookiecutter--self contained and not a dependency on -the host machine. - -**Publisher** - The publisher is responsible for taking the finished directory, -and publishing it to a remote registry. This could be a Git repository or -something similar. Right now, the scaffolder only supports one publishing method -for the entire lifecycle, but it could be configured from the frontend and -passed through to the scaffolder backend. - -### How it works - -Most of the heavy lifting is done in the -[router.ts](https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) -file in the `scaffolder-backend` plugin. - -There are two routes defined in the router: `POST /v1/jobs` and -`GET /v1/job/:jobId` - -To create a scaffolding job, a JSON object containing the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) + -additional templating values must be posted as the post body. - -```js -{ - "template": { - "apiVersion": "backstage/v1alpha1", - "kind": "Template", - // more stuff here - }, - "values": { - "component_id": "test", - "description": "somethingelse" - } -} -``` - -The values should represent something that is valid with the `schema` part of -the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - -Once that has been posted, a job will be setup with different stages, and the -job processor will complete each stage before moving onto the next stage, whilst -collecting logs and mutating the running job. - -Here's some further reading that you might find useful: - -- [Adding your own Template](../adding-templates.md) -- [Creating your own Templater](./create-your-own-templater.md) -- [Creating your own Publisher](./create-your-own-publisher.md) -- [Creating your own Preparer](./create-your-own-preparer.md) From 37744d8f4bd05e457ce680a22c58ca83ab2ef972 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Fri, 26 Feb 2021 21:41:37 +0100 Subject: [PATCH 44/66] Add docs for writing templates Co-authored-by: blam Signed-off-by: Johan Haals --- .../software-templates/writing-templates.md | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 docs/features/software-templates/writing-templates.md diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md new file mode 100644 index 0000000000..e7abd002e4 --- /dev/null +++ b/docs/features/software-templates/writing-templates.md @@ -0,0 +1,286 @@ +--- +id: writing-templates +title: Writing Templates +description: Details around creating your own custom Software Templates +--- + +Templates are stored in the **Service Catalog** under a kind `Template`. You can +create your own templates with a small `yaml` definition which describes the +template and it's metadata, along with some input variables that your template +will need, and then a list of actions which are then executed by the scaffolding +service. + +Let's take a look at a simple example: + +```yaml +# Notice the v1beta2 version +apiVersion: backstage.io/v1beta2 +kind: Template +# some metadata about the template itself +metadata: + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo +spec: + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + # some outputs which are saved along with the job for use in the frontend + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' +``` + +Let's dive in an pick apart what each of these sections do and what they are. + +### `spec.parameters` - `FormStep | FormStep[]` + +These `parameters` are template variables which can be modified in the frontend +as a sequence. It can either be one `Step` if you just want one big list of +different fields in the frontend, or it can be broken up into multiple different +steps which would be rendered as different steps in the scaffolder plugin +frontend. + +Each `Step` is `JSONSchema` with some extra goodies for styling what it might +look like in the frontend. For these steps we rely very heavily on this library: +https://github.com/rjsf-team/react-jsonschema-form. They have some great docs +too here: https://react-jsonschema-form.readthedocs.io/ and a playground where +you can play around with some examples here +https://rjsf-team.github.io/react-jsonschema-form. + +There's another option for that library called `uiSchema` which we've taken +advantage of, and we've merged it with the existing `JSONSchema` that you +provide to the library. These are the little `ui:*` properties that you can see +in the step definitions. + +For example if we take the **simple** example from the playground it looks like +this: + +```json +// jsonSchema: +{ + "title": "A registration form", + "description": "A simple form example.", + "type": "object", + "required": [ + "firstName", + "lastName" + ], + "properties": { + "firstName": { + "type": "string", + "title": "First name", + "default": "Chuck" + }, + "lastName": { + "type": "string", + "title": "Last name" + }, + "telephone": { + "type": "string", + "title": "Telephone", + "minLength": 10 + } + } +} + +// uiSchema: +{ + "firstName": { + "ui:autofocus": true, + "ui:emptyValue": "", + "ui:autocomplete": "family-name" + }, + "lastName": { + "ui:emptyValue": "", + "ui:autocomplete": "given-name" + }, + "telephone": { + "ui:options": { + "inputType": "tel" + } + } +} +``` + +It would look something like the following in a template: + +```yaml +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: A registration form + description: A simple form example. + type: object + required: + - firstName + - lastName + properties: + firstName: + type: string + title: First name + default: Chuck + ui:autofocus: true + ui:emptyValue: '' + ui:autocomplete: family-name + lastName: + type: string + title: Last name + ui:emptyValue: '' + ui:autocomplete: given-name + telephone: + type: string + title: Telephone + minLength: 10 + ui:options: + inputType: tel +``` + +#### The Repository Picker + +So in order to make working with repository providers easier, we've built a +custom picker that can be used by overriding the `ui:field` option in the +`uiSchema` for a `string` field. Instead of displaing a text input block it will +render our custom component that we've built which makes it easy to select a +repository provider, and insert a project or owner, and repository name. + +You can see it in the above full example which is a seperate step and it looks a +little like this: + +```yaml + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com +``` + +The `allowedHosts` part should be set to where you wish to enable this template +to publish to. And it can be any host that is listed in your `integrations` +config in `app-config.yaml`. + +The `RepoUrlPicker` is a custom field that we provide part of the +`plugin-scaffolder`. It's currently not possible to create your own fields yet, +but contributions are welcome! :) + +### `spec.steps` - `Action[]` + +The `steps` is an array of the things that you want to happen part of this +template. These follow the same standard format: + +```yaml +- id: fetch-base # A unique id for the step + name: Fetch Base # A title displayed in the frontend + action: fetch:cookiecutter # an action to call + input: # input that is passed as arguments to the action handler + url: ./template + values: + name: '{{ parameters.name }}' +``` + +By default we ship some built in actions that you can take a look at +[here](./builtin-actions.md), or you can create your own custom actions by +looking at the docs [here](./writing-custom-actions.md) + +### Outputs + +Each individual step can output some variables that can be used in the +scaffolder frontend for after the job is finished. This is useful for things +like linking to the entity that has been created with the backend, and also +linking to the created repository. + +The main two that are used are the following: + +```yaml +output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' # link to the remote repository + entityRef: '{{ steps.register.output.entityRef }}' # link to the entitiy that has been ingested to the catalog +``` + +### The templating syntax + +You might have noticed in the examples that there are `{{ }}`, and these are a +`handlebars` templates for linking and glueing all these different parts of +`yaml` together. All the form inputs from the `parameters` section, when passed +to the steps will be available by using the template syntax +`{{ parameters.something }}`. This is great for passing the values from the form +into different steps and reusing these input variables. + +As you can see above in the `Outputs` section, `actions` and `steps` can also +output things. So you can grab that output by using +`steps.$stepId.output.$property`. + +You can read more about all the `inputs` and `outputs` defined in the actions in +code part of the `JSONSchema` or you can read more about our built in ones +[here](./builtin-actions.md). From 053d6268ecac583e13a97b796a722a6abd299778 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 09:16:27 +0100 Subject: [PATCH 45/66] chore: remove dev comments Signed-off-by: Johan Haals --- .../src/scaffolder/actions/createTemplateAction.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts index f501c3f7ac..0bd52d0540 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -// function createTemplateAction( -// options: TemplateAction, -// ): TemplateAction; - import { InputBase, TemplateAction } from './types'; export const createTemplateAction = ( From 4061baea2378df2a483cd4ae95825d45df4d539d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 09:17:09 +0100 Subject: [PATCH 46/66] Update plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelรถw Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/publish/azure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index e459555adb..58ba6d3131 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -94,7 +94,7 @@ export function createPublishAzureAction(options: { if (!returnedRepo) { throw new InputError( `Unable to create the repository with Organization ${organization}, Project ${owner} and Repo ${repo}. - Please make sure you that both the Org and Project are typed corrected and exist.`, + Please make sure that both the Org and Project are typed corrected and exist.`, ); } const remoteUrl = returnedRepo.remoteUrl; From bd630612b74897d7f99be92ffc896890e28e8575 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 09:23:24 +0100 Subject: [PATCH 47/66] chore: remove redundant if statement Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/catalog/register.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts index b23e41b736..7cf1ac2120 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -100,9 +100,7 @@ export function createCatalogRegisterAction(options: { if (result.entities.length >= 1) { const { kind, name, namespace } = getEntityName(result.entities[0]); ctx.output('entityRef', `${kind}:${namespace}/${name}`); - if (catalogInfoUrl) { - ctx.output('catalogInfoUrl', catalogInfoUrl); - } + ctx.output('catalogInfoUrl', catalogInfoUrl); } }, }); From e2d3b6ada13b6051fa791432932c261a59e4f8b9 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 09:26:31 +0100 Subject: [PATCH 48/66] chore: set test entity kind to Component Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/catalog/register.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts index 9fa96132d6..070ad15ab7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts @@ -74,7 +74,7 @@ describe('catalog:register', () => { namespace: 'default', name: 'test', }, - kind: 'service', + kind: 'Component', } as Entity, ], }); @@ -91,7 +91,7 @@ describe('catalog:register', () => { expect(mockContext.output).toBeCalledWith( 'entityRef', - 'service:default/test', + 'Component:default/test', ); expect(mockContext.output).toBeCalledWith( 'catalogInfoUrl', From 442c2a1ce3d0046b0bedad3c727ca81b769b8b3b Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 10:12:48 +0100 Subject: [PATCH 49/66] pass reader to createRouter, fix imports Signed-off-by: Johan Haals --- .../packages/backend/src/plugins/scaffolder.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index 49cd8a9cda..a3416c5080 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -2,13 +2,10 @@ import { SingleHostDiscovery } from '@backstage/backend-common'; import { CatalogClient } from '@backstage/catalog-client'; import { CookieCutter, - - - - CreateReactAppTemplater, createRouter, + CreateReactAppTemplater, + createRouter, Preparers, Publishers, - Templaters } from '@backstage/plugin-scaffolder-backend'; import Docker from 'dockerode'; @@ -19,6 +16,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment): Promise { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -44,5 +42,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + reader }); } From a01e6d920d024fad91311bd968d6bcfd01988742 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 10:20:11 +0100 Subject: [PATCH 50/66] chore: fix typos Signed-off-by: Johan Haals --- docs/features/software-templates/writing-templates.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md index e7abd002e4..d9dbb67ab2 100644 --- a/docs/features/software-templates/writing-templates.md +++ b/docs/features/software-templates/writing-templates.md @@ -205,11 +205,11 @@ spec: So in order to make working with repository providers easier, we've built a custom picker that can be used by overriding the `ui:field` option in the -`uiSchema` for a `string` field. Instead of displaing a text input block it will -render our custom component that we've built which makes it easy to select a -repository provider, and insert a project or owner, and repository name. +`uiSchema` for a `string` field. Instead of displaying a text input block it +will render our custom component that we've built which makes it easy to select +a repository provider, and insert a project or owner, and repository name. -You can see it in the above full example which is a seperate step and it looks a +You can see it in the above full example which is a separate step and it looks a little like this: ```yaml From 74850569bfa69763bfe58bf23fb1557bfe831cf1 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 10:58:31 +0100 Subject: [PATCH 51/66] Document builtin actions Signed-off-by: Johan Haals --- .../software-templates/builtin-actions.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/features/software-templates/builtin-actions.md diff --git a/docs/features/software-templates/builtin-actions.md b/docs/features/software-templates/builtin-actions.md new file mode 100644 index 0000000000..ae8273e390 --- /dev/null +++ b/docs/features/software-templates/builtin-actions.md @@ -0,0 +1,115 @@ +--- +id: builtin-actions +title: Builtin actions +description: Documentation describing the built-in template actions. +--- + +# Built-in Actions + +This is the list of built-in template actions + +## `fetch:plain` + +Downloads content and places it in the workspacePath or optionally in a +subdirectory specified by the `targetPath` input option. + +input: + +- `url` (required) Relative path or absolute URL pointing to the directory tree + to fetch. +- `targetPath` Target path within the working directory to download the contents + to. + +output: nothing + +## `fetch:cookiecutter` + +Downloads template from `url` and templates with cookiecutter + +input: + +- `url` (required) Relative path or absolute URL pointing to the directory tree + to fetch. +- `targetPath` Target path within the working directory to download the contents + to. +- `values` Values to pass on to cookiecutter for templating. + +output: nothing + +## `catalog:register` + +Registers entity in the software catalog. + +input: + +- `catalogInfoUrl` (required) An absolute URL pointing to the catalog info file + location. + +output: + +- `repoContentsUrl` An absolute URL pointing to the root of a repository + directory tree. +- `catalogInfoPath` A relative path from the repo root pointing to the catalog + info file, defaults to /catalog-info.yaml + +## `publish:github` + +Initializes a git repository of contents in workspacePath and publishes to +GitHub. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public' or 'internal'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:gitlab` + +Initializes a git repository of contents in workspacePath and publishes to +Gitlab. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public' or 'internal'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:bitbucket` + +Initializes a git repository of contents in workspacePath and publishes to +Bitbucket. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:azure` + +Initializes a git repository of contents in workspacePath and publishes to +Azure. + +input: + +- `repoUrl` (required) Repository location + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository From 8106c9528ff863b6bc19e4598cad7b5db09bd2c5 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:13:42 +0100 Subject: [PATCH 52/66] Add changeset Signed-off-by: Johan Haals --- .changeset/pink-ads-shave.md | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .changeset/pink-ads-shave.md diff --git a/.changeset/pink-ads-shave.md b/.changeset/pink-ads-shave.md new file mode 100644 index 0000000000..c1c281b6a1 --- /dev/null +++ b/.changeset/pink-ads-shave.md @@ -0,0 +1,37 @@ +--- +'@backstage/plugin-scaffolder': minor +'@backstage/plugin-scaffolder-backend': minor +--- + +The scaffolder has been updated to support the new `v1beta2` template schema which allows for custom template actions! + +See documentation for more information how to create and register new template actions. + +**Breaking changes** + +The backend scaffolder plugin now needs a `UrlReader` which can be receieved from the PluginEnvironment. + +The following change is required in `backend/src/plugins/scaffolder.ts` + +```diff + export default async function createPlugin({ + logger, + config, + database, ++ reader, + }: PluginEnvironment): Promise { + + // omitted code + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, ++ reader, + }); +``` From 41b044c81ae8a445608e6569d156a97de4b9bd15 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:20:44 +0100 Subject: [PATCH 53/66] chore: use interface type instead of implementation Signed-off-by: Johan Haals --- .../src/scaffolder/actions/builtin/publish/azure.ts | 4 ++-- .../src/scaffolder/actions/builtin/publish/bitbucket.ts | 4 ++-- .../src/scaffolder/actions/builtin/publish/github.ts | 4 ++-- .../src/scaffolder/actions/builtin/publish/gitlab.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 58ba6d3131..5eaba05ea0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -15,7 +15,7 @@ */ import { InputError } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; +import { ScmIntegrationRegistry } from '@backstage/integration'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; @@ -23,7 +23,7 @@ import { parseRepoUrl } from './util'; import { createTemplateAction } from '../../createTemplateAction'; export function createPublishAzureAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; }) { const { integrations } = options; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 1cc77cbf81..4c3cf30ecd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -17,7 +17,7 @@ import { InputError } from '@backstage/backend-common'; import { BitbucketIntegrationConfig, - ScmIntegrations, + ScmIntegrationRegistry, } from '@backstage/integration'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; @@ -157,7 +157,7 @@ const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { }; export function createPublishBitbucketAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; }) { const { integrations } = options; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index d449ee135c..be0820d0b1 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -17,7 +17,7 @@ import { InputError } from '@backstage/backend-common'; import { GithubCredentialsProvider, - ScmIntegrations, + ScmIntegrationRegistry, } from '@backstage/integration'; import { Octokit } from '@octokit/rest'; import { initRepoAndPush } from '../../../stages/publish/helpers'; @@ -25,7 +25,7 @@ import { parseRepoUrl } from './util'; import { createTemplateAction } from '../../createTemplateAction'; export function createPublishGithubAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; }) { const { integrations } = options; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index 603ca84f05..9122c828a0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -15,14 +15,14 @@ */ import { InputError } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; +import { ScmIntegrationRegistry } from '@backstage/integration'; import { Gitlab } from '@gitbeaker/node'; import { initRepoAndPush } from '../../../stages/publish/helpers'; import { parseRepoUrl } from './util'; import { createTemplateAction } from '../../createTemplateAction'; export function createPublishGitlabAction(options: { - integrations: ScmIntegrations; + integrations: ScmIntegrationRegistry; }) { const { integrations } = options; From 78cfb21b1a55371c9884ac135a4afea7ace2bfc7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:30:13 +0100 Subject: [PATCH 54/66] Update scaffolder documentation Co-authored-by: Ben Lambert Signed-off-by: Johan Haals --- .../software-catalog/descriptor-format.md | 155 +++++++++++------ .../software-templates/adding-templates.md | 159 ++++++++---------- .../software-templates/installation.md | 49 ++++-- docs/features/software-templates/legacy.md | 118 +++++++++++++ .../writing-custom-actions.md | 0 5 files changed, 328 insertions(+), 153 deletions(-) create mode 100644 docs/features/software-templates/legacy.md create mode 100644 docs/features/software-templates/writing-custom-actions.md diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 6658dfc055..59f09b6e58 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -522,49 +522,90 @@ consumed by the component, e.g. `artist-api`. This field is optional. ## Kind: Template -Describes the following entity kind: +The following describes the following entity kind: -| Field | Value | -| ------------ | ----------------------- | -| `apiVersion` | `backstage.io/v1alpha1` | -| `kind` | `Template` | +| Field | Value | +| ------------ | ---------------------- | +| `apiVersion` | `backstage.io/v1beta2` | +| `kind` | `Template` | -A Template describes a skeleton for use with the Scaffolder. It is used for -describing what templating library is supported, and also for documenting the -variables that the template requires using -[JSON Forms Schema](https://jsonforms.io/). +If you're looking for docs on `v1alpha1` you can find them +[here](../software-templates/legacy) + +A template definition describes both the parameters that are rendered in the +frontend part of the scaffolding wizard, and the steps that are executed when +scaffolding that component. Descriptor files for this kind may look as follows. ```yaml -apiVersion: backstage.io/v1alpha1 +apiVersion: backstage.io/v1beta2 kind: Template +# some metadata about the template itself metadata: - name: react-ssr-template - title: React SSR Template - description: - Next.js application skeleton for creating isomorphic web applications. - tags: - - recommended - - react + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo spec: - owner: web@example.com - templater: cookiecutter - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' ``` In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) @@ -572,7 +613,7 @@ shape, this kind has the following structure. ### `apiVersion` and `kind` [required] -Exactly equal to `backstage.io/v1alpha1` and `Template`, respectively. +Exactly equal to `backstage.io/v1beta2` and `Template`, respectively. ### `metadata.title` [required] @@ -605,31 +646,43 @@ The current set of well-known and common values for this field is: - `website` - a website - `library` - a software library, such as an npm module or a Java library -### `spec.templater` [required] +### `spec.parameters` [required] -The templating library that is supported by the template skeleton as a string, -e.g `cookiecutter`. +You can find out more about the `parameters` key +[here](../software-templates/writing-templates.md) -Different skeletons will use different templating syntax, so it's common that -the template will need to be run with a particular piece of software. +### `spec.steps` [optional] -This key will be used to identify the correct templater which is registered into -the `TemplatersBuilder`. +You can find out more about the `steps` key +[here](../software-templates/writing-templates.md) -The values which are available by default are: +## Kind: APIis `JSONSchema` with some extra goodies for styling what it might -- `cookiecutter` - [cookiecutter](https://github.com/cookiecutter/cookiecutter). +look like in the frontend. For these steps we rely very heavily on this library: +https://github.com/rjsf-team/react-jsonschema-form. They have some great docs +too here: https://react-jsonschema-form.readthedocs.io/ and a playground where +you can play around with some examples here +https://rjsf-team.github.io/react-jsonschema-form -### `spec.path` [optional] +For example here is a simple parameters step -The string location where the templater should be run if it is not on the same -level as the `template.yaml` definition, e.g. `./cookiecutter/skeleton`. +```yaml + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 +``` -This will set the `cwd` when running the templater to the folder path that you -specify relative to the `template.yaml` definition. +You can find out more [here](../software-templates/writing-templates.md) -This is also particularly useful when you have multiple template definitions in -the same repository but only a single `template.yaml` registered in backstage. +### `spec.steps` [optional] ## Kind: API diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index e2b69d6767..37659f214e 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -4,49 +4,80 @@ title: Adding your own Templates description: Documentation on Adding your own Templates --- -Templates are stored in the **Service Catalog** under a kind `Template`. The +Templates are stored in the **Software Catalog** under a kind `Template`. The minimum that is needed to define a template is a `template.yaml` file, but it would be good to also have some files in there that can be templated in. A simple `template.yaml` definition might look something like this: ```yaml -apiVersion: backstage.io/v1alpha1 +apiVersion: backstage.io/v1beta2 kind: Template +# some metadata about the template itself metadata: - # unique name per namespace for the template - name: react-ssr-template - # title of the template - title: React SSR Template - # a description of the template - description: - Next.js application skeleton for creating isomorphic web applications. - # some tags to display in the frontend - tags: - - recommended - - react + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo spec: - # which templater key to use in the templaters builder - templater: cookiecutter - # what does this template create - type: website - # if the template is not in the current directory where this definition is kept then specify - path: './template' - # the schema for the form which is displayed in the frontend. - # should follow JSON schema for forms: https://jsonforms.io/ - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' ``` [Template Entity](../software-catalog/descriptor-format.md#kind-template) @@ -55,14 +86,6 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. -_NOTE_: When the `publish` step is completed, it is currently assumed by the -scaffolder that the final repository should contain a `catalog-info.yaml` in -order to register this with the Catalog in Backstage. - -Currently the catalog supports loading definitions from GitHub + Local Files. To -load from other places, not only will there need to be another preparer, but the -support to load the location will also need to be added to the Catalog. - You can add the template files to the catalog through [static location configuration](../software-catalog/configuration.md#static-location-configuration), for example: @@ -76,52 +99,8 @@ catalog: - allow: [Template] ``` -Templates can also be added by posting the to the catalog directly. Note that if -you're doing this, you need to configure the catalog to allow template entities -to be ingested from any source, for example: +Or you can add the template using the `catalog-import` plugin, which unless +configured differently should be running on `/catalog-import`. -```yaml -catalog: - rules: - - allow: [Component, API, Template] -``` - -For loading from a file, the following command should work when the backend is -running: - -```sh -curl \ - --location \ - --request POST 'localhost:7000/api/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" -``` - -If loading from a Git location, you can run the following - -```sh -curl \ - --location \ - --request POST 'localhost:7000/api/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"github\", \"target\": \"https://${GITHUB URL}/${YOUR GITHUB ORG/REPO}/blob/master/${PATH TO FOLDER}/template.yaml\"}" -``` - -This should then have been added the catalog, and be listed under the create -page at http://localhost:3000/create. - -The `type` field which is chosen in the request to add the `template.yaml` to -the Service Catalog here, will become the `PreparerKey` which will be used to -select the `Preparer` when creating a job. - -### Adding form values in the Scaffolder Wizard - -The `spec.schema` property in the -[Template Entity](../software-catalog/descriptor-format.md#kind-template) is a -`yaml` version of the JSON Form Schema standard. - -Here you can define the key/values and then the wizard will convert this to a -form for the user to fill in when your template is selected. - -You can find out much more about the standard and how to use it here: -https://jsonforms.io +For information about writing your own templates, you can check out the docs +[here](./writing-templates) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 9aa1d3bc8f..fd51bffa5b 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -99,6 +99,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -124,6 +125,7 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + reader, }); } ``` @@ -136,10 +138,11 @@ import scaffolder from './plugins/scaffolder'; const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); -const service = createServiceBuilder(module) - .loadConfig(configReader) - /** several different routers */ - .addRouter('/scaffolder', await scaffolder(scaffolderEnv)); +const apiRouter = Router(); +/* several router .use calls */ + +/* add this line */ +apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); ``` ### Adding Templates @@ -170,14 +173,10 @@ catalog: ### Runtime Dependencies / Configuration -For the scaffolder backend plugin to function, it needs a GitHub access token, -and access to a running Docker daemon. You can create a GitHub access token -[here](https://github.com/settings/tokens/new), select `repo` scope only. Full -docs on creating private GitHub access tokens is available -[here](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). -Note that the need for private GitHub access tokens will be replaced with GitHub -Apps integration further down the line by using the existing `integrations` -config. +For the scaffolder backend plugin to function, you'll need to setup the +integrations config in your `app-config.yaml`. + +You can find help for different providers below. > Note: Some of this configuration may already be set up as part of your > `app-config.yaml`. We're moving away from the duplicated config for @@ -220,6 +219,32 @@ integrations: $env: GITLAB_TOKEN ``` +#### BitBucket + +For Bitbucket there are two authentication methods supported. Either `token` or +a combination of `appPassword` and `username`. It looks like either of the +following: + +```yaml +integrations: + bitbucket: + - host: bitbucket.org + token: + $env: BITBUCKET_TOKEN +``` + +or + +```yaml +integrations: + bitbucket: + - host: bitbucket.org + appPassword: + $env: BITBUCKET_APP_PASSWORD + username: + $env: BITBUCKET_USERNAME +``` + #### Azure DevOps For Azure DevOps we support both the preparer and publisher stage with the diff --git a/docs/features/software-templates/legacy.md b/docs/features/software-templates/legacy.md new file mode 100644 index 0000000000..d70773e251 --- /dev/null +++ b/docs/features/software-templates/legacy.md @@ -0,0 +1,118 @@ +--- +id: template-legacy +title: Writing Templates (Legacy) +description: + Old documentation describing the backstage.io/v1alpha1 format of the Template + Schema +--- + +## Kind: Template + +Describes the following entity kind: + +| Field | Value | +| ------------ | ----------------------- | +| `apiVersion` | `backstage.io/v1alpha1` | +| `kind` | `Template` | + +A Template describes a skeleton for use with the Scaffolder. It is used for +describing what templating library is supported, and also for documenting the +variables that the template requires using +[JSON Forms Schema](https://jsonforms.io/). + +Descriptor files for this kind may look as follows. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: Template +metadata: + name: react-ssr-template + title: React SSR Template + description: + Next.js application skeleton for creating isomorphic web applications. + tags: + - recommended + - react +spec: + owner: web@example.com + templater: cookiecutter + type: website + path: '.' + schema: + required: + - component_id + - description + properties: + component_id: + title: Name + type: string + description: Unique name of the component + description: + title: Description + type: string + description: Description of the component +``` + +In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) +shape, this kind has the following structure. + +### `apiVersion` and `kind` [required] + +Exactly equal to `backstage.io/v1alpha1` and `Template`, respectively. + +### `metadata.title` [required] + +The nice display name for the template as a string, e.g. `React SSR Template`. +This field is required as is used to reference the template to the user instead +of the `metadata.name` field. + +### `metadata.tags` [optional] + +A list of strings that can be associated with the template, e.g. +`['recommended', 'react']`. + +This list will also be used in the frontend to display to the user so you can +potentially search and group templates by these tags. + +### `spec.type` [optional] + +The type of component as a string, e.g. `website`. This field is optional but +recommended. + +The software catalog accepts any type value, but an organization should take +great care to establish a proper taxonomy for these. Tools including Backstage +itself may read this field and behave differently depending on its value. For +example, a website type component may present tooling in the Backstage interface +that is specific to just websites. + +The current set of well-known and common values for this field is: + +- `service` - a backend service, typically exposing an API +- `website` - a website +- `library` - a software library, such as an npm module or a Java library + +### `spec.templater` [required] + +The templating library that is supported by the template skeleton as a string, +e.g `cookiecutter`. + +Different skeletons will use different templating syntax, so it's common that +the template will need to be run with a particular piece of software. + +This key will be used to identify the correct templater which is registered into +the `TemplatersBuilder`. + +The values which are available by default are: + +- `cookiecutter` - [cookiecutter](https://github.com/cookiecutter/cookiecutter). + +### `spec.path` [optional] + +The string location where the templater should be run if it is not on the same +level as the `template.yaml` definition, e.g. `./cookiecutter/skeleton`. + +This will set the `cwd` when running the templater to the folder path that you +specify relative to the `template.yaml` definition. + +This is also particularly useful when you have multiple template definitions in +the same repository but only a single `template.yaml` registered in backstage. diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md new file mode 100644 index 0000000000..e69de29bb2 From abadf05a9f44a456c6576cd8868f8a548439c63d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:41:16 +0100 Subject: [PATCH 55/66] fix links Co-authored-by: blam Signed-off-by: Johan Haals --- docs/features/software-catalog/descriptor-format.md | 2 +- docs/features/software-templates/adding-templates.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 59f09b6e58..136f9d5cc2 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -530,7 +530,7 @@ The following describes the following entity kind: | `kind` | `Template` | If you're looking for docs on `v1alpha1` you can find them -[here](../software-templates/legacy) +[here](../software-templates/legacy.md) A template definition describes both the parameters that are rendered in the frontend part of the scaffolding wizard, and the steps that are executed when diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index 37659f214e..ecb7d11be9 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -103,4 +103,4 @@ Or you can add the template using the `catalog-import` plugin, which unless configured differently should be running on `/catalog-import`. For information about writing your own templates, you can check out the docs -[here](./writing-templates) +[here](./writing-templates.md) From fe2823307fc5ebf2ad950975f0f8caf406adb486 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:43:05 +0100 Subject: [PATCH 56/66] chore: fix typos Co-authored-by: blam Signed-off-by: Johan Haals --- .changeset/pink-ads-shave.md | 2 +- docs/features/software-templates/builtin-actions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/pink-ads-shave.md b/.changeset/pink-ads-shave.md index c1c281b6a1..50d0ddef22 100644 --- a/.changeset/pink-ads-shave.md +++ b/.changeset/pink-ads-shave.md @@ -9,7 +9,7 @@ See documentation for more information how to create and register new template a **Breaking changes** -The backend scaffolder plugin now needs a `UrlReader` which can be receieved from the PluginEnvironment. +The backend scaffolder plugin now needs a `UrlReader` which can be pulled from the PluginEnvironment. The following change is required in `backend/src/plugins/scaffolder.ts` diff --git a/docs/features/software-templates/builtin-actions.md b/docs/features/software-templates/builtin-actions.md index ae8273e390..e6886b1f04 100644 --- a/docs/features/software-templates/builtin-actions.md +++ b/docs/features/software-templates/builtin-actions.md @@ -71,7 +71,7 @@ output: ## `publish:gitlab` Initializes a git repository of contents in workspacePath and publishes to -Gitlab. +GitLab. input: From c3a845c1a4ba45ef6daf99cb63283be9998f8e92 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:53:07 +0100 Subject: [PATCH 57/66] Wildcard export RepoUrlPicker Signed-off-by: Johan Haals --- plugins/scaffolder/src/components/fields/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder/src/components/fields/index.ts b/plugins/scaffolder/src/components/fields/index.ts index b0f3df61d9..fc3e70378d 100644 --- a/plugins/scaffolder/src/components/fields/index.ts +++ b/plugins/scaffolder/src/components/fields/index.ts @@ -13,4 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { RepoUrlPicker } from './RepoUrlPicker'; +export * from './RepoUrlPicker'; From 0c5318df7d2b960f3e1635393802cfba9971f409 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 11:59:53 +0100 Subject: [PATCH 58/66] Update sidebar Signed-off-by: Johan Haals --- microsite/sidebars.json | 49 ++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 8b5c0e5f0c..4510f254e1 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -34,7 +34,10 @@ ] } ], - "CLI": ["cli/index", "cli/commands"], + "CLI": [ + "cli/index", + "cli/commands" + ], "Core Features": [ { "type": "subcategory", @@ -69,10 +72,10 @@ "features/software-templates/software-templates-index", "features/software-templates/installation", "features/software-templates/adding-templates", - "features/software-templates/extending/extending-index", - "features/software-templates/extending/extending-templater", - "features/software-templates/extending/extending-publisher", - "features/software-templates/extending/extending-preparer" + "features/software-templates/writing-templates", + "featires/software-templates/builtin-actions", + "features/software-templates/writing-custom-actions", + "features/software-templates/templates-legacy" ] }, { @@ -105,17 +108,23 @@ { "type": "subcategory", "label": "GitHub", - "ids": ["integrations/github/org"] + "ids": [ + "integrations/github/org" + ] }, { "type": "subcategory", "label": "LDAP", - "ids": ["integrations/ldap/org"] + "ids": [ + "integrations/ldap/org" + ] }, { "type": "subcategory", "label": "Google Analytics", - "ids": ["integrations/google-analytics/installation"] + "ids": [ + "integrations/google-analytics/installation" + ] } ], "Plugins": [ @@ -138,7 +147,9 @@ { "type": "subcategory", "label": "Testing", - "ids": ["plugins/testing"] + "ids": [ + "plugins/testing" + ] }, { "type": "subcategory", @@ -165,7 +176,6 @@ "auth/glossary", "auth/auth-backend-classes" ], - "Designing for Backstage": [ "dls/design", "dls/contributing-to-storybook", @@ -185,7 +195,9 @@ { "type": "subcategory", "label": "Backend APIs", - "ids": ["api/backend"] + "ids": [ + "api/backend" + ] } ], "Tutorials": [ @@ -207,8 +219,15 @@ "architecture-decisions/adrs-adr010", "architecture-decisions/adrs-adr011" ], - "Support": ["support/support", "support/project-structure"], - "Glossary": ["glossary"], - "FAQ": ["FAQ"] + "Support": [ + "support/support", + "support/project-structure" + ], + "Glossary": [ + "glossary" + ], + "FAQ": [ + "FAQ" + ] } -} +} \ No newline at end of file From 043171a8e49122c38c227584faa9160876b741bb Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 12:00:24 +0100 Subject: [PATCH 59/66] Add docs page for writing custom actions Signed-off-by: Johan Haals --- docs/features/software-templates/writing-custom-actions.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index e69de29bb2..6416fa8344 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -0,0 +1,7 @@ +--- +id: writing-custom-actions +title: Writing Custom Actions +description: How to write your own actions +--- + +# TODO From cf938a5611bc4159fae871d048fe7dd6745fc976 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 13:25:16 +0100 Subject: [PATCH 60/66] Document creation of custom actions Co-authored-by: blam Signed-off-by: Johan Haals --- .../writing-custom-actions.md | 160 +++++++++++++++++- 1 file changed, 159 insertions(+), 1 deletion(-) diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 6416fa8344..74fdb46629 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -4,4 +4,162 @@ title: Writing Custom Actions description: How to write your own actions --- -# TODO +If you're wanting to extend the functionality of the Scaffolder, you can do so +by writing custom actions which can be used along side our +[built-in actions](./builtin-actions.md) + +### Writing your Custom Action + +Your custom action can live where you choose, but simplest is to include it +alongside your `backend` package in `packages/backend`. + +Let's create a simple action that adds a new file and some contents that are +passed as `input` to the function. + +In `packages/backend/src/actions/custom.ts` we can create a new action. + +```ts +import { createTemplateAction } from '../../createTemplateAction'; +import fs from 'fs-extra'; + +export const createNewFileAction = () => { + return createTemplateAction<{ contents: string; filename: string }>({ + id: 'mycompany:create-file', + schema: { + input: { + required: ['contents', 'filename'], + type: 'object', + properties: { + contents: { + type: 'string', + title: 'Contents', + description: 'The contents of the file', + }, + contents: { + type: 'string', + title: 'Filename', + description: 'The filename of the file that will be created', + }, + }, + }, + }, + async handler(ctx) { + await fs.outputFile( + `${ctx.workspacePath}/${ctx.input.filename}`, + ctx.input.content, + ); + }, + }); +}; +``` + +So let's break this down. The `createNewFileAction` is a function that returns a +`createTemplateAction`, and it's a good place to pass in dependencies which +close over the `TemplateAction`. Take a look at our +[built-in actions](https://github.com/backstage/backstage/blob/7f5716081f45a41dc8a4246134b50c893e15c5e1/../plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts) +for reference. + +We set the type generic to `{ contents: string, filename: string}` which is +there to set the type on the handler `ctx` `inputs` property so we get good +typechecking. This could be generated from the next part of this guide, the +`input` schema, but it's not supported right now. Feel free to contribute ๐Ÿš€ ๐Ÿ‘. + +The `createTemplateAction` takes an object which specifies the following: + +- `id` - a unique ID for your custom action. We encourage you to namespace these + in someway so they wont collide with future built-in actions that we may ship + with the `scaffolder-backend` plugin. +- `schema.input` - A JSON schema for input values to your function +- `schema.output` - A JSON schema for values which are outputed from the + function using `ctx.output` +- `handler` the actual code pwhich is run part of the action, with a context. + +#### The context object + +When the action `handler` is called, we provide you a `context` as the only +argument. It looks like the following: + +- `ctx.baseUrl` - a string where the template is located +- `ctx.logger` - a winston logger for additional logging inside your action +- `ctx.logStream` - a stream version of the logger if needed +- `ctx.workspacePath` - a string of the working directory of the template run +- `ctx.input` - an object which should match the JSON schema provided in the + `schema.input` part of the action definition +- `ctx.output` - a function which you can call to set outputs that match the + JSON schema in `schema.output` for ex. `ctx.output('downloadUrl', something)` +- `createTemporaryDirectory` a function to call to give you a temporary + directory somewhere on the runner so you can store some files there rather + than polluting the `workspacePath` + +### Registering Custom Actions + +Once you have your Custom Action ready for usage with the scaffolder, you'll +need to pass this into the `scaffolder-backend` `createRouter` function. You +should have something similar to the below in +`packages/backend/src/plugins/scaffolder.ts` + +```ts +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, +}); +``` + +There's another property you can pass here, which is an array of `actions` which +will set the available actions that the scaffolder has access to. + +```ts +const actions = [createNewFileAction()]; +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, + actions, +}); +``` + +**NOTE** - the actions array will replace the built-in actions too, so if you +want to have those as well as your new one, you'll need to do the following: + +```ts + +import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend`; + +const builtInActions = createBuiltinActions({ + dockerClient, + integrations, + catalogClient, + templaters, + reader, +}); + +const actions = [...builtInActions, createNewFileAction()]; + +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, + actions, +}); +``` + +Have fun! ๐Ÿš€ From 7cef8640ba05edbb227eb3a30705e0cbef998b5c Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 13:44:03 +0100 Subject: [PATCH 61/66] chore: add linebreak Signed-off-by: Johan Haals --- microsite/sidebars.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 4510f254e1..e0d52bc771 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -230,4 +230,4 @@ "FAQ" ] } -} \ No newline at end of file +} From d0af57563d7db464699ea193df10861ca82bb5ba Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 13:48:39 +0100 Subject: [PATCH 62/66] Format with correct prettier config Signed-off-by: Johan Haals --- microsite/sidebars.json | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index e0d52bc771..83bfe55dfc 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -34,10 +34,7 @@ ] } ], - "CLI": [ - "cli/index", - "cli/commands" - ], + "CLI": ["cli/index", "cli/commands"], "Core Features": [ { "type": "subcategory", @@ -108,23 +105,17 @@ { "type": "subcategory", "label": "GitHub", - "ids": [ - "integrations/github/org" - ] + "ids": ["integrations/github/org"] }, { "type": "subcategory", "label": "LDAP", - "ids": [ - "integrations/ldap/org" - ] + "ids": ["integrations/ldap/org"] }, { "type": "subcategory", "label": "Google Analytics", - "ids": [ - "integrations/google-analytics/installation" - ] + "ids": ["integrations/google-analytics/installation"] } ], "Plugins": [ @@ -147,9 +138,7 @@ { "type": "subcategory", "label": "Testing", - "ids": [ - "plugins/testing" - ] + "ids": ["plugins/testing"] }, { "type": "subcategory", @@ -195,9 +184,7 @@ { "type": "subcategory", "label": "Backend APIs", - "ids": [ - "api/backend" - ] + "ids": ["api/backend"] } ], "Tutorials": [ @@ -219,15 +206,8 @@ "architecture-decisions/adrs-adr010", "architecture-decisions/adrs-adr011" ], - "Support": [ - "support/support", - "support/project-structure" - ], - "Glossary": [ - "glossary" - ], - "FAQ": [ - "FAQ" - ] + "Support": ["support/support", "support/project-structure"], + "Glossary": ["glossary"], + "FAQ": ["FAQ"] } } From 8c1adc2404307d0a7bc4363c74e949f3019c1bd7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 14:18:02 +0100 Subject: [PATCH 63/66] Reset mock after each test Co-authored-by: Patrik Oldsberg Signed-off-by: Johan Haals --- .../scaffolder/actions/builtin/fetch/cookiecutter.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 923e22d0a5..7d26bcb20a 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -57,8 +57,6 @@ describe('fetch:cookiecutter', () => { createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), }; - mock({ [`${mockContext.workspacePath}/result`]: {} }); - const mockReader: UrlReader = { read: jest.fn(), readTree: jest.fn(), @@ -75,9 +73,14 @@ describe('fetch:cookiecutter', () => { templaters.register('cookiecutter', cookiecutterTemplater); beforeEach(() => { + mock({ [`${mockContext.workspacePath}/result`]: {} }); jest.restoreAllMocks(); }); + afterEach(() => { + mock.restore(); + }); + it('should call fetchContents with the correct values', async () => { await action.handler(mockContext); From 2a80b93703041a9b20a34e186c9e477898a151b7 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 14:31:53 +0100 Subject: [PATCH 64/66] Fix sidebar typos Signed-off-by: Johan Haals --- microsite/sidebars.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 83bfe55dfc..65fd5b7d15 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -70,9 +70,9 @@ "features/software-templates/installation", "features/software-templates/adding-templates", "features/software-templates/writing-templates", - "featires/software-templates/builtin-actions", + "features/software-templates/builtin-actions", "features/software-templates/writing-custom-actions", - "features/software-templates/templates-legacy" + "features/software-templates/template-legacy" ] }, { From f123395f82b02343943263e579db2bd08fd9941d Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 14:34:21 +0100 Subject: [PATCH 65/66] Fix typos Signed-off-by: Johan Haals --- .../software-templates/writing-custom-actions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md index 74fdb46629..36d51decb5 100644 --- a/docs/features/software-templates/writing-custom-actions.md +++ b/docs/features/software-templates/writing-custom-actions.md @@ -60,9 +60,9 @@ close over the `TemplateAction`. Take a look at our for reference. We set the type generic to `{ contents: string, filename: string}` which is -there to set the type on the handler `ctx` `inputs` property so we get good -typechecking. This could be generated from the next part of this guide, the -`input` schema, but it's not supported right now. Feel free to contribute ๐Ÿš€ ๐Ÿ‘. +there to set the type on the handler `ctx` `inputs` property so we get good type +checking. This could be generated from the next part of this guide, the `input` +schema, but it's not supported right now. Feel free to contribute ๐Ÿš€ ๐Ÿ‘. The `createTemplateAction` takes an object which specifies the following: @@ -70,9 +70,9 @@ The `createTemplateAction` takes an object which specifies the following: in someway so they wont collide with future built-in actions that we may ship with the `scaffolder-backend` plugin. - `schema.input` - A JSON schema for input values to your function -- `schema.output` - A JSON schema for values which are outputed from the +- `schema.output` - A JSON schema for values which are outputted from the function using `ctx.output` -- `handler` the actual code pwhich is run part of the action, with a context. +- `handler` the actual code which is run part of the action, with a context. #### The context object From 8a01a6085950f1e20671c523aff544356ca19782 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 1 Mar 2021 15:56:49 +0100 Subject: [PATCH 66/66] Fix descriptor format docs Signed-off-by: Johan Haals --- .../software-catalog/descriptor-format.md | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 136f9d5cc2..17eaf80832 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -656,34 +656,6 @@ You can find out more about the `parameters` key You can find out more about the `steps` key [here](../software-templates/writing-templates.md) -## Kind: APIis `JSONSchema` with some extra goodies for styling what it might - -look like in the frontend. For these steps we rely very heavily on this library: -https://github.com/rjsf-team/react-jsonschema-form. They have some great docs -too here: https://react-jsonschema-form.readthedocs.io/ and a playground where -you can play around with some examples here -https://rjsf-team.github.io/react-jsonschema-form - -For example here is a simple parameters step - -```yaml - - title: Fill in some steps - required: - - name - properties: - name: - title: Name - type: string - description: Unique name of the component - ui:autofocus: true - ui:options: - rows: 5 -``` - -You can find out more [here](../software-templates/writing-templates.md) - -### `spec.steps` [optional] - ## Kind: API Describes the following entity kind: