diff --git a/.changeset/pink-ads-shave.md b/.changeset/pink-ads-shave.md new file mode 100644 index 0000000000..50d0ddef22 --- /dev/null +++ b/.changeset/pink-ads-shave.md @@ -0,0 +1,37 @@ +--- +'@backstage/plugin-scaffolder': minor +'@backstage/plugin-scaffolder-backend': minor +--- + +The scaffolder has been updated to support the new `v1beta2` template schema which allows for custom template actions! + +See documentation for more information how to create and register new template actions. + +**Breaking changes** + +The backend scaffolder plugin now needs a `UrlReader` which can be pulled from the PluginEnvironment. + +The following change is required in `backend/src/plugins/scaffolder.ts` + +```diff + export default async function createPlugin({ + logger, + config, + database, ++ reader, + }: PluginEnvironment): Promise { + + // omitted code + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, ++ reader, + }); +``` diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 6658dfc055..17eaf80832 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -522,49 +522,90 @@ consumed by the component, e.g. `artist-api`. This field is optional. ## Kind: Template -Describes the following entity kind: +The following describes the following entity kind: -| Field | Value | -| ------------ | ----------------------- | -| `apiVersion` | `backstage.io/v1alpha1` | -| `kind` | `Template` | +| Field | Value | +| ------------ | ---------------------- | +| `apiVersion` | `backstage.io/v1beta2` | +| `kind` | `Template` | -A Template describes a skeleton for use with the Scaffolder. It is used for -describing what templating library is supported, and also for documenting the -variables that the template requires using -[JSON Forms Schema](https://jsonforms.io/). +If you're looking for docs on `v1alpha1` you can find them +[here](../software-templates/legacy.md) + +A template definition describes both the parameters that are rendered in the +frontend part of the scaffolding wizard, and the steps that are executed when +scaffolding that component. Descriptor files for this kind may look as follows. ```yaml -apiVersion: backstage.io/v1alpha1 +apiVersion: backstage.io/v1beta2 kind: Template +# some metadata about the template itself metadata: - name: react-ssr-template - title: React SSR Template - description: - Next.js application skeleton for creating isomorphic web applications. - tags: - - recommended - - react + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo spec: - owner: web@example.com - templater: cookiecutter - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' ``` In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) @@ -572,7 +613,7 @@ shape, this kind has the following structure. ### `apiVersion` and `kind` [required] -Exactly equal to `backstage.io/v1alpha1` and `Template`, respectively. +Exactly equal to `backstage.io/v1beta2` and `Template`, respectively. ### `metadata.title` [required] @@ -605,31 +646,15 @@ The current set of well-known and common values for this field is: - `website` - a website - `library` - a software library, such as an npm module or a Java library -### `spec.templater` [required] +### `spec.parameters` [required] -The templating library that is supported by the template skeleton as a string, -e.g `cookiecutter`. +You can find out more about the `parameters` key +[here](../software-templates/writing-templates.md) -Different skeletons will use different templating syntax, so it's common that -the template will need to be run with a particular piece of software. +### `spec.steps` [optional] -This key will be used to identify the correct templater which is registered into -the `TemplatersBuilder`. - -The values which are available by default are: - -- `cookiecutter` - [cookiecutter](https://github.com/cookiecutter/cookiecutter). - -### `spec.path` [optional] - -The string location where the templater should be run if it is not on the same -level as the `template.yaml` definition, e.g. `./cookiecutter/skeleton`. - -This will set the `cwd` when running the templater to the folder path that you -specify relative to the `template.yaml` definition. - -This is also particularly useful when you have multiple template definitions in -the same repository but only a single `template.yaml` registered in backstage. +You can find out more about the `steps` key +[here](../software-templates/writing-templates.md) ## Kind: API diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index e2b69d6767..ecb7d11be9 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -4,49 +4,80 @@ title: Adding your own Templates description: Documentation on Adding your own Templates --- -Templates are stored in the **Service Catalog** under a kind `Template`. The +Templates are stored in the **Software Catalog** under a kind `Template`. The minimum that is needed to define a template is a `template.yaml` file, but it would be good to also have some files in there that can be templated in. A simple `template.yaml` definition might look something like this: ```yaml -apiVersion: backstage.io/v1alpha1 +apiVersion: backstage.io/v1beta2 kind: Template +# some metadata about the template itself metadata: - # unique name per namespace for the template - name: react-ssr-template - # title of the template - title: React SSR Template - # a description of the template - description: - Next.js application skeleton for creating isomorphic web applications. - # some tags to display in the frontend - tags: - - recommended - - react + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo spec: - # which templater key to use in the templaters builder - templater: cookiecutter - # what does this template create - type: website - # if the template is not in the current directory where this definition is kept then specify - path: './template' - # the schema for the form which is displayed in the frontend. - # should follow JSON schema for forms: https://jsonforms.io/ - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' ``` [Template Entity](../software-catalog/descriptor-format.md#kind-template) @@ -55,14 +86,6 @@ contains more information about the required fields. Once we have a `template.yaml` ready, we can then add it to the service catalog for use by the scaffolder. -_NOTE_: When the `publish` step is completed, it is currently assumed by the -scaffolder that the final repository should contain a `catalog-info.yaml` in -order to register this with the Catalog in Backstage. - -Currently the catalog supports loading definitions from GitHub + Local Files. To -load from other places, not only will there need to be another preparer, but the -support to load the location will also need to be added to the Catalog. - You can add the template files to the catalog through [static location configuration](../software-catalog/configuration.md#static-location-configuration), for example: @@ -76,52 +99,8 @@ catalog: - allow: [Template] ``` -Templates can also be added by posting the to the catalog directly. Note that if -you're doing this, you need to configure the catalog to allow template entities -to be ingested from any source, for example: +Or you can add the template using the `catalog-import` plugin, which unless +configured differently should be running on `/catalog-import`. -```yaml -catalog: - rules: - - allow: [Component, API, Template] -``` - -For loading from a file, the following command should work when the backend is -running: - -```sh -curl \ - --location \ - --request POST 'localhost:7000/api/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"file\", \"target\": \"${YOUR PATH HERE}/template.yaml\"}" -``` - -If loading from a Git location, you can run the following - -```sh -curl \ - --location \ - --request POST 'localhost:7000/api/catalog/locations' \ - --header 'Content-Type: application/json' \ - --data-raw "{\"type\": \"github\", \"target\": \"https://${GITHUB URL}/${YOUR GITHUB ORG/REPO}/blob/master/${PATH TO FOLDER}/template.yaml\"}" -``` - -This should then have been added the catalog, and be listed under the create -page at http://localhost:3000/create. - -The `type` field which is chosen in the request to add the `template.yaml` to -the Service Catalog here, will become the `PreparerKey` which will be used to -select the `Preparer` when creating a job. - -### Adding form values in the Scaffolder Wizard - -The `spec.schema` property in the -[Template Entity](../software-catalog/descriptor-format.md#kind-template) is a -`yaml` version of the JSON Form Schema standard. - -Here you can define the key/values and then the wizard will convert this to a -form for the user to fill in when your template is selected. - -You can find out much more about the standard and how to use it here: -https://jsonforms.io +For information about writing your own templates, you can check out the docs +[here](./writing-templates.md) diff --git a/docs/features/software-templates/builtin-actions.md b/docs/features/software-templates/builtin-actions.md new file mode 100644 index 0000000000..e6886b1f04 --- /dev/null +++ b/docs/features/software-templates/builtin-actions.md @@ -0,0 +1,115 @@ +--- +id: builtin-actions +title: Builtin actions +description: Documentation describing the built-in template actions. +--- + +# Built-in Actions + +This is the list of built-in template actions + +## `fetch:plain` + +Downloads content and places it in the workspacePath or optionally in a +subdirectory specified by the `targetPath` input option. + +input: + +- `url` (required) Relative path or absolute URL pointing to the directory tree + to fetch. +- `targetPath` Target path within the working directory to download the contents + to. + +output: nothing + +## `fetch:cookiecutter` + +Downloads template from `url` and templates with cookiecutter + +input: + +- `url` (required) Relative path or absolute URL pointing to the directory tree + to fetch. +- `targetPath` Target path within the working directory to download the contents + to. +- `values` Values to pass on to cookiecutter for templating. + +output: nothing + +## `catalog:register` + +Registers entity in the software catalog. + +input: + +- `catalogInfoUrl` (required) An absolute URL pointing to the catalog info file + location. + +output: + +- `repoContentsUrl` An absolute URL pointing to the root of a repository + directory tree. +- `catalogInfoPath` A relative path from the repo root pointing to the catalog + info file, defaults to /catalog-info.yaml + +## `publish:github` + +Initializes a git repository of contents in workspacePath and publishes to +GitHub. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public' or 'internal'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:gitlab` + +Initializes a git repository of contents in workspacePath and publishes to +GitLab. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public' or 'internal'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:bitbucket` + +Initializes a git repository of contents in workspacePath and publishes to +Bitbucket. + +input: + +- `repoUrl` (required) Repository location +- `repoVisibility` (optional, default: 'private') Possible values: 'private', + 'public'. + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository + +## `publish:azure` + +Initializes a git repository of contents in workspacePath and publishes to +Azure. + +input: + +- `repoUrl` (required) Repository location + +output: + +- `remoteUrl` A URL to the repository with the provider +- `repoContentsUrl` A URL to the root of the repository diff --git a/docs/features/software-templates/extending/create-your-own-preparer.md b/docs/features/software-templates/extending/create-your-own-preparer.md deleted file mode 100644 index 5419748c3a..0000000000 --- a/docs/features/software-templates/extending/create-your-own-preparer.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -id: extending-preparer -title: Create your own Preparer -description: Documentation on Creating your own Preparer ---- - -Preparers are responsible for reading the location of the definition of a -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) and -making a temporary folder with the contents of the selected skeleton. - -Currently, we provide two different providers that can parse two different -location protocols: - -- `file://` -- `github://` - -These two are added to the `PreparersBuilder` and then passed into the -`createRouter` function of the `@backstage/plugin-scaffolder-backend`. - -A full example backend can be found in -[`scaffolder.ts`](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - createRouter, - FilePreparer, - GithubPreparer, - Preparers, -} from '@backstage/plugin-scaffolder-backend'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const preparers = new Preparers(); - - const filePreparer = new FilePreparer(); - const githubPreparer = new GithubPreparer(); - - preparers.register('file', filePreparer); - preparers.register('github', githubPreparer); - - return await createRouter({ - preparers, - templaters, - logger, - }); -} -``` - -As you can see in the above code, a `PreparerBuilder` is created, and then two -of the `preparers` are registered with the different protocols that they accept. - -The `protocol` is set on the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -when added to the service catalog. You can see more about this `PreparerKey` -here in [Register your own template](../adding-templates.md) - -**note:** Currently the catalog supports loading definitions from GitHub + Local -Files, which translate into the two `PreparerKeys`: `file` and `github`. To load -from other places, not only will there need to be another preparer, but the -support to load the location will also need to be added to the Catalog. - -### Creating your own Preparer to add to the `PreparerBuilder` - -All preparers need to implement the `PreparerBase` type. - -That type looks like the following: - -```ts -export type PreparerBase = { - prepare( - template: TemplateEntityV1alpha1, - opts: { logger: Logger }, - ): Promise; -}; -``` - -The `prepare` function will be given the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -along with the source of where the `template.yaml` was loaded from under the -`metedata.annotations.managed-by-location` property. - -Now it's up to you to implement a function which can go and fetch the skeleton -and put the contents into a temporary directory and return that directory path. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts - -### Registering your own Preparer - -You can register the preparer that you have created with the `PreparerBuilder` -by using the `PreparerKey` from the Catalog, for example like this: - -```ts -const preparers = new Preparers(); -preparers.register('gcs', new GoogleCloudStoragePreparer()); -``` - -And then pass this into the `createRouter` function. diff --git a/docs/features/software-templates/extending/create-your-own-publisher.md b/docs/features/software-templates/extending/create-your-own-publisher.md deleted file mode 100644 index 5e2809e006..0000000000 --- a/docs/features/software-templates/extending/create-your-own-publisher.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -id: extending-publisher -title: Create your own Publisher -description: Documentation on Creating your own Publisher ---- - -Publishers are responsible for pushing and storing the templated skeleton after -the values have been templated by the `Templater`. See -[Create your own templater](./create-your-own-templater.md) for more info. - -They receive a directory or location where the templater has successfully run -and is now ready to store somewhere. They also are given some other options -which are sent from the frontend, such as the `storePath` which is a string of -where the frontend thinks we should save this templated folder. - -Currently we provide the following `publishers`: - -- `github` - -This publisher is passed through to the `createRouter` function of the -`@backstage/plugin-scaffolder-backend`. Currently, only one publisher is -supported, but PR's are always welcome. - -An full example backend can be found -[here](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - createRouter, - GithubPublisher, -} from '@backstage/plugin-scaffolder-backend'; -import { Octokit } from '@octokit/rest'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const githubClient = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const publisher = new GithubPublisher({ client: githubClient }); - - return await createRouter({ - publisher, - logger, - }); -} -``` - -The publisher will always be called with the location from the selected -`Preparer`. - -### Create your own Publisher and register it with the Scaffolder - -All `publishers` need to implement the `PublisherBase` type. - -That type looks like the following: - -```ts -export type PublisherBase = { - publish(opts: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - directory: string; - }): Promise<{ remoteUrl: string }>; -}; -``` - -The `publisher` function will be called with an `options` object which contains -the following: - -- `entity` - the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is currently being scaffolded -- `values` - a json object which will resemble the `spec.schema` from the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is defined here under spec.schema`. More info can be found here - [Register your own template](../adding-templates.md#adding-form-values-in-the-scaffolder-wizard) -- `directory` - a string containing the returned path from the `templater`. See - more information here in - [Create your own templater](./create-your-own-templater.md) - -Now it's up to you to implement the `publish` function and return -`{ remoteUrl: string }` which can be used to identify the finished product. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts - -### Registering your own Publisher - -Currently, we only support one `publisher` (PR's welcome), but you can register -any single `publisher` with the `createRouter`. - -```ts -return await createRouter({ - publisher: new MyGitlabPublisher(), -}); -``` diff --git a/docs/features/software-templates/extending/create-your-own-templater.md b/docs/features/software-templates/extending/create-your-own-templater.md deleted file mode 100644 index 28769d77e1..0000000000 --- a/docs/features/software-templates/extending/create-your-own-templater.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -id: extending-templater -title: Creating your own Templater -description: Documentation on Creating your own Templater ---- - -Templaters are responsible for taking the directory path for the skeleton -returned by the preparers, and then executing the templating command on top of -the file and returning the completed template path. This may or may not be the -same directory as the input directory. - -They also receive additional values from the frontend, which can be used to -interpolate into the skeleton files. - -Currently we provide the following templaters: - -- `cookiecutter` - -This templater is added to the `TemplaterBuilder` and then passed into the -`createRouter` function of the `@backstage/plugin-scaffolder-backend` - -An full example backend can be found -[here](https://github.com/backstage/backstage/blob/d91c10f654475a60829fa33a5c81018e517a319a/packages/backend/src/plugins/scaffolder.ts), -but it looks something like the following - -```ts -import { - CookieCutter, - createRouter, - Templaters, -} from '@backstage/plugin-scaffolder-backend'; -import type { PluginEnvironment } from '../types'; - -export default async function createPlugin({ logger }: PluginEnvironment) { - const templaters = new Templaters(); - const cookiecutterTemplater = new CookieCutter(); - templaters.register('cookiecutter', cookiecutterTemplater); - - return await createRouter({ - templaters, - }); -} -``` - -As you can see in the above code a `TemplaterBuilder` is created and the default -`cookiecutter` `templater` is registered under the key `cookicutter`. - -This `TemplaterKey` is used to select the correct templater from the -`spec.templater` in the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template). - -If you wish to add a new templater, you'll need to register it with the -`TemplaterBuilder`. - -### Creating your own Templater to add to the `TemplaterBuilder` - -All templaters need to implement the `TemplaterBase` type. - -That type looks like the following: - -```ts -export type TemplaterRunOptions = { - directory: string; - values: TemplaterValues; - logStream?: Writable; - dockerClient: Docker; -}; - -export type TemplaterBase = { - run(opts: TemplaterRunOptions): Promise; -}; -``` - -The `run` function will be given a `TemplaterRunOptions` object which is as -follows: - -- `directory`- the skeleton directory returned from the `Preparer`, more info at - [Create your own preparer](./create-your-own-preparer.md). -- `values` - a json object which will resemble the `spec.schema` from the - [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - which is defined here under spec.schema`. More info can be found here - [Register your own template](../adding-templates.md#adding-form-values-in-the-scaffolder-wizard) -- `logStream` - a stream that you can write to for displaying in the frontend. -- `dockerClient` - a [dockerode](https://github.com/apocas/dockerode) client to - be able to run docker containers. - -_note_ Currently the templaters that we provide are basically Docker action -containers that are run on top of the skeleton folder. This keeps dependencies -to a minimum for running Backstage scaffolder, but you don't _have_ to use -Docker. You can `pip install cookiecutter` to run it locally in your backend. -You could create your own templater that spins up an EC2 instance and downloads -the folder and does everything using an AMI if you want. It's entirely up to -you! - -Now it's up to you to implement the `run` function, and then return a -`TemplaterRunResult` which is `{ resultDir: string }`. - -Some good examples exist here: - -- https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts - -### Registering your own Templater - -If you try to process a -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) -with a new `spec.templater` value, you'll need to register that with the -`TemplaterBuilder`. - -For example let's say you have the following -[Template Entity](../../software-catalog/descriptor-format.md#kind-template): - -```yaml -apiVersion: backstage.io/v1alpha1 -kind: Template -metadata: - name: react-ssr-template - title: React SSR Template - description: - Next.js application skeleton for creating isomorphic web applications. - tags: - - recommended - - react -spec: - owner: web@example.com - templater: handlebars - type: website - path: '.' - schema: - required: - - component_id - - description - properties: - component_id: - title: Name - type: string - description: Unique name of the component - description: - title: Description - type: string - description: Description of the component -``` - -You see that the `spec.templater` is set as `handlebars`, so you'll need to -register this with the `TemplaterBuilder` like so: - -```ts -const templaters = new Templaters(); -templaters.register('handlebars', new HandlebarsTemplater()); -``` - -And then pass this into the `createRouter` function. diff --git a/docs/features/software-templates/extending/index.md b/docs/features/software-templates/extending/index.md deleted file mode 100644 index aaf42d6b12..0000000000 --- a/docs/features/software-templates/extending/index.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -id: extending-index -title: Extending the Scaffolder ---- - -Welcome. Take a seat. You're at the Scaffolder Documentation. - -So, you want to create stuff inside your company from some prebaked templates? -You're at the right place. - -This guide is going to take you through how the Scaffolder in Backstage works. -We'll dive into some jargon and run through what's going on in the backend to be -able to create these templates. There's also more guides that you might find -useful at the bottom of this document. At its core, there are 3 simple stages. - -1. Pick a skeleton -2. Template some variables into the skeleton -3. Send the templated skeleton somewhere - -These three steps are translated to the following stages under the hood in the -scaffolder that you will need to know: - -1. Prepare -2. Template -3. Publish - -Each of these steps can be configured for your own use case, but we provide some -sensible defaults, too. - -Let's dive a little deeper into these phases. - -### Glossary and Jargon - -**Preparer** - The preparer is responsible for fetching the skeleton code and -placing it into a directory and then will return that directory. It is -registered with the `Preparers` with a particular type, which is then used in -the router to pick the correct `Preparer` to run for the `Template` entity. - -**Templater** - The templater is responsible for actually running the chosen -templater on top of the previously returned temporary directory from the -**Preparer**. We advise making these Docker containers as it can keep all -dependencies--for example Cookiecutter--self contained and not a dependency on -the host machine. - -**Publisher** - The publisher is responsible for taking the finished directory, -and publishing it to a remote registry. This could be a Git repository or -something similar. Right now, the scaffolder only supports one publishing method -for the entire lifecycle, but it could be configured from the frontend and -passed through to the scaffolder backend. - -### How it works - -Most of the heavy lifting is done in the -[router.ts](https://github.com/backstage/backstage/blob/master/plugins/scaffolder-backend/src/service/router.ts#L93) -file in the `scaffolder-backend` plugin. - -There are two routes defined in the router: `POST /v1/jobs` and -`GET /v1/job/:jobId` - -To create a scaffolding job, a JSON object containing the -[Template Entity](../../software-catalog/descriptor-format.md#kind-template) + -additional templating values must be posted as the post body. - -```js -{ - "template": { - "apiVersion": "backstage/v1alpha1", - "kind": "Template", - // more stuff here - }, - "values": { - "component_id": "test", - "description": "somethingelse" - } -} -``` - -The values should represent something that is valid with the `schema` part of -the [Template Entity](../../software-catalog/descriptor-format.md#kind-template) - -Once that has been posted, a job will be setup with different stages, and the -job processor will complete each stage before moving onto the next stage, whilst -collecting logs and mutating the running job. - -Here's some further reading that you might find useful: - -- [Adding your own Template](../adding-templates.md) -- [Creating your own Templater](./create-your-own-templater.md) -- [Creating your own Publisher](./create-your-own-publisher.md) -- [Creating your own Preparer](./create-your-own-preparer.md) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 9aa1d3bc8f..fd51bffa5b 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -99,6 +99,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -124,6 +125,7 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + reader, }); } ``` @@ -136,10 +138,11 @@ import scaffolder from './plugins/scaffolder'; const scaffolderEnv = useHotMemoize(module, () => createEnv('scaffolder')); -const service = createServiceBuilder(module) - .loadConfig(configReader) - /** several different routers */ - .addRouter('/scaffolder', await scaffolder(scaffolderEnv)); +const apiRouter = Router(); +/* several router .use calls */ + +/* add this line */ +apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); ``` ### Adding Templates @@ -170,14 +173,10 @@ catalog: ### Runtime Dependencies / Configuration -For the scaffolder backend plugin to function, it needs a GitHub access token, -and access to a running Docker daemon. You can create a GitHub access token -[here](https://github.com/settings/tokens/new), select `repo` scope only. Full -docs on creating private GitHub access tokens is available -[here](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). -Note that the need for private GitHub access tokens will be replaced with GitHub -Apps integration further down the line by using the existing `integrations` -config. +For the scaffolder backend plugin to function, you'll need to setup the +integrations config in your `app-config.yaml`. + +You can find help for different providers below. > Note: Some of this configuration may already be set up as part of your > `app-config.yaml`. We're moving away from the duplicated config for @@ -220,6 +219,32 @@ integrations: $env: GITLAB_TOKEN ``` +#### BitBucket + +For Bitbucket there are two authentication methods supported. Either `token` or +a combination of `appPassword` and `username`. It looks like either of the +following: + +```yaml +integrations: + bitbucket: + - host: bitbucket.org + token: + $env: BITBUCKET_TOKEN +``` + +or + +```yaml +integrations: + bitbucket: + - host: bitbucket.org + appPassword: + $env: BITBUCKET_APP_PASSWORD + username: + $env: BITBUCKET_USERNAME +``` + #### Azure DevOps For Azure DevOps we support both the preparer and publisher stage with the diff --git a/docs/features/software-templates/legacy.md b/docs/features/software-templates/legacy.md new file mode 100644 index 0000000000..d70773e251 --- /dev/null +++ b/docs/features/software-templates/legacy.md @@ -0,0 +1,118 @@ +--- +id: template-legacy +title: Writing Templates (Legacy) +description: + Old documentation describing the backstage.io/v1alpha1 format of the Template + Schema +--- + +## Kind: Template + +Describes the following entity kind: + +| Field | Value | +| ------------ | ----------------------- | +| `apiVersion` | `backstage.io/v1alpha1` | +| `kind` | `Template` | + +A Template describes a skeleton for use with the Scaffolder. It is used for +describing what templating library is supported, and also for documenting the +variables that the template requires using +[JSON Forms Schema](https://jsonforms.io/). + +Descriptor files for this kind may look as follows. + +```yaml +apiVersion: backstage.io/v1alpha1 +kind: Template +metadata: + name: react-ssr-template + title: React SSR Template + description: + Next.js application skeleton for creating isomorphic web applications. + tags: + - recommended + - react +spec: + owner: web@example.com + templater: cookiecutter + type: website + path: '.' + schema: + required: + - component_id + - description + properties: + component_id: + title: Name + type: string + description: Unique name of the component + description: + title: Description + type: string + description: Description of the component +``` + +In addition to the [common envelope metadata](#common-to-all-kinds-the-metadata) +shape, this kind has the following structure. + +### `apiVersion` and `kind` [required] + +Exactly equal to `backstage.io/v1alpha1` and `Template`, respectively. + +### `metadata.title` [required] + +The nice display name for the template as a string, e.g. `React SSR Template`. +This field is required as is used to reference the template to the user instead +of the `metadata.name` field. + +### `metadata.tags` [optional] + +A list of strings that can be associated with the template, e.g. +`['recommended', 'react']`. + +This list will also be used in the frontend to display to the user so you can +potentially search and group templates by these tags. + +### `spec.type` [optional] + +The type of component as a string, e.g. `website`. This field is optional but +recommended. + +The software catalog accepts any type value, but an organization should take +great care to establish a proper taxonomy for these. Tools including Backstage +itself may read this field and behave differently depending on its value. For +example, a website type component may present tooling in the Backstage interface +that is specific to just websites. + +The current set of well-known and common values for this field is: + +- `service` - a backend service, typically exposing an API +- `website` - a website +- `library` - a software library, such as an npm module or a Java library + +### `spec.templater` [required] + +The templating library that is supported by the template skeleton as a string, +e.g `cookiecutter`. + +Different skeletons will use different templating syntax, so it's common that +the template will need to be run with a particular piece of software. + +This key will be used to identify the correct templater which is registered into +the `TemplatersBuilder`. + +The values which are available by default are: + +- `cookiecutter` - [cookiecutter](https://github.com/cookiecutter/cookiecutter). + +### `spec.path` [optional] + +The string location where the templater should be run if it is not on the same +level as the `template.yaml` definition, e.g. `./cookiecutter/skeleton`. + +This will set the `cwd` when running the templater to the folder path that you +specify relative to the `template.yaml` definition. + +This is also particularly useful when you have multiple template definitions in +the same repository but only a single `template.yaml` registered in backstage. diff --git a/docs/features/software-templates/writing-custom-actions.md b/docs/features/software-templates/writing-custom-actions.md new file mode 100644 index 0000000000..36d51decb5 --- /dev/null +++ b/docs/features/software-templates/writing-custom-actions.md @@ -0,0 +1,165 @@ +--- +id: writing-custom-actions +title: Writing Custom Actions +description: How to write your own actions +--- + +If you're wanting to extend the functionality of the Scaffolder, you can do so +by writing custom actions which can be used along side our +[built-in actions](./builtin-actions.md) + +### Writing your Custom Action + +Your custom action can live where you choose, but simplest is to include it +alongside your `backend` package in `packages/backend`. + +Let's create a simple action that adds a new file and some contents that are +passed as `input` to the function. + +In `packages/backend/src/actions/custom.ts` we can create a new action. + +```ts +import { createTemplateAction } from '../../createTemplateAction'; +import fs from 'fs-extra'; + +export const createNewFileAction = () => { + return createTemplateAction<{ contents: string; filename: string }>({ + id: 'mycompany:create-file', + schema: { + input: { + required: ['contents', 'filename'], + type: 'object', + properties: { + contents: { + type: 'string', + title: 'Contents', + description: 'The contents of the file', + }, + contents: { + type: 'string', + title: 'Filename', + description: 'The filename of the file that will be created', + }, + }, + }, + }, + async handler(ctx) { + await fs.outputFile( + `${ctx.workspacePath}/${ctx.input.filename}`, + ctx.input.content, + ); + }, + }); +}; +``` + +So let's break this down. The `createNewFileAction` is a function that returns a +`createTemplateAction`, and it's a good place to pass in dependencies which +close over the `TemplateAction`. Take a look at our +[built-in actions](https://github.com/backstage/backstage/blob/7f5716081f45a41dc8a4246134b50c893e15c5e1/../plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts) +for reference. + +We set the type generic to `{ contents: string, filename: string}` which is +there to set the type on the handler `ctx` `inputs` property so we get good type +checking. This could be generated from the next part of this guide, the `input` +schema, but it's not supported right now. Feel free to contribute 🚀 👍. + +The `createTemplateAction` takes an object which specifies the following: + +- `id` - a unique ID for your custom action. We encourage you to namespace these + in someway so they wont collide with future built-in actions that we may ship + with the `scaffolder-backend` plugin. +- `schema.input` - A JSON schema for input values to your function +- `schema.output` - A JSON schema for values which are outputted from the + function using `ctx.output` +- `handler` the actual code which is run part of the action, with a context. + +#### The context object + +When the action `handler` is called, we provide you a `context` as the only +argument. It looks like the following: + +- `ctx.baseUrl` - a string where the template is located +- `ctx.logger` - a winston logger for additional logging inside your action +- `ctx.logStream` - a stream version of the logger if needed +- `ctx.workspacePath` - a string of the working directory of the template run +- `ctx.input` - an object which should match the JSON schema provided in the + `schema.input` part of the action definition +- `ctx.output` - a function which you can call to set outputs that match the + JSON schema in `schema.output` for ex. `ctx.output('downloadUrl', something)` +- `createTemporaryDirectory` a function to call to give you a temporary + directory somewhere on the runner so you can store some files there rather + than polluting the `workspacePath` + +### Registering Custom Actions + +Once you have your Custom Action ready for usage with the scaffolder, you'll +need to pass this into the `scaffolder-backend` `createRouter` function. You +should have something similar to the below in +`packages/backend/src/plugins/scaffolder.ts` + +```ts +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, +}); +``` + +There's another property you can pass here, which is an array of `actions` which +will set the available actions that the scaffolder has access to. + +```ts +const actions = [createNewFileAction()]; +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, + actions, +}); +``` + +**NOTE** - the actions array will replace the built-in actions too, so if you +want to have those as well as your new one, you'll need to do the following: + +```ts + +import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend`; + +const builtInActions = createBuiltinActions({ + dockerClient, + integrations, + catalogClient, + templaters, + reader, +}); + +const actions = [...builtInActions, createNewFileAction()]; + +return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, + database, + catalogClient, + reader, + actions, +}); +``` + +Have fun! 🚀 diff --git a/docs/features/software-templates/writing-templates.md b/docs/features/software-templates/writing-templates.md new file mode 100644 index 0000000000..d9dbb67ab2 --- /dev/null +++ b/docs/features/software-templates/writing-templates.md @@ -0,0 +1,286 @@ +--- +id: writing-templates +title: Writing Templates +description: Details around creating your own custom Software Templates +--- + +Templates are stored in the **Service Catalog** under a kind `Template`. You can +create your own templates with a small `yaml` definition which describes the +template and it's metadata, along with some input variables that your template +will need, and then a list of actions which are then executed by the scaffolding +service. + +Let's take a look at a simple example: + +```yaml +# Notice the v1beta2 version +apiVersion: backstage.io/v1beta2 +kind: Template +# some metadata about the template itself +metadata: + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo +spec: + owner: backstage/techdocs-core + type: service + + # these are the steps which are rendered in the frontend with the form input + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + # here's the steps that are executed in series in the scaffolder backend + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + input: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + input: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + input: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + input: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + # some outputs which are saved along with the job for use in the frontend + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' +``` + +Let's dive in an pick apart what each of these sections do and what they are. + +### `spec.parameters` - `FormStep | FormStep[]` + +These `parameters` are template variables which can be modified in the frontend +as a sequence. It can either be one `Step` if you just want one big list of +different fields in the frontend, or it can be broken up into multiple different +steps which would be rendered as different steps in the scaffolder plugin +frontend. + +Each `Step` is `JSONSchema` with some extra goodies for styling what it might +look like in the frontend. For these steps we rely very heavily on this library: +https://github.com/rjsf-team/react-jsonschema-form. They have some great docs +too here: https://react-jsonschema-form.readthedocs.io/ and a playground where +you can play around with some examples here +https://rjsf-team.github.io/react-jsonschema-form. + +There's another option for that library called `uiSchema` which we've taken +advantage of, and we've merged it with the existing `JSONSchema` that you +provide to the library. These are the little `ui:*` properties that you can see +in the step definitions. + +For example if we take the **simple** example from the playground it looks like +this: + +```json +// jsonSchema: +{ + "title": "A registration form", + "description": "A simple form example.", + "type": "object", + "required": [ + "firstName", + "lastName" + ], + "properties": { + "firstName": { + "type": "string", + "title": "First name", + "default": "Chuck" + }, + "lastName": { + "type": "string", + "title": "Last name" + }, + "telephone": { + "type": "string", + "title": "Telephone", + "minLength": 10 + } + } +} + +// uiSchema: +{ + "firstName": { + "ui:autofocus": true, + "ui:emptyValue": "", + "ui:autocomplete": "family-name" + }, + "lastName": { + "ui:emptyValue": "", + "ui:autocomplete": "given-name" + }, + "telephone": { + "ui:options": { + "inputType": "tel" + } + } +} +``` + +It would look something like the following in a template: + +```yaml +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: A registration form + description: A simple form example. + type: object + required: + - firstName + - lastName + properties: + firstName: + type: string + title: First name + default: Chuck + ui:autofocus: true + ui:emptyValue: '' + ui:autocomplete: family-name + lastName: + type: string + title: Last name + ui:emptyValue: '' + ui:autocomplete: given-name + telephone: + type: string + title: Telephone + minLength: 10 + ui:options: + inputType: tel +``` + +#### The Repository Picker + +So in order to make working with repository providers easier, we've built a +custom picker that can be used by overriding the `ui:field` option in the +`uiSchema` for a `string` field. Instead of displaying a text input block it +will render our custom component that we've built which makes it easy to select +a repository provider, and insert a project or owner, and repository name. + +You can see it in the above full example which is a separate step and it looks a +little like this: + +```yaml + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com +``` + +The `allowedHosts` part should be set to where you wish to enable this template +to publish to. And it can be any host that is listed in your `integrations` +config in `app-config.yaml`. + +The `RepoUrlPicker` is a custom field that we provide part of the +`plugin-scaffolder`. It's currently not possible to create your own fields yet, +but contributions are welcome! :) + +### `spec.steps` - `Action[]` + +The `steps` is an array of the things that you want to happen part of this +template. These follow the same standard format: + +```yaml +- id: fetch-base # A unique id for the step + name: Fetch Base # A title displayed in the frontend + action: fetch:cookiecutter # an action to call + input: # input that is passed as arguments to the action handler + url: ./template + values: + name: '{{ parameters.name }}' +``` + +By default we ship some built in actions that you can take a look at +[here](./builtin-actions.md), or you can create your own custom actions by +looking at the docs [here](./writing-custom-actions.md) + +### Outputs + +Each individual step can output some variables that can be used in the +scaffolder frontend for after the job is finished. This is useful for things +like linking to the entity that has been created with the backend, and also +linking to the created repository. + +The main two that are used are the following: + +```yaml +output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' # link to the remote repository + entityRef: '{{ steps.register.output.entityRef }}' # link to the entitiy that has been ingested to the catalog +``` + +### The templating syntax + +You might have noticed in the examples that there are `{{ }}`, and these are a +`handlebars` templates for linking and glueing all these different parts of +`yaml` together. All the form inputs from the `parameters` section, when passed +to the steps will be available by using the template syntax +`{{ parameters.something }}`. This is great for passing the values from the form +into different steps and reusing these input variables. + +As you can see above in the `Outputs` section, `actions` and `steps` can also +output things. So you can grab that output by using +`steps.$stepId.output.$property`. + +You can read more about all the `inputs` and `outputs` defined in the actions in +code part of the `JSONSchema` or you can read more about our built in ones +[here](./builtin-actions.md). diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 608d911957..1558730228 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -69,10 +69,10 @@ "features/software-templates/software-templates-index", "features/software-templates/installation", "features/software-templates/adding-templates", - "features/software-templates/extending/extending-index", - "features/software-templates/extending/extending-templater", - "features/software-templates/extending/extending-publisher", - "features/software-templates/extending/extending-preparer" + "features/software-templates/writing-templates", + "features/software-templates/builtin-actions", + "features/software-templates/writing-custom-actions", + "features/software-templates/template-legacy" ] }, { diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 1b97f735a0..fca3f79270 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -32,6 +32,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment): Promise { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -57,5 +58,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + reader, }); } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index 49cd8a9cda..a3416c5080 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -2,13 +2,10 @@ import { SingleHostDiscovery } from '@backstage/backend-common'; import { CatalogClient } from '@backstage/catalog-client'; import { CookieCutter, - - - - CreateReactAppTemplater, createRouter, + CreateReactAppTemplater, + createRouter, Preparers, Publishers, - Templaters } from '@backstage/plugin-scaffolder-backend'; import Docker from 'dockerode'; @@ -19,6 +16,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment): Promise { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -44,5 +42,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + reader }); } diff --git a/plugins/scaffolder-backend/sample-templates/local-templates.yaml b/plugins/scaffolder-backend/sample-templates/local-templates.yaml index 468fb72ece..b0e16f9dea 100644 --- a/plugins/scaffolder-backend/sample-templates/local-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/local-templates.yaml @@ -9,3 +9,4 @@ spec: - ./docs-template/template.yaml - ./react-ssr-template/template.yaml - ./springboot-grpc-template/template.yaml + - ./v1beta2-demo/template.yaml diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml new file mode 100644 index 0000000000..b1a4b0d91d --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml @@ -0,0 +1,68 @@ +apiVersion: backstage.io/v1beta2 +kind: Template +metadata: + name: v1beta2-demo + title: Test Action template + description: scaffolder v1beta2 template demo publishing to github +spec: + owner: backstage/techdocs-core + type: service + + parameters: + - title: Fill in some steps + required: + - name + properties: + name: + title: Name + type: string + description: Unique name of the component + ui:autofocus: true + ui:options: + rows: 5 + - title: Choose a location + required: + - repoUrl + properties: + repoUrl: + title: Repository Location + type: string + ui:field: RepoUrlPicker + ui:options: + allowedHosts: + - github.com + + steps: + - id: fetch-base + name: Fetch Base + action: fetch:cookiecutter + parameters: + url: ./template + values: + name: '{{ parameters.name }}' + + - id: fetch-docs + name: Fetch Docs + action: fetch:plain + parameters: + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions + + - id: publish + name: Publish + action: publish:github + parameters: + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + repoUrl: '{{ parameters.repoUrl }}' + + - id: register + name: Register + action: catalog:register + parameters: + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + + output: + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' diff --git a/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml new file mode 100644 index 0000000000..dd1e0ebd09 --- /dev/null +++ b/plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml @@ -0,0 +1,8 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: {{cookiecutter.name | jsonify}} +spec: + type: website + lifecycle: experimental + owner: guest diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts new file mode 100644 index 0000000000..2a841a564b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/TemplateActionRegistry.ts @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputBase, TemplateAction } from './types'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; + +export class TemplateActionRegistry { + private readonly actions = new Map>(); + + register(action: TemplateAction) { + if (this.actions.has(action.id)) { + throw new ConflictError( + `Template action with ID '${action.id}' has already been registered`, + ); + } + this.actions.set(action.id, action); + } + + get(actionId: string): TemplateAction { + const action = this.actions.get(actionId); + if (!action) { + throw new NotFoundError( + `Template action with ID '${actionId}' is not registered.`, + ); + } + return action; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts new file mode 100644 index 0000000000..2e25b5592f --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createCatalogRegisterAction } from './register'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts new file mode 100644 index 0000000000..070ad15ab7 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PassThrough } from 'stream'; +import os from 'os'; +import { getVoidLogger } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { createCatalogRegisterAction } from './register'; +import { Entity } from '@backstage/catalog-model'; + +describe('catalog:register', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const addLocation = jest.fn(); + const catalogClient = { + addLocation: addLocation, + }; + + const action = createCatalogRegisterAction({ + integrations, + catalogClient: (catalogClient as unknown) as CatalogApi, + }); + + const mockContext = { + workspacePath: os.tmpdir(), + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should reject registrations for locations that does not match any integration', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoContentsUrl: 'https://google.com/foo/bar', + }, + }), + ).rejects.toThrow( + /No integration found for host https:\/\/google.com\/foo\/bar/, + ); + }); + + it('should register location in catalog', async () => { + addLocation.mockResolvedValue({ + entities: [ + { + metadata: { + namespace: 'default', + name: 'test', + }, + kind: 'Component', + } as Entity, + ], + }); + await action.handler({ + ...mockContext, + input: { + catalogInfoUrl: 'http://foo/var', + }, + }); + expect(addLocation).toBeCalledWith({ + type: 'url', + target: 'http://foo/var', + }); + + expect(mockContext.output).toBeCalledWith( + 'entityRef', + 'Component:default/test', + ); + expect(mockContext.output).toBeCalledWith( + 'catalogInfoUrl', + 'http://foo/var', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts new file mode 100644 index 0000000000..7cf1ac2120 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/catalog/register.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { CatalogApi } from '@backstage/catalog-client'; +import { getEntityName } from '@backstage/catalog-model'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createCatalogRegisterAction(options: { + catalogClient: CatalogApi; + integrations: ScmIntegrations; +}) { + const { catalogClient, integrations } = options; + + return createTemplateAction< + | { catalogInfoUrl: string } + | { repoContentsUrl: string; catalogInfoPath?: string } + >({ + id: 'catalog:register', + schema: { + input: { + oneOf: [ + { + type: 'object', + required: ['catalogInfoUrl'], + properties: { + catalogInfoUrl: { + title: 'Catalog Info URL', + description: + 'An absolute URL pointing to the catalog info file location', + type: 'string', + }, + }, + }, + { + type: 'object', + required: ['repoContentsUrl'], + properties: { + repoContentsUrl: { + title: 'Repository Contents URL', + description: + 'An absolute URL pointing to the root of a repository directory tree', + type: 'string', + }, + catalogInfoPath: { + title: 'Fetch URL', + description: + 'A relative path from the repo root pointing to the catalog info file, defaults to /catalog-info.yaml', + type: 'string', + }, + }, + }, + ], + }, + }, + async handler(ctx) { + const { input } = ctx; + + let catalogInfoUrl; + if ('catalogInfoUrl' in input) { + catalogInfoUrl = input.catalogInfoUrl; + } else { + const { + repoContentsUrl, + catalogInfoPath = '/catalog-info.yaml', + } = input; + const integration = integrations.byUrl(repoContentsUrl); + if (!integration) { + throw new InputError( + `No integration found for host ${repoContentsUrl}`, + ); + } + + catalogInfoUrl = integration.resolveUrl({ + base: repoContentsUrl, + url: catalogInfoPath, + }); + } + + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); + + const result = await catalogClient.addLocation({ + type: 'url', + target: catalogInfoUrl, + }); + if (result.entities.length >= 1) { + const { kind, name, namespace } = getEntityName(result.entities[0]); + ctx.output('entityRef', `${kind}:${namespace}/${name}`); + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts new file mode 100644 index 0000000000..0829a101f2 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader } from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { ScmIntegrations } from '@backstage/integration'; +import { createCatalogRegisterAction } from './catalog'; +import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch'; +import { + createPublishAzureAction, + createPublishBitbucketAction, + createPublishGithubAction, + createPublishGitlabAction, +} from './publish'; +import Docker from 'dockerode'; +import { TemplaterBuilder } from '../../stages'; + +export const createBuiltinActions = (options: { + reader: UrlReader; + integrations: ScmIntegrations; + dockerClient: Docker; + catalogClient: CatalogApi; + templaters: TemplaterBuilder; +}) => { + const { + reader, + integrations, + dockerClient, + templaters, + catalogClient, + } = options; + + return [ + createFetchPlainAction({ + reader, + integrations, + }), + createFetchCookiecutterAction({ + reader, + integrations, + dockerClient, + templaters, + }), + createPublishGithubAction({ + integrations, + }), + createPublishGitlabAction({ + integrations, + }), + createPublishBitbucketAction({ + integrations, + }), + createPublishAzureAction({ + integrations, + }), + createCatalogRegisterAction({ catalogClient, integrations }), + ]; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts new file mode 100644 index 0000000000..7d26bcb20a --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -0,0 +1,135 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('./helpers'); + +import os from 'os'; +import { createFetchCookiecutterAction } from './cookiecutter'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { Templaters } from '../../../stages/templater'; +import { PassThrough } from 'stream'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { fetchContents } from './helpers'; +import mock from 'mock-fs'; + +describe('fetch:cookiecutter', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + + const templaters = new Templaters(); + const cookiecutterTemplater = { run: jest.fn() }; + const mockDockerClient = {}; + const mockTmpDir = os.tmpdir(); + const mockContext = { + input: { + url: 'https://google.com/cookie/cutter', + targetPath: 'something', + values: { + help: 'me', + }, + }, + baseUrl: 'somebase', + workspacePath: mockTmpDir, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), + }; + + const mockReader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + const action = createFetchCookiecutterAction({ + integrations, + templaters, + dockerClient: mockDockerClient as any, + reader: mockReader, + }); + + templaters.register('cookiecutter', cookiecutterTemplater); + + beforeEach(() => { + mock({ [`${mockContext.workspacePath}/result`]: {} }); + jest.restoreAllMocks(); + }); + + afterEach(() => { + mock.restore(); + }); + + it('should call fetchContents with the correct values', async () => { + await action.handler(mockContext); + + expect(fetchContents).toHaveBeenCalledWith({ + reader: mockReader, + integrations, + baseUrl: mockContext.baseUrl, + fetchUrl: mockContext.input.url, + outputPath: `${mockContext.workspacePath}/template/{{cookiecutter and 'contents'}}`, + }); + }); + + it('should execute the cookiecutter templater with the correct values', async () => { + await action.handler(mockContext); + + expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ + workspacePath: mockTmpDir, + dockerClient: mockDockerClient, + logStream: mockContext.logStream, + values: mockContext.input.values, + }); + }); + + it('should throw if there is no cookiecutter templater initialized', async () => { + const templatersWithoutCookiecutter = new Templaters(); + + const newAction = createFetchCookiecutterAction({ + integrations, + templaters: templatersWithoutCookiecutter, + dockerClient: mockDockerClient as any, + reader: mockReader, + }); + + await expect(newAction.handler(mockContext)).rejects.toThrow( + /No templater registered/, + ); + }); + + it('should throw if the target directory is outside of the workspace path', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + ...mockContext.input, + targetPath: '/foo', + }, + }), + ).rejects.toThrow( + /targetPath may not specify a path outside the working directory/, + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts new file mode 100644 index 0000000000..2a04ed301d --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath } from 'path'; +import Docker from 'dockerode'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { JsonObject } from '@backstage/config'; +import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater'; +import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createFetchCookiecutterAction(options: { + dockerClient: Docker; + reader: UrlReader; + integrations: ScmIntegrations; + templaters: TemplaterBuilder; +}) { + const { dockerClient, reader, templaters, integrations } = options; + + return createTemplateAction<{ + url: string; + targetPath?: string; + values: JsonObject; + }>({ + id: 'fetch:cookiecutter', + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + values: { + title: 'Template Values', + description: 'Values to pass on to cookiecutter for templating', + type: 'object', + }, + }, + }, + }, + async handler(ctx) { + ctx.logger.info('Fetching and then templating using cookiecutter'); + const workDir = await ctx.createTemporaryDirectory(); + const templateDir = resolvePath(workDir, 'template'); + const templateContentsDir = resolvePath( + templateDir, + "{{cookiecutter and 'contents'}}", + ); + const resultDir = resolvePath(workDir, 'result'); + + await fetchContents({ + reader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.input.url, + outputPath: templateContentsDir, + }); + + const cookiecutter = templaters.get('cookiecutter'); + + // Will execute the template in ./template and put the result in ./result + await cookiecutter.run({ + workspacePath: workDir, + dockerClient, + logStream: ctx.logStream, + values: ctx.input.values as TemplaterValues, + }); + + // Finally move the template result into the task workspace + const targetPath = ctx.input.targetPath ?? './'; + const outputPath = resolvePath(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + await fs.copy(resultDir, outputPath); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts new file mode 100644 index 0000000000..e475764a0b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('fs-extra'); +import fs from 'fs-extra'; + +import { UrlReader } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchContents } from './helpers'; +import os from 'os'; + +describe('fetchContent helper', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + + const readTree = jest.fn(); + const reader: UrlReader = { + read: jest.fn(), + readTree, + search: jest.fn(), + }; + + const options = { + reader, + integrations, + outputPath: os.tmpdir(), + }; + + it('should reject non string fetchUrls', async () => { + await expect( + fetchContents({ + ...options, + fetchUrl: false, + }), + ).rejects.toThrow('Invalid url parameter, expected string, got boolean'); + }); + + it('should reject absolute file locations', async () => { + await expect( + fetchContents({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: '/etc/passwd', + }), + ).rejects.toThrow( + 'Fetch URL may not be absolute for file locations, /etc/passwd', + ); + }); + + it('should copy file to outputpath', async () => { + await fetchContents({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: 'foo', + outputPath: 'somepath', + }); + expect(fs.copy).toBeCalledWith('/some/foo', 'somepath'); + }); + + it('should reject if no integration matches location', async () => { + await expect( + fetchContents({ + ...options, + baseUrl: 'http://example.com/some/folder', + }), + ).rejects.toThrow( + 'No integration found for location http://example.com/some/folder', + ); + }); + + it('should reject if fetch url is relative and no base url is specified', async () => { + await expect( + fetchContents({ + ...options, + fetchUrl: 'foo', + }), + ).rejects.toThrow( + 'Failed to fetch, template location could not be determined and the fetch URL is relative, foo', + ); + }); + + it('should fetch url contents', async () => { + const dirFunction = jest.fn(); + readTree.mockResolvedValue({ + dir: dirFunction, + }); + await fetchContents({ + ...options, + outputPath: 'foo', + fetchUrl: 'https://github.com/backstage/foo', + }); + expect(fs.ensureDir).toBeCalled(); + expect(dirFunction).toBeCalledWith({ targetDir: 'foo' }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts new file mode 100644 index 0000000000..652a1a56ca --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { resolve as resolvePath, isAbsolute } from 'path'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { JsonValue } from '@backstage/config'; + +export async function fetchContents({ + reader, + integrations, + baseUrl, + fetchUrl = '.', + outputPath, +}: { + reader: UrlReader; + integrations: ScmIntegrations; + baseUrl?: string; + fetchUrl?: JsonValue; + outputPath: string; +}) { + if (typeof fetchUrl !== 'string') { + throw new InputError( + `Invalid url parameter, expected string, got ${typeof fetchUrl}`, + ); + } + + let fetchUrlIsAbsolute = false; + try { + // eslint-disable-next-line no-new + new URL(fetchUrl); + fetchUrlIsAbsolute = true; + } catch { + /* ignored */ + } + + // We handle both file locations and url ones + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { + const basePath = baseUrl.slice('file://'.length); + if (isAbsolute(fetchUrl)) { + throw new InputError( + `Fetch URL may not be absolute for file locations, ${fetchUrl}`, + ); + } + const srcDir = resolvePath(basePath, '..', fetchUrl); + await fs.copy(srcDir, outputPath); + } else { + let readUrl; + + if (fetchUrlIsAbsolute) { + readUrl = fetchUrl; + } else if (baseUrl) { + const integration = integrations.byUrl(baseUrl); + if (!integration) { + throw new InputError(`No integration found for location ${baseUrl}`); + } + + readUrl = integration.resolveUrl({ + url: fetchUrl, + base: baseUrl, + }); + } else { + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); + } + + const res = await reader.readTree(readUrl); + await fs.ensureDir(outputPath); + await res.dir({ targetDir: outputPath }); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts new file mode 100644 index 0000000000..0a7239b6b7 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createFetchPlainAction } from './plain'; +export { createFetchCookiecutterAction } from './cookiecutter'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts new file mode 100644 index 0000000000..88eeb5997b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('./helpers'); + +import os from 'os'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { createFetchPlainAction } from './plain'; +import { PassThrough } from 'stream'; +import { fetchContents } from './helpers'; + +describe('fetch:plain', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + const reader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + const action = createFetchPlainAction({ integrations, reader }); + const mockContext = { + workspacePath: os.tmpdir(), + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + it('should disallow a target path outside working directory', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + url: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + targetPath: '/foobar', + }, + }), + ).rejects.toThrow( + /Fetch action targetPath may not specify a path outside the working directory/, + ); + }); + + it('should fetch plain', async () => { + await action.handler({ + ...mockContext, + input: { + url: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + targetPath: 'lol', + }, + }); + expect(fetchContents).toBeCalledWith( + expect.objectContaining({ + outputPath: `${mockContext.workspacePath}/lol`, + fetchUrl: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets', + }), + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts new file mode 100644 index 0000000000..1eb1a6dc82 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plain.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; +import { InputError, UrlReader } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchContents } from './helpers'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createFetchPlainAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}) { + const { reader, integrations } = options; + + return createTemplateAction<{ url: string; targetPath?: string }>({ + id: 'fetch:plain', + schema: { + input: { + type: 'object', + required: ['url'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the directory tree to fetch', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the contents to.', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + ctx.logger.info('Fetching plain content from remote URL'); + + // Finally move the template result into the task workspace + const targetPath = ctx.input.targetPath ?? './'; + const outputPath = path.resolve(ctx.workspacePath, targetPath); + if (!outputPath.startsWith(ctx.workspacePath)) { + throw new InputError( + `Fetch action targetPath may not specify a path outside the working directory`, + ); + } + + await fetchContents({ + reader, + integrations, + baseUrl: ctx.baseUrl, + fetchUrl: ctx.input.url, + outputPath, + }); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts new file mode 100644 index 0000000000..e4281c172b --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './catalog'; +export * from './fetch'; +export * from './publish'; +export { createBuiltinActions } from './createBuiltinActions'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts new file mode 100644 index 0000000000..4e2d721af9 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts @@ -0,0 +1,186 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('azure-devops-node-api', () => ({ + WebApi: jest.fn(), + getPersonalAccessTokenHandler: jest.fn().mockReturnValue(() => {}), +})); + +import { createPublishAzureAction } from './azure'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { WebApi } from 'azure-devops-node-api'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('publish:azure', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + const action = createPublishAzureAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'dev.azure.com?repo=repo&owner=owner&organization=org', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const mockGitClient = { + createRepository: jest.fn(), + }; + const mockGitApi = { + getGitApi: jest.fn().mockReturnValue(mockGitClient), + }; + + ((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi); + + beforeEach(() => { + jest.restoreAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?owner=owner&repo=repo' }, + }), + ).rejects.toThrow(/missing organization/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'azure.com?repo=bob&owner=owner&organization=org' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: + 'myazurehostnotoken.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/No token provided for Azure Integration/); + }); + + it('should throw when no repo is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'dev.azure.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/Unable to create the repository/); + }); + + it('should throw if there is no remoteUrl returned', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: null, + })); + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'dev.azure.com?repo=bob&owner=owner&organization=org', + }, + }), + ).rejects.toThrow(/No remote URL returned/); + }); + + it('should call the azureApis with the correct values', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'http://google.com', + })); + + await action.handler(mockContext); + + expect(WebApi).toHaveBeenCalledWith( + 'https://dev.azure.com/org', + expect.any(Function), + ); + + expect(mockGitClient.createRepository).toHaveBeenCalledWith( + { + name: 'bob', + }, + 'owner', + ); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + })); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + auth: { username: 'notempty', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call output with the remoteUrl and the repoContentsUrl', async () => { + mockGitClient.createRepository.mockImplementation(() => ({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + })); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://dev.azure.com/organization/project/_git/repo', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://dev.azure.com/organization/project/_git/repo', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts new file mode 100644 index 0000000000..5eaba05ea0 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrationRegistry } from '@backstage/integration'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; +import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; +import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createPublishAzureAction(options: { + integrations: ScmIntegrationRegistry; +}) { + const { integrations } = options; + + return createTemplateAction<{ + repoUrl: string; + description?: string; + }>({ + id: 'publish:azure', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + }, + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + const { owner, repo, host, organization } = parseRepoUrl( + ctx.input.repoUrl, + ); + + if (!organization) { + throw new InputError( + `Invalid URL provider was included in the repo URL to create ${ctx.input.repoUrl}, missing organization`, + ); + } + + const integrationConfig = integrations.azure.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your integrations config`, + ); + } + if (!integrationConfig.config.token) { + throw new InputError(`No token provided for Azure Integration ${host}`); + } + const authHandler = getPersonalAccessTokenHandler( + integrationConfig.config.token, + ); + + const webApi = new WebApi(`https://${host}/${organization}`, authHandler); + const client = await webApi.getGitApi(); + const createOptions: GitRepositoryCreateOptions = { name: repo }; + const returnedRepo = await client.createRepository(createOptions, owner); + + if (!returnedRepo) { + throw new InputError( + `Unable to create the repository with Organization ${organization}, Project ${owner} and Repo ${repo}. + Please make sure that both the Org and Project are typed corrected and exist.`, + ); + } + const remoteUrl = returnedRepo.remoteUrl; + + if (!remoteUrl) { + throw new InputError( + 'No remote URL returned from create repository for Azure', + ); + } + + // blam: Repo contents is serialized into the path, + // so it's just the base path I think + const repoContentsUrl = remoteUrl; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'notempty', + password: integrationConfig.config.token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts new file mode 100644 index 0000000000..40e7f613f2 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); + +import { createPublishBitbucketAction } from './bitbucket'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('publish:bitbucket', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + token: 'tokenlols', + }, + { + host: 'hosted.bitbucket.com', + token: 'thing', + }, + { + host: 'notoken.bitbucket.com', + }, + ], + }, + }), + ); + const action = createPublishBitbucketAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'bitbucket.org?repo=repo&owner=owner', + repoVisibility: 'private', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + const server = setupServer(); + msw.setupDefaultHandlers(server); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'bitbucket.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'bitbucket.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'notoken.bitbucket.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/Authorization has not been provided for Bitbucket/); + }); + + it('should call the correct APIs when the host is bitbucket cloud', async () => { + expect.assertions(2); + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Bearer tokenlols'); + expect(req.body).toEqual({ is_private: true, scm: 'git' }); + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/repo', + }, + ], + }, + }), + ); + }, + ), + ); + + await action.handler(mockContext); + }); + + it('should call the correct APIs when the host is hosted bitbucket', async () => { + expect.assertions(2); + server.use( + rest.post( + 'https://hosted.bitbucket.com/rest/api/1.0/projects/owner/repos', + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Bearer thing'); + expect(req.body).toEqual({ is_private: true, name: 'repo' }); + return res( + ctx.status(201), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/project/repos/repo', + }, + ], + clone: [ + { + name: 'http', + href: 'https://bitbucket.mycompany.com/scm/project/repo', + }, + ], + }, + }), + ); + }, + ), + ); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoUrl: 'hosted.bitbucket.com?owner=owner&repo=repo', + }, + }); + }); + + it('should call initAndPush with the correct values', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/cloneurl', + }, + ], + }, + }), + ), + ), + ); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://bitbucket.org/owner/cloneurl', + auth: { username: 'x-token-auth', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call outputs with the correct urls', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/owner/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/owner/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/owner/cloneurl', + }, + ], + }, + }), + ), + ), + ); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://bitbucket.org/owner/cloneurl', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://bitbucket.org/owner/repo/src/master', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts new file mode 100644 index 0000000000..4c3cf30ecd --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -0,0 +1,251 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { + BitbucketIntegrationConfig, + ScmIntegrationRegistry, +} from '@backstage/integration'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; +import fetch from 'cross-fetch'; +import { createTemplateAction } from '../../createTemplateAction'; + +const createBitbucketCloudRepository = async (opts: { + owner: string; + repo: string; + description: string; + repoVisibility: 'private' | 'public'; + authorization: string; +}) => { + const { owner, repo, description, repoVisibility, authorization } = opts; + + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + scm: 'git', + description: description, + is_private: repoVisibility === 'private', + }), + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + }; + + let response: Response; + try { + response = await fetch( + `https://api.bitbucket.org/2.0/repositories/${owner}/${repo}`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + + if (response.status !== 200) { + throw new Error( + `Unable to create repository, ${response.status} ${ + response.statusText + }, ${await response.text()}`, + ); + } + + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'https') { + remoteUrl = link.href; + } + } + + // TODO use the urlReader to get the default branch + const repoContentsUrl = `${r.links.html.href}/src/master`; + return { remoteUrl, repoContentsUrl }; +}; + +const createBitbucketServerRepository = async (opts: { + host: string; + owner: string; + repo: string; + description: string; + repoVisibility: 'private' | 'public'; + authorization: string; +}) => { + const { + host, + owner, + repo, + description, + authorization, + repoVisibility, + } = opts; + + let response: Response; + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + name: repo, + description: description, + is_private: repoVisibility === 'private', + }), + headers: { + Authorization: authorization, + 'Content-Type': 'application/json', + }, + }; + + try { + response = await fetch( + `https://${host}/rest/api/1.0/projects/${owner}/repos`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + + if (response.status !== 201) { + throw new Error( + `Unable to create repository, ${response.status} ${ + response.statusText + }, ${await response.text()}`, + ); + } + + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'http') { + remoteUrl = link.href; + } + } + + const repoContentsUrl = `${r.links.self[0].href}`; + return { remoteUrl, repoContentsUrl }; +}; + +const getAuthorizationHeader = (config: BitbucketIntegrationConfig) => { + if (config.username && config.appPassword) { + const buffer = Buffer.from( + `${config.username}:${config.appPassword}`, + 'utf8', + ); + + return `Basic ${buffer.toString('base64')}`; + } + + if (config.token) { + return `Bearer ${config.token}`; + } + + throw new Error( + `Authorization has not been provided for Bitbucket. Please add either username + appPassword or token to the Integrations config`, + ); +}; + +export function createPublishBitbucketAction(options: { + integrations: ScmIntegrationRegistry; +}) { + const { integrations } = options; + + return createTemplateAction<{ + repoUrl: string; + description: string; + repoVisibility: 'private' | 'public'; + }>({ + id: 'publish:bitbucket', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public'], + }, + }, + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + const { repoUrl, description, repoVisibility = 'private' } = ctx.input; + + const { owner, repo, host } = parseRepoUrl(repoUrl); + + const integrationConfig = integrations.bitbucket.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your integrations config`, + ); + } + + const authorization = getAuthorizationHeader(integrationConfig.config); + + const createMethod = + host === 'bitbucket.org' + ? createBitbucketCloudRepository + : createBitbucketServerRepository; + + const { remoteUrl, repoContentsUrl } = await createMethod({ + authorization, + host, + owner, + repo, + repoVisibility, + description, + }); + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: integrationConfig.config.username + ? integrationConfig.config.username + : 'x-token-auth', + password: integrationConfig.config.appPassword + ? integrationConfig.config.appPassword + : integrationConfig.config.token ?? '', + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts new file mode 100644 index 0000000000..6f789a7b76 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitab.test.ts @@ -0,0 +1,165 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('@gitbeaker/node'); + +import { createPublishGitlabAction } from './gitlab'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('publish:gitlab', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + token: 'tokenlols', + apiBaseUrl: 'https://api.gitlab.com', + }, + { + host: 'hosted.gitlab.com', + apiBaseUrl: 'https://api.hosted.gitlab.com', + }, + ], + }, + }), + ); + const action = createPublishGitlabAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'gitlab.com?repo=repo&owner=owner', + repoVisibility: 'private', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const { mockGitlabClient } = require('@gitbeaker/node'); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'gitlab.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'gitlab.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'hosted.gitlab.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/No token available for host/); + }); + + it('should call the correct Gitlab APIs when the owner is an organization', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ + namespace_id: 1234, + name: 'repo', + visibility: 'private', + }); + }); + + it('should call the correct Gitlab APIs when the owner is not an organization', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: null }); + mockGitlabClient.Users.current.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ + namespace_id: 12345, + name: 'repo', + visibility: 'private', + }); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'http://mockurl.git', + auth: { username: 'oauth2', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should call output with the remoteUrl and repoContentsUrl', async () => { + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'http://mockurl', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'http://mockurl/-/blob/master', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts new file mode 100644 index 0000000000..9880dda621 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -0,0 +1,258 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +jest.mock('../../../stages/publish/helpers'); +jest.mock('@octokit/rest'); + +import { createPublishGithubAction } from './github'; +import { ScmIntegrations } from '@backstage/integration'; +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { PassThrough } from 'stream'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; + +describe('publish:github', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'github.com', token: 'tokenlols' }, + { host: 'ghe.github.com' }, + ], + }, + }), + ); + const action = createPublishGithubAction({ integrations }); + const mockContext = { + input: { + repoUrl: 'github.com?repo=repo&owner=owner', + description: 'description', + repoVisibility: 'private', + access: 'owner/blam', + }, + workspacePath: 'lol', + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + const { mockGithubClient } = require('@octokit/rest'); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should throw an error when the repoUrl is not well formed', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'github.com?repo=bob' }, + }), + ).rejects.toThrow(/missing owner/); + + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'github.com?owner=owner' }, + }), + ).rejects.toThrow(/missing repo/); + }); + + it('should throw if there is no integration config provided', async () => { + await expect( + action.handler({ + ...mockContext, + input: { repoUrl: 'missing.com?repo=bob&owner=owner' }, + }), + ).rejects.toThrow(/No matching integration configuration/); + }); + + it('should throw if there is no token in the integration config that is returned', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + repoUrl: 'ghe.github.com?repo=bob&owner=owner', + }, + }), + ).rejects.toThrow(/No token available for host/); + }); + + it('should call the githubApis with the correct values for createInOrg', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'Organization' }, + }); + + mockGithubClient.repos.createInOrg.mockResolvedValue({ data: {} }); + + await action.handler(mockContext); + expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + org: 'owner', + private: true, + visibility: 'private', + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoVisibility: 'public', + }, + }); + expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + org: 'owner', + private: false, + visibility: 'public', + }); + }); + + it('should call the githubApis with the correct values for createForAuthenticatedUser', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: {}, + }); + + await action.handler(mockContext); + expect( + mockGithubClient.repos.createForAuthenticatedUser, + ).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + private: true, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + repoVisibility: 'public', + }, + }); + expect( + mockGithubClient.repos.createForAuthenticatedUser, + ).toHaveBeenCalledWith({ + description: 'description', + name: 'repo', + private: false, + }); + }); + + it('should call initRepoAndPush with the correct values', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + remoteUrl: 'https://github.com/clone/url.git', + auth: { username: 'x-access-token', password: 'tokenlols' }, + logger: mockContext.logger, + }); + }); + + it('should add access for the team when it starts with the owner', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect( + mockGithubClient.teams.addOrUpdateRepoPermissionsInOrg, + ).toHaveBeenCalledWith({ + org: 'owner', + team_slug: 'blam', + owner: 'owner', + repo: 'repo', + permission: 'admin', + }); + }); + + it('should add outside collaborators when provided', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + access: 'outsidecollaborator', + }, + }); + + expect(mockGithubClient.repos.addCollaborator).toHaveBeenCalledWith({ + username: 'outsidecollaborator', + owner: 'owner', + repo: 'repo', + permission: 'admin', + }); + }); + + it('should call output with the remoteUrl and the repoContentsUrl', async () => { + mockGithubClient.users.getByUsername.mockResolvedValue({ + data: { type: 'User' }, + }); + + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'https://github.com/clone/url.git', + html_url: 'https://github.com/html/url', + }, + }); + + await action.handler(mockContext); + + expect(mockContext.output).toHaveBeenCalledWith( + 'remoteUrl', + 'https://github.com/clone/url.git', + ); + expect(mockContext.output).toHaveBeenCalledWith( + 'repoContentsUrl', + 'https://github.com/html/url/blob/master', + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts new file mode 100644 index 0000000000..be0820d0b1 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -0,0 +1,174 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { + GithubCredentialsProvider, + ScmIntegrationRegistry, +} from '@backstage/integration'; +import { Octokit } from '@octokit/rest'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createPublishGithubAction(options: { + integrations: ScmIntegrationRegistry; +}) { + const { integrations } = options; + + const credentialsProviders = new Map( + integrations.github.list().map(integration => { + const provider = GithubCredentialsProvider.create(integration.config); + return [integration.config.host, provider]; + }), + ); + + return createTemplateAction<{ + repoUrl: string; + description?: string; + access?: string; + repoVisibility: 'private' | 'internal' | 'public'; + }>({ + id: 'publish:github', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + description: { + title: 'Repository Description', + type: 'string', + }, + access: { + title: 'Repository Access', + type: 'string', + }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public', 'internal'], + }, + }, + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + const { + repoUrl, + description, + access, + repoVisibility = 'private', + } = ctx.input; + + const { owner, repo, host } = parseRepoUrl(repoUrl); + + const credentialsProvider = credentialsProviders.get(host); + const integrationConfig = integrations.github.byHost(host); + + if (!credentialsProvider || !integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your integrations config`, + ); + } + + const { token } = await credentialsProvider.getCredentials({ + url: `${host}/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + }); + + if (!token) { + throw new InputError( + `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`, + ); + } + + const client = new Octokit({ + auth: token, + baseUrl: integrationConfig.config.apiBaseUrl, + }); + + const user = await client.users.getByUsername({ + username: owner, + }); + + const repoCreationPromise = + user.data.type === 'Organization' + ? client.repos.createInOrg({ + name: repo, + org: owner, + private: repoVisibility !== 'public', + visibility: repoVisibility, + description: description, + }) + : client.repos.createForAuthenticatedUser({ + name: repo, + private: repoVisibility === 'private', + description: description, + }); + + const { data } = await repoCreationPromise; + if (access?.startsWith(`${owner}/`)) { + const [, team] = access.split('/'); + await client.teams.addOrUpdateRepoPermissionsInOrg({ + org: owner, + team_slug: team, + owner, + repo, + permission: 'admin', + }); + // no need to add access if it's the person who own's the personal account + } else if (access && access !== owner) { + await client.repos.addCollaborator({ + owner, + repo, + username: access, + permission: 'admin', + }); + } + + const remoteUrl = data.clone_url; + const repoContentsUrl = `${data.html_url}/blob/master`; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'x-access-token', + password: token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts new file mode 100644 index 0000000000..9122c828a0 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -0,0 +1,120 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; +import { ScmIntegrationRegistry } from '@backstage/integration'; +import { Gitlab } from '@gitbeaker/node'; +import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { parseRepoUrl } from './util'; +import { createTemplateAction } from '../../createTemplateAction'; + +export function createPublishGitlabAction(options: { + integrations: ScmIntegrationRegistry; +}) { + const { integrations } = options; + + return createTemplateAction<{ + repoUrl: string; + repoVisibility: 'private' | 'internal' | 'public'; + }>({ + id: 'publish:gitlab', + schema: { + input: { + type: 'object', + required: ['repoUrl'], + properties: { + repoUrl: { + title: 'Repository Location', + type: 'string', + }, + repoVisibility: { + title: 'Repository Visiblity', + type: 'string', + enum: ['private', 'public', 'internal'], + }, + }, + }, + output: { + type: 'object', + properties: { + remoteUrl: { + title: 'A URL to the repository with the provider', + type: 'string', + }, + repoContentsUrl: { + title: 'A URL to the root of the repository', + type: 'string', + }, + }, + }, + }, + async handler(ctx) { + const { repoUrl, repoVisibility = 'private' } = ctx.input; + + const { owner, repo, host } = parseRepoUrl(repoUrl); + + const integrationConfig = integrations.gitlab.byHost(host); + + if (!integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your integrations config`, + ); + } + + if (!integrationConfig.config.token) { + throw new InputError(`No token available for host ${host}`); + } + + const client = new Gitlab({ + host: integrationConfig.config.baseUrl, + token: integrationConfig.config.token, + }); + + let { id: targetNamespace } = (await client.Namespaces.show(owner)) as { + id: number; + }; + + if (!targetNamespace) { + const { id } = (await client.Users.current()) as { + id: number; + }; + targetNamespace = id; + } + + const { http_url_to_repo } = await client.Projects.create({ + namespace_id: targetNamespace, + name: repo, + visibility: repoVisibility, + }); + + const remoteUrl = (http_url_to_repo as string).replace(/\.git$/, ''); + const repoContentsUrl = `${remoteUrl}/-/blob/master`; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl: http_url_to_repo as string, + auth: { + username: 'oauth2', + password: integrationConfig.config.token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }); +} diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts new file mode 100644 index 0000000000..70a7909144 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { createPublishGithubAction } from './github'; +export { createPublishAzureAction } from './azure'; +export { createPublishGitlabAction } from './gitlab'; +export { createPublishBitbucketAction } from './bitbucket'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts new file mode 100644 index 0000000000..9349d5b5eb --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/util.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError } from '@backstage/backend-common'; + +export const parseRepoUrl = (repoUrl: string) => { + let parsed; + try { + parsed = new URL(`https://${repoUrl}`); + } catch (error) { + throw new InputError( + `Invalid repo URL passed to publisher, got ${repoUrl}, ${error}`, + ); + } + const host = parsed.host; + const owner = parsed.searchParams.get('owner'); + + if (!owner) { + throw new InputError( + `Invalid repo URL passed to publisher: ${repoUrl}, missing owner`, + ); + } + const repo = parsed.searchParams.get('repo'); + if (!repo) { + throw new InputError( + `Invalid repo URL passed to publisher: ${repoUrl}, missing repo`, + ); + } + + const organization = parsed.searchParams.get('organization'); + + return { host, owner, repo, organization }; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts new file mode 100644 index 0000000000..0bd52d0540 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/createTemplateAction.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputBase, TemplateAction } from './types'; + +export const createTemplateAction = ( + templateAction: TemplateAction, +): TemplateAction => { + // TODO(blam): Can add some more validation here to validate the action later on + return templateAction; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts new file mode 100644 index 0000000000..98ae406792 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './builtin'; +export { TemplateActionRegistry } from './TemplateActionRegistry'; +export { createTemplateAction } from './createTemplateAction'; +export type { ActionContext, TemplateAction } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/types.ts b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts new file mode 100644 index 0000000000..20b7696d70 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/types.ts @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; +import { Writable } from 'stream'; +import { JsonValue, JsonObject } from '@backstage/config'; +import { Schema } from 'jsonschema'; + +type PartialJsonObject = Partial; +type PartialJsonValue = PartialJsonObject | JsonValue | undefined; +export type InputBase = Partial<{ [name: string]: PartialJsonValue }>; + +export type ActionContext = { + /** + * Base URL for the location of the task spec, typically the url of the source entity file. + */ + baseUrl?: string; + + logger: Logger; + logStream: Writable; + + workspacePath: string; + input: Input; + output(name: string, value: JsonValue): void; + + /** + * Creates a temporary directory for use by the action, which is then cleaned up automatically. + */ + createTemporaryDirectory(): Promise; +}; + +export type TemplateAction = { + id: string; + schema?: { + input?: Schema; + output?: Schema; + }; + handler: (ctx: ActionContext) => Promise; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 470bc230d6..9de10ca1eb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -15,3 +15,4 @@ */ export * from './stages'; export * from './jobs'; +export * from './actions'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts index dfe49a8aa5..ef23af1b3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts @@ -17,3 +17,5 @@ export * from './prepare'; export * from './publish'; export * from './templater'; export * from './helpers'; + +export { createLegacyActions } from './legacy'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index efba57a063..c7c52fa320 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -14,120 +14,95 @@ * limitations under the License. */ -import { TemplateActionRegistry } from '../tasks/TemplateConverter'; import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; import { PublisherBuilder } from './publish'; -import { CatalogApi } from '@backstage/catalog-client'; -import { getEntityName } from '@backstage/catalog-model'; +import { createTemplateAction } from '../actions'; type Options = { dockerClient: Docker; preparers: PreparerBuilder; templaters: TemplaterBuilder; publishers: PublisherBuilder; - catalogClient: CatalogApi; }; -export function registerLegacyActions( - registry: TemplateActionRegistry, - options: Options, -) { - const { - dockerClient, - preparers, - templaters, - publishers, - catalogClient, - } = options; +export function createLegacyActions(options: Options) { + const { dockerClient, preparers, templaters, publishers } = options; - registry.register({ - id: 'legacy:prepare', - async handler(ctx) { - ctx.logger.info('Preparing the skeleton'); - const { protocol, url } = ctx.parameters; - const preparer = - protocol === 'file' ? new FilePreparer() : preparers.get(url as string); + return [ + createTemplateAction({ + id: 'legacy:prepare', + async handler(ctx) { + ctx.logger.info('Preparing the skeleton'); + const { protocol, url } = ctx.input; + const preparer = + protocol === 'file' + ? new FilePreparer() + : preparers.get(url as string); - await preparer.prepare({ - url: url as string, - logger: ctx.logger, - workspacePath: ctx.workspacePath, - }); - }, - }); + await preparer.prepare({ + url: url as string, + logger: ctx.logger, + workspacePath: ctx.workspacePath, + }); + }, + }), + createTemplateAction({ + id: 'legacy:template', + async handler(ctx) { + ctx.logger.info('Running the templater'); + const templater = templaters.get(ctx.input.templater as string); + await templater.run({ + workspacePath: ctx.workspacePath, + dockerClient, + logStream: ctx.logStream, + values: ctx.input.values as TemplaterValues, + }); + }, + }), + createTemplateAction({ + id: 'legacy:publish', + async handler(ctx) { + const { values } = ctx.input; + if ( + typeof values !== 'object' || + values === null || + Array.isArray(values) + ) { + throw new Error( + `Invalid values passed to publish, got ${typeof values}`, + ); + } + const storePath = values.storePath as unknown; + if (typeof storePath !== 'string') { + throw new Error( + `Invalid store path passed to publish, got ${typeof storePath}`, + ); + } + const owner = values.owner as unknown; + if (typeof owner !== 'string') { + throw new Error( + `Invalid owner passed to publish, got ${typeof owner}`, + ); + } - registry.register({ - id: 'legacy:template', - async handler(ctx) { - ctx.logger.info('Running the templater'); - const templater = templaters.get(ctx.parameters.templater as string); - await templater.run({ - workspacePath: ctx.workspacePath, - dockerClient, - logStream: ctx.logStream, - values: ctx.parameters.values as TemplaterValues, - }); - }, - }); - - registry.register({ - id: 'legacy:publish', - async handler(ctx) { - const { values } = ctx.parameters; - if ( - typeof values !== 'object' || - values === null || - Array.isArray(values) - ) { - throw new Error( - `Invalid values passed to publish, got ${typeof values}`, - ); - } - const storePath = values.storePath as unknown; - if (typeof storePath !== 'string') { - throw new Error( - `Invalid store path passed to publish, got ${typeof storePath}`, - ); - } - const owner = values.owner as unknown; - if (typeof owner !== 'string') { - throw new Error(`Invalid owner passed to publish, got ${typeof owner}`); - } - - const publisher = publishers.get(storePath); - ctx.logger.info('Will now store the template'); - const { remoteUrl, catalogInfoUrl } = await publisher.publish({ - values: { - ...values, - owner, - storePath, - }, - workspacePath: ctx.workspacePath, - logger: ctx.logger, - }); - ctx.output('remoteUrl', remoteUrl); - if (catalogInfoUrl) { - ctx.output('catalogInfoUrl', catalogInfoUrl); - } - }, - }); - - registry.register({ - id: 'catalog:register', - async handler(ctx) { - const { catalogInfoUrl } = ctx.parameters; - ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); - - const result = await catalogClient.addLocation({ - type: 'url', - target: catalogInfoUrl as string, - }); - if (result.entities.length >= 1) { - const { kind, name, namespace } = getEntityName(result.entities[0]); - ctx.output('entityRef', `${kind}:${namespace}/${name}`); - } - }, - }); + const publisher = publishers.get(storePath); + ctx.logger.info('Will now store the template'); + const { remoteUrl, catalogInfoUrl } = await publisher.publish({ + values: { + ...values, + owner, + storePath, + }, + workspacePath: ctx.workspacePath, + logger: ctx.logger, + }); + ctx.output('remoteUrl', remoteUrl); + if (catalogInfoUrl) { + ctx.output('catalogInfoUrl', catalogInfoUrl); + } + }, + }), + ]; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 9d0c0863aa..4d8f652ae0 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -18,11 +18,13 @@ import { PassThrough } from 'stream'; import { Logger } from 'winston'; import * as winston from 'winston'; import { JsonValue, JsonObject } from '@backstage/config'; +import { validate as validateJsonSchema } from 'jsonschema'; import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; -import { TemplateActionRegistry } from './TemplateConverter'; +import { TemplateActionRegistry } from '../actions/TemplateActionRegistry'; import * as handlebars from 'handlebars'; +import { InputError } from '@backstage/backend-common'; type Options = { logger: Logger; @@ -44,10 +46,11 @@ export class TaskWorker { } async runOneTask(task: Task) { + let workspacePath: string | undefined = undefined; try { const { actionRegistry } = this.options; - const workspacePath = path.join( + workspacePath = path.join( this.options.workingDirectory, await task.getWorkspaceName(), ); @@ -95,8 +98,8 @@ export class TaskWorker { throw new Error(`Action '${step.action}' does not exist`); } - const parameters = JSON.parse( - JSON.stringify(step.parameters), + const input = JSON.parse( + JSON.stringify(step.input), (_key, value) => { if (typeof value === 'string') { return handlebars.compile(value, { @@ -110,18 +113,46 @@ export class TaskWorker { }, ); + if (action.schema?.input) { + const validateResult = validateJsonSchema(input, action.schema, { + propertyName: 'input', + }); + if (!validateResult.valid) { + const errors = validateResult.errors.join(', '); + throw new InputError( + `Invalid input passed to action ${action.id}, ${errors}`, + ); + } + } + const stepOutputs: { [name: string]: JsonValue } = {}; + // Keep track of all tmp dirs that are created by the action so we can remove them after + const tmpDirs = new Array(); + await action.handler({ + baseUrl: task.spec.baseUrl, logger: taskLogger, logStream: stream, - parameters, + input, workspacePath, + async createTemporaryDirectory() { + const tmpDir = await fs.mkdtemp( + `${workspacePath}_step-${step.id}-`, + ); + tmpDirs.push(tmpDir); + return tmpDir; + }, output(name: string, value: JsonValue) { stepOutputs[name] = value; }, }); + // Remove all temporary directories that were created when executing the action + for (const tmpDir of tmpDirs) { + await fs.remove(tmpDir); + } + templateCtx.steps[step.id] = { output: stepOutputs }; await task.emitLog(`Finished step ${step.name}`, { @@ -157,6 +188,10 @@ export class TaskWorker { await task.complete('failed', { error: { name: error.name, message: error.message }, }); + } finally { + if (workspacePath) { + await fs.remove(workspacePath); + } } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index 7f76ac33d0..03f95b2aae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -15,13 +15,9 @@ */ import { resolve as resolvePath, dirname } from 'path'; -import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { Logger } from 'winston'; -import { Writable } from 'stream'; - +import parseGitUrl from 'git-url-parse'; import { TaskSpec } from './types'; -import { ConflictError, NotFoundError } from '@backstage/backend-common'; import { getTemplaterKey, joinGitUrlPath, @@ -31,7 +27,7 @@ import { export function templateEntityToSpec( template: TemplateEntityV1alpha1, - values: TemplaterValues, + inputValues: TemplaterValues, ): TaskSpec { const steps: TaskSpec['steps'] = []; @@ -47,11 +43,18 @@ export function templateEntityToSpec( } const templater = getTemplaterKey(template); + const values = { + ...inputValues, + destination: { + git: parseGitUrl(inputValues.storePath), + }, + } as TemplaterValues; + steps.push({ id: 'prepare', name: 'Prepare', action: 'legacy:prepare', - parameters: { + input: { protocol, url, }, @@ -61,7 +64,7 @@ export function templateEntityToSpec( id: 'template', name: 'Template', action: 'legacy:template', - parameters: { + input: { templater, values, }, @@ -71,7 +74,7 @@ export function templateEntityToSpec( id: 'publish', name: 'Publish', action: 'legacy:publish', - parameters: { + input: { values, }, }); @@ -80,55 +83,19 @@ export function templateEntityToSpec( id: 'register', name: 'Register', action: 'catalog:register', - parameters: { + input: { catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', }, }); return { + baseUrl: undefined, // not used by legacy actions values: {}, steps, output: { remoteUrl: '{{ steps.publish.output.remoteUrl }}', - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', + catalogInfoUrl: '{{ steps.register.output.catalogInfoUrl }}', entityRef: '{{ steps.register.output.entityRef }}', }, }; } - -type ActionContext = { - logger: Logger; - logStream: Writable; - - workspacePath: string; - parameters: { [name: string]: JsonValue }; - output(name: string, value: JsonValue): void; -}; - -type TemplateAction = { - id: string; - handler: (ctx: ActionContext) => Promise; -}; - -export class TemplateActionRegistry { - private readonly actions = new Map(); - - register(action: TemplateAction) { - if (this.actions.has(action.id)) { - throw new ConflictError( - `Template action with ID '${action.id}' has already been registered`, - ); - } - this.actions.set(action.id, action); - } - - get(actionId: string): TemplateAction { - const action = this.actions.get(actionId); - if (!action) { - throw new NotFoundError( - `Template action with ID '${actionId}' is not registered.`, - ); - } - return action; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 27a1b62d04..6963817a1e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -43,12 +43,13 @@ export type DbTaskEventRow = { }; export type TaskSpec = { + baseUrl?: string; values: JsonObject; steps: Array<{ id: string; name: string; action: string; - parameters?: JsonObject; + input?: JsonObject; }>; output: { [name: string]: string }; }; @@ -90,6 +91,7 @@ export type TaskStoreGetEventsOptions = { taskId: string; after?: number | undefined; }; + export interface TaskStore { createTask(task: TaskSpec): Promise<{ taskId: string }>; getTask(taskId: string): Promise; diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index dd3d43c6d1..905d85c35d 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -18,6 +18,11 @@ import os from 'os'; import fs from 'fs-extra'; import { Logger } from 'winston'; import { Config } from '@backstage/config'; +import { + Entity, + LOCATION_ANNOTATION, + SOURCE_LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; export async function getWorkingDirectory( config: Config, @@ -42,3 +47,34 @@ export async function getWorkingDirectory( } return workingDirectory; } + +/** + * Gets the base URL of the entity location that points to the source location + * of the entity description within a repo. If there is not source location + * or if it has an invalid type, undefined will be returned instead. + * + * For file locations this will return a `file://` URL. + */ +export function getEntityBaseUrl(entity: Entity): string | undefined { + let location = entity.metadata.annotations?.[SOURCE_LOCATION_ANNOTATION]; + if (!location) { + location = entity.metadata.annotations?.[LOCATION_ANNOTATION]; + } + if (!location) { + return undefined; + } + + const [type, url] = location.split(/:(.+)/); + if (!url) { + return undefined; + } + + if (type === 'url') { + return url; + } else if (type === 'file') { + return `file://${url}`; + } + // Only url and file location are handled, as we otherwise don't know if + // what the url is pointing to makes sense to use as a baseUrl + return undefined; +} diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 8fd53e5fb6..b7c2582c81 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -32,6 +32,7 @@ import { SingleConnectionDatabaseManager, PluginDatabaseManager, getVoidLogger, + UrlReaders, } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import express from 'express'; @@ -61,6 +62,11 @@ function createDatabase(): PluginDatabaseManager { ).forPlugin('scaffolder'); } +const mockUrlReader = UrlReaders.default({ + logger: getVoidLogger(), + config: new ConfigReader({}), +}); + describe('createRouter - working directory', () => { const mockPrepare = jest.fn(); const mockPreparers = new Preparers(); @@ -112,6 +118,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + reader: mockUrlReader, }), ).rejects.toThrow('access error'); }); @@ -126,6 +133,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + reader: mockUrlReader, }); const app = express().use(router); @@ -155,6 +163,7 @@ describe('createRouter - working directory', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + reader: mockUrlReader, }); const app = express().use(router); @@ -228,6 +237,7 @@ describe('createRouter', () => { dockerClient: new Docker(), database: createDatabase(), catalogClient: createCatalogClient([template]), + reader: mockUrlReader, }); app = express().use(router); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 1645cb928d..3d47d5123b 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -38,16 +38,15 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { - TemplateActionRegistry, - templateEntityToSpec, -} from '../scaffolder/tasks/TemplateConverter'; -import { registerLegacyActions } from '../scaffolder/stages/legacy'; -import { getWorkingDirectory } from './helpers'; +import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; +import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; +import { createLegacyActions } from '../scaffolder/stages/legacy'; +import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { InputError, NotFoundError, PluginDatabaseManager, + UrlReader, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; import { @@ -55,6 +54,9 @@ import { TemplateEntityV1beta2, Entity, } from '@backstage/catalog-model'; +import { ScmIntegrations } from '@backstage/integration'; +import { TemplateAction } from '../scaffolder/actions'; +import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; export interface RouterOptions { preparers: PreparerBuilder; @@ -63,9 +65,11 @@ export interface RouterOptions { logger: Logger; config: Config; + reader: UrlReader; dockerClient: Docker; database: PluginDatabaseManager; catalogClient: CatalogApi; + actions?: TemplateAction[]; } function isAlpha1Template( @@ -95,15 +99,18 @@ export async function createRouter( publishers, logger: parentLogger, config, + reader, dockerClient, database, catalogClient, + actions, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); const entityClient = new CatalogEntityClient(catalogClient); + const integrations = ScmIntegrations.fromConfig(config); const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), @@ -117,13 +124,25 @@ export async function createRouter( workingDirectory, }); - registerLegacyActions(actionRegistry, { - dockerClient, - preparers, - publishers, - templaters, - catalogClient, - }); + const actionsToRegister = Array.isArray(actions) + ? actions + : [ + ...createLegacyActions({ + dockerClient, + preparers, + publishers, + templaters, + }), + ...createBuiltinActions({ + dockerClient, + integrations, + catalogClient, + templaters, + reader, + }), + ]; + + actionsToRegister.forEach(action => actionRegistry.register(action)); worker.start(); @@ -330,12 +349,7 @@ export async function createRouter( ) .post('/v2/tasks', async (req, res) => { const templateName: string = req.body.templateName; - const values: TemplaterValues = { - ...req.body.values, - destination: { - git: parseGitUrl(req.body.values.storePath), - }, - }; + const values: TemplaterValues = req.body.values; const template = await entityClient.findTemplate(templateName); let taskSpec; @@ -358,7 +372,10 @@ export async function createRouter( } } + const baseUrl = getEntityBaseUrl(template); + taskSpec = { + baseUrl, values, steps: template.spec.steps.map((step, index) => ({ ...step, diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index 1501a1cd5e..a41d088329 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -16,7 +16,7 @@ import React from 'react'; import { createDevApp } from '@backstage/dev-utils'; -import { discoveryApiRef, identityApiRef } from '@backstage/core'; +import { configApiRef, discoveryApiRef, identityApiRef } from '@backstage/core'; import { CatalogClient } from '@backstage/catalog-client'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { ScaffolderPage } from '../src/plugin'; @@ -30,9 +30,13 @@ createDevApp() }) .registerApi({ api: scaffolderApiRef, - deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, - factory: ({ discoveryApi, identityApi }) => - new ScaffolderClient({ discoveryApi, identityApi }), + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + configApi: configApiRef, + }, + factory: ({ discoveryApi, identityApi, configApi }) => + new ScaffolderClient({ discoveryApi, identityApi, configApi }), }) .addPage({ path: '/create', diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 6e26d056cc..f1dae43a40 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -34,6 +34,7 @@ "@backstage/catalog-model": "^0.7.2", "@backstage/config": "^0.1.3", "@backstage/core": "^0.6.3", + "@backstage/integration": "^0.5.0", "@backstage/plugin-catalog-react": "^0.1.0", "@backstage/theme": "^0.2.3", "@material-ui/core": "^4.11.0", diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index ac165e021a..2be4caa81f 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -20,8 +20,10 @@ import { createApiRef, DiscoveryApi, Observable, + ConfigApi, IdentityApi, } from '@backstage/core'; +import { ScmIntegrations } from '@backstage/integration'; import ObservableImpl from 'zen-observable'; import { ScaffolderTask, Status } from './types'; @@ -66,6 +68,10 @@ export interface ScaffolderApi { getTask(taskId: string): Promise; + getIntegrationsList(options: { + allowedHosts: string[]; + }): Promise<{ type: string; title: string; host: string }[]>; + streamLogs({ taskId, after, @@ -77,13 +83,31 @@ export interface ScaffolderApi { export class ScaffolderClient implements ScaffolderApi { private readonly discoveryApi: DiscoveryApi; private readonly identityApi: IdentityApi; + private readonly configApi: ConfigApi; constructor(options: { discoveryApi: DiscoveryApi; identityApi: IdentityApi; + configApi: ConfigApi; }) { this.discoveryApi = options.discoveryApi; this.identityApi = options.identityApi; + this.configApi = options.configApi; + } + + async getIntegrationsList(options: { allowedHosts: string[] }) { + const integrations = ScmIntegrations.fromConfig( + this.configApi.getConfig('integrations'), + ); + + return [ + ...integrations.azure.list(), + ...integrations.bitbucket.list(), + ...integrations.github.list(), + ...integrations.gitlab.list(), + ] + .map(c => ({ type: c.type, title: c.title, host: c.config.host })) + .filter(c => options.allowedHosts.includes(c.host)); } async getTemplateParameterSchema( diff --git a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx index eb51e86d59..876d6dfa66 100644 --- a/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx +++ b/plugins/scaffolder/src/components/MultistepJsonForm/MultistepJsonForm.tsx @@ -45,6 +45,8 @@ type Props = { onChange: (e: IChangeEvent) => void; onReset: () => void; onFinish: () => void; + widgets?: FormProps['widgets']; + fields?: FormProps['fields']; }; export const MultistepJsonForm = ({ @@ -53,6 +55,8 @@ export const MultistepJsonForm = ({ onChange, onReset, onFinish, + fields, + widgets, }: Props) => { const [activeStep, setActiveStep] = useState(0); @@ -67,32 +71,37 @@ export const MultistepJsonForm = ({ return ( <> - {steps.map(({ title, schema, ...formProps }) => ( - - - {title} - - -
{ - if (e.errors.length === 0) handleNext(); - }} - {...formProps} - {...transformSchemaToProps(schema)} - > - - -
- - - ))} + {steps.map(({ title, schema, ...formProps }) => { + return ( + + + {title} + + +
{ + if (e.errors.length === 0) handleNext(); + }} + {...formProps} + {...transformSchemaToProps(schema)} + > + + +
+ + + ); + })} {activeStep === steps.length && ( diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 3f6ceb64c7..ebce35a91f 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -30,7 +30,7 @@ import { WarningPanel, } from '@backstage/core'; import { useStarredEntities } from '@backstage/plugin-catalog-react'; -import { Button, Grid, Link, makeStyles, Typography } from '@material-ui/core'; +import { Box, Button, Link, makeStyles, Typography } from '@material-ui/core'; import StarIcon from '@material-ui/icons/Star'; import { EntityFilterGroupsProvider, useFilteredEntities } from '../../filter'; import { TemplateCard, TemplateCardProps } from '../TemplateCard'; @@ -46,6 +46,12 @@ const useStyles = makeStyles(theme => ({ gridTemplateColumns: '250px 1fr', gridColumnGap: theme.spacing(2), }, + templateGrid: { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fill, minmax(22em, 1fr))', + gridAutoRows: '1fr', + gridGap: theme.spacing(2), + }, })); const getTemplateCardProps = ( @@ -177,23 +183,13 @@ export const ScaffolderPageContents = () => { )} - + {matchingEntities && matchingEntities?.length > 0 && - matchingEntities.map(template => { - return ( - - - - ); - })} - + matchingEntities.map(template => ( + + ))} + diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index c38f4e09a5..abff0538ab 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -37,9 +37,18 @@ const useStyles = makeStyles(theme => ({ backgroundPosition: 0, }, description: { - height: 175, overflow: 'hidden', textOverflow: 'ellipsis', + display: '-webkit-box', + '-webkit-line-clamp': 10, + '-webkit-box-orient': 'vertical', + }, + card: { + display: 'flex', + flexDirection: 'column', + }, + cardContent: { + flexGrow: 1, }, })); @@ -69,12 +78,12 @@ export const TemplateCard = ({ }); return ( - +
{type} {title}
- + {tags?.map(tag => ( ))} diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 1d66e07c6a..1b11b704ae 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -22,7 +22,7 @@ import { act } from 'react-dom/test-utils'; import { MemoryRouter, Route } from 'react-router'; import { ScaffolderApi, scaffolderApiRef } from '../../api'; import { rootRouteRef } from '../../routes'; -import { TemplatePage } from './TemplatePage'; +import { TemplatePage, createValidator } from './TemplatePage'; jest.mock('react-router-dom', () => { return { @@ -36,6 +36,7 @@ jest.mock('react-router-dom', () => { const scaffolderApiMock: jest.Mocked = { scaffold: jest.fn(), getTemplateParameterSchema: jest.fn(), + getIntegrationsList: jest.fn(), getTask: jest.fn(), streamLogs: jest.fn(), }; @@ -122,3 +123,38 @@ describe('TemplatePage', () => { expect(rendered.queryByText('This is root')).toBeInTheDocument(); }); }); + +describe('createValidator', () => { + it('should validate deep schema', () => { + const validator = createValidator({ + type: 'object', + properties: { + foo: { + type: 'object', + properties: { + bar: { + type: 'string', + 'ui:field': 'RepoUrlPicker', + }, + }, + }, + }, + }); + + const errors = { foo: { bar: { addError: jest.fn() } } }; + validator({ foo: { bar: 'github.com?owner=a' } }, errors as any); + expect(errors.foo.bar.addError).toHaveBeenCalledWith( + 'Incomplete repository location provided', + ); + jest.resetAllMocks(); + + validator({ foo: { bar: 'github.com?repo=b' } }, errors as any); + expect(errors.foo.bar.addError).toHaveBeenCalledWith( + 'Incomplete repository location provided', + ); + jest.resetAllMocks(); + + validator({ foo: { bar: 'github.com?owner=a&repo=b' } }, errors as any); + expect(errors.foo.bar.addError).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 1c8a662acb..5d173173ee 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -33,6 +33,8 @@ import { useAsync } from 'react-use'; import { scaffolderApiRef } from '../../api'; import { rootRouteRef } from '../../routes'; import { MultistepJsonForm } from '../MultistepJsonForm'; +import { RepoUrlPicker } from '../fields'; +import { JsonObject } from '@backstage/config'; const useTemplateParameterSchema = (templateName: string) => { const scaffolderApi = useApi(scaffolderApiRef); @@ -48,12 +50,69 @@ const useTemplateParameterSchema = (templateName: string) => { return { schema: value, loading, error }; }; +function isObject(obj: unknown): obj is JsonObject { + return typeof obj === 'object' && obj !== null && !Array.isArray(obj); +} + +export const createValidator = (rootSchema: JsonObject) => { + function validate( + schema: JsonObject, + formData: JsonObject, + errors: FormValidation, + ) { + const schemaProps = schema.properties; + if (!isObject(schemaProps)) { + return; + } + + for (const [key, propData] of Object.entries(formData)) { + const propErrors = errors[key]; + + if (isObject(propData)) { + const propSchemaProps = schemaProps[key]; + if (isObject(propSchemaProps)) { + validate( + propSchemaProps, + propData as JsonObject, + propErrors as FormValidation, + ); + } + } else { + const propSchema = schemaProps[key]; + if ( + isObject(propSchema) && + propSchema['ui:field'] === 'RepoUrlPicker' + ) { + try { + const { host, searchParams } = new URL(`https://${propData}`); + if ( + !host || + !searchParams.get('owner') || + !searchParams.get('repo') + ) { + propErrors.addError('Incomplete repository location provided'); + } + } catch { + propErrors.addError('Unable to parse the Repository URL'); + } + } + } + } + } + + return (formData: JsonObject, errors: FormValidation) => { + validate(rootSchema, formData, errors); + return errors; + }; +}; + const storePathValidator = ( formData: { storePath?: string }, errors: FormValidation, ) => { const { storePath } = formData; if (!storePath) { + errors.storePath.addError('Store path is required and not present'); return errors; } @@ -131,19 +190,24 @@ export const TemplatePage = () => { { - // TODO: Using this workaround to keep storePath validation, but we should replace - // it with a custom store path selection widget + // TODO: Can delete this function when the migration from v1 to v2 beta is completed + // And just have the default validator for all fields. if ((step.schema as any)?.properties?.storePath) { return { ...step, validate: (a, b) => storePathValidator(a, b), }; } - return step; + + return { + ...step, + validate: createValidator(step.schema), + }; })} /> diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx new file mode 100644 index 0000000000..31b4c4fa9b --- /dev/null +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -0,0 +1,217 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useCallback, useEffect } from 'react'; +import { Field } from '@rjsf/core'; +import { useApi, Progress } from '@backstage/core'; +import { scaffolderApiRef } from '../../../api'; +import { useAsync } from 'react-use'; +import Select from '@material-ui/core/Select'; +import InputLabel from '@material-ui/core/InputLabel'; +import Input from '@material-ui/core/Input'; +import FormControl from '@material-ui/core/FormControl'; +import FormHelperText from '@material-ui/core/FormHelperText'; + +function splitFormData(url: string | undefined) { + let host = undefined; + let owner = undefined; + let repo = undefined; + let organization = undefined; + + try { + if (url) { + const parsed = new URL(`https://${url}`); + host = parsed.host; + owner = parsed.searchParams.get('owner') || undefined; + repo = parsed.searchParams.get('repo') || undefined; + // This is azure dev ops specific. not used for any other provider. + organization = parsed.searchParams.get('organization') || undefined; + } + } catch { + /* ok */ + } + + return { host, owner, repo, organization }; +} + +function serializeFormData(data: { + host?: string; + owner?: string; + repo?: string; + organization?: string; +}) { + if (!data.host) { + return undefined; + } + const params = new URLSearchParams(); + if (data.owner) { + params.set('owner', data.owner); + } + if (data.repo) { + params.set('repo', data.repo); + } + if (data.organization) { + params.set('organization', data.organization); + } + + return `${data.host}?${params.toString()}`; +} + +export const RepoUrlPicker: Field = ({ + onChange, + uiSchema, + rawErrors, + formData, +}) => { + const api = useApi(scaffolderApiRef); + const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; + + const { value: integrations, loading } = useAsync(async () => { + return await api.getIntegrationsList({ allowedHosts }); + }); + + const { host, owner, repo, organization } = splitFormData(formData); + const updateHost = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ + host: evt.target.value as string, + owner, + repo, + organization, + }), + ), + [onChange, owner, repo, organization], + ); + + const updateOwner = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ + host, + owner: evt.target.value as string, + repo, + organization, + }), + ), + [onChange, host, repo, organization], + ); + + const updateRepo = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ + host, + owner, + repo: evt.target.value as string, + organization, + }), + ), + [onChange, host, owner, organization], + ); + + const updateOrganization = useCallback( + (evt: React.ChangeEvent<{ name?: string; value: unknown }>) => + onChange( + serializeFormData({ + host, + owner, + repo, + organization: evt.target.value as string, + }), + ), + [onChange, host, owner, repo], + ); + + useEffect(() => { + if (host === undefined && integrations?.length) { + onChange( + serializeFormData({ + host: integrations[0].host, + owner, + repo, + organization, + }), + ); + } + }, [onChange, integrations, host, owner, repo, organization]); + + if (loading) { + return ; + } + + return ( + <> + 0 && !host} + > + Host + + + The host where the repository will be created + + + {host === 'dev.azure.com' && ( + 0 && !organization} + > + Organization + + The name of the organization + + )} + 0 && !owner} + > + Owner + + + The organization, user or project that this repo will belong to + + + 0 && !repo} + > + Repository + + The name of the repository + + + ); +}; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts new file mode 100644 index 0000000000..b0f3df61d9 --- /dev/null +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { RepoUrlPicker } from './RepoUrlPicker'; diff --git a/plugins/scaffolder/src/components/fields/index.ts b/plugins/scaffolder/src/components/fields/index.ts new file mode 100644 index 0000000000..fc3e70378d --- /dev/null +++ b/plugins/scaffolder/src/components/fields/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './RepoUrlPicker'; diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index a6ba1a9899..8ba03f4682 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -19,6 +19,7 @@ import { createApiFactory, discoveryApiRef, identityApiRef, + configApiRef, createRoutableExtension, } from '@backstage/core'; import { rootRouteRef } from './routes'; @@ -29,9 +30,13 @@ export const scaffolderPlugin = createPlugin({ apis: [ createApiFactory({ api: scaffolderApiRef, - deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, - factory: ({ discoveryApi, identityApi }) => - new ScaffolderClient({ discoveryApi, identityApi }), + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + configApi: configApiRef, + }, + factory: ({ discoveryApi, identityApi, configApi }) => + new ScaffolderClient({ discoveryApi, identityApi, configApi }), }), ], routes: {