From 6a09ed555d092aec01700375bc18b89d35f08b51 Mon Sep 17 00:00:00 2001 From: blam Date: Sat, 19 Jun 2021 02:17:01 +0200 Subject: [PATCH 01/18] breaking: removing alphav1 support for templates Signed-off-by: blam --- packages/backend/src/plugins/scaffolder.ts | 30 +- .../src/kinds/TemplateEntityV1alpha1.test.ts | 106 ------ .../src/kinds/TemplateEntityV1alpha1.ts | 36 -- packages/catalog-model/src/kinds/index.ts | 5 - .../kinds/Template.v1alpha1.schema.json | 99 ------ .../src/ingestion/LocationReaders.ts | 1 + .../BuiltinKindsEntityProcessor.test.ts | 19 +- .../processors/BuiltinKindsEntityProcessor.ts | 6 +- .../DefaultCatalogProcessingOrchestrator.ts | 1 + .../sample-templates/remote-templates.yaml | 2 +- .../src/lib/catalog/CatalogEntityClient.ts | 9 +- .../actions/builtin/createBuiltinActions.ts | 18 +- .../actions/builtin/fetch/cookiecutter.ts | 110 +++++- .../publish => actions/builtin}/helpers.ts | 41 +++ .../actions/builtin/publish/azure.ts | 2 +- .../actions/builtin/publish/bitbucket.ts | 2 +- .../actions/builtin/publish/github.ts | 2 +- .../actions/builtin/publish/gitlab.ts | 2 +- .../src/scaffolder/index.ts | 3 - .../src/scaffolder/jobs/index.ts | 17 - .../src/scaffolder/jobs/logger.test.ts | 49 --- .../src/scaffolder/jobs/logger.ts | 48 --- .../src/scaffolder/jobs/processor.test.ts | 329 ----------------- .../src/scaffolder/jobs/processor.ts | 180 ---------- .../src/scaffolder/jobs/types.ts | 68 ---- .../src/scaffolder/stages/helpers.test.ts | 332 ------------------ .../src/scaffolder/stages/helpers.ts | 62 ---- .../src/scaffolder/stages/index.ts | 19 - .../src/scaffolder/stages/legacy.ts | 105 ------ .../scaffolder/stages/prepare/azure.test.ts | 116 ------ .../src/scaffolder/stages/prepare/azure.ts | 63 ---- .../stages/prepare/bitbucket.test.ts | 113 ------ .../scaffolder/stages/prepare/bitbucket.ts | 82 ----- .../scaffolder/stages/prepare/file.test.ts | 79 ----- .../src/scaffolder/stages/prepare/file.ts | 37 -- .../scaffolder/stages/prepare/github.test.ts | 97 ----- .../src/scaffolder/stages/prepare/github.ts | 71 ---- .../scaffolder/stages/prepare/gitlab.test.ts | 82 ----- .../src/scaffolder/stages/prepare/gitlab.ts | 62 ---- .../src/scaffolder/stages/prepare/index.ts | 23 -- .../stages/prepare/preparers.test.ts | 50 --- .../scaffolder/stages/prepare/preparers.ts | 81 ----- .../src/scaffolder/stages/prepare/types.ts | 40 --- .../scaffolder/stages/publish/azure.test.ts | 89 ----- .../src/scaffolder/stages/publish/azure.ts | 83 ----- .../stages/publish/bitbucket.test.ts | 228 ------------ .../scaffolder/stages/publish/bitbucket.ts | 207 ----------- .../scaffolder/stages/publish/github.test.ts | 315 ----------------- .../src/scaffolder/stages/publish/github.ts | 178 ---------- .../scaffolder/stages/publish/gitlab.test.ts | 151 -------- .../src/scaffolder/stages/publish/gitlab.ts | 106 ------ .../src/scaffolder/stages/publish/index.ts | 27 -- .../stages/publish/publishers.test.ts | 127 ------- .../scaffolder/stages/publish/publishers.ts | 113 ------ .../src/scaffolder/stages/publish/types.ts | 47 --- .../stages/templater/cookiecutter.test.ts | 279 --------------- .../stages/templater/cookiecutter.ts | 107 ------ .../scaffolder/stages/templater/cra/index.ts | 159 --------- .../scaffolder/stages/templater/helpers.ts | 75 ---- .../src/scaffolder/stages/templater/index.ts | 20 -- .../stages/templater/templaters.test.ts | 43 --- .../scaffolder/stages/templater/templaters.ts | 39 -- .../src/scaffolder/stages/templater/types.ts | 73 ---- .../src/scaffolder/tasks/TemplateConverter.ts | 101 ------ .../scaffolder-backend/src/service/router.ts | 113 +----- 65 files changed, 194 insertions(+), 5185 deletions(-) delete mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts delete mode 100644 packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts delete mode 100644 packages/catalog-model/src/schema/kinds/Template.v1alpha1.schema.json rename plugins/scaffolder-backend/src/scaffolder/{stages/publish => actions/builtin}/helpers.ts (77%) delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/jobs/types.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts delete mode 100644 plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 09b3732301..619134a990 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -14,19 +14,9 @@ * limitations under the License. */ -import { - DockerContainerRunner, - SingleHostDiscovery, -} from '@backstage/backend-common'; +import { DockerContainerRunner } from '@backstage/backend-common'; import { CatalogClient } from '@backstage/catalog-client'; -import { - CookieCutter, - CreateReactAppTemplater, - createRouter, - Preparers, - Publishers, - Templaters, -} from '@backstage/plugin-scaffolder-backend'; +import { createRouter } from '@backstage/plugin-scaffolder-backend'; import Docker from 'dockerode'; import { Router } from 'express'; import type { PluginEnvironment } from '../types'; @@ -36,27 +26,15 @@ export default async function createPlugin({ config, database, reader, + discovery, }: PluginEnvironment): Promise { const dockerClient = new Docker(); const containerRunner = new DockerContainerRunner({ dockerClient }); - const cookiecutterTemplater = new CookieCutter({ containerRunner }); - const craTemplater = new CreateReactAppTemplater({ containerRunner }); - const templaters = new Templaters(); - - templaters.register('cookiecutter', cookiecutterTemplater); - templaters.register('cra', craTemplater); - - const preparers = await Preparers.fromConfig(config, { logger }); - const publishers = await Publishers.fromConfig(config, { logger }); - - const discovery = SingleHostDiscovery.fromConfig(config); const catalogClient = new CatalogClient({ discoveryApi: discovery }); return await createRouter({ - preparers, - templaters, - publishers, + containerRunner, logger, config, database, diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts deleted file mode 100644 index f03519a3c6..0000000000 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - TemplateEntityV1alpha1, - templateEntityV1alpha1Validator as validator, -} from './TemplateEntityV1alpha1'; - -describe('templateEntityV1alpha1Validator', () => { - let entity: TemplateEntityV1alpha1; - - beforeEach(() => { - entity = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - name: 'test', - }, - spec: { - type: 'website', - templater: 'cookiecutter', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-a@example.com', - }, - }; - }); - - it('happy path: accepts valid data', async () => { - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('silently accepts v1beta1 as well', async () => { - (entity as any).apiVersion = 'backstage.io/v1beta1'; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('ignores unknown apiVersion', async () => { - (entity as any).apiVersion = 'backstage.io/v1beta0'; - await expect(validator.check(entity)).resolves.toBe(false); - }); - - it('ignores unknown kind', async () => { - (entity as any).kind = 'Wizard'; - await expect(validator.check(entity)).resolves.toBe(false); - }); - - it('rejects missing type', async () => { - delete (entity as any).spec.type; - await expect(validator.check(entity)).rejects.toThrow(/type/); - }); - - it('accepts any other type', async () => { - (entity as any).spec.type = 'hallo'; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects empty type', async () => { - (entity as any).spec.type = ''; - await expect(validator.check(entity)).rejects.toThrow(/type/); - }); - - it('rejects missing templater', async () => { - (entity as any).spec.templater = ''; - await expect(validator.check(entity)).rejects.toThrow(/templater/); - }); - it('accepts missing owner', async () => { - delete (entity as any).spec.owner; - await expect(validator.check(entity)).resolves.toBe(true); - }); - it('rejects empty owner', async () => { - (entity as any).spec.owner = ''; - await expect(validator.check(entity)).rejects.toThrow(/owner/); - }); - it('rejects wrong type owner', async () => { - (entity as any).spec.owner = 5; - await expect(validator.check(entity)).rejects.toThrow(/owner/); - }); -}); diff --git a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts b/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts deleted file mode 100644 index bdc6f35df2..0000000000 --- a/packages/catalog-model/src/kinds/TemplateEntityV1alpha1.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { Entity } from '../entity/Entity'; -import schema from '../schema/kinds/Template.v1alpha1.schema.json'; -import type { JSONSchema } from '../types'; -import { ajvCompiledJsonSchemaValidator } from './util'; - -export interface TemplateEntityV1alpha1 extends Entity { - apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; - kind: 'Template'; - spec: { - type: string; - templater: string; - path?: string; - schema: JSONSchema; - owner?: string; - }; -} - -export const templateEntityV1alpha1Validator = ajvCompiledJsonSchemaValidator( - schema, -); diff --git a/packages/catalog-model/src/kinds/index.ts b/packages/catalog-model/src/kinds/index.ts index ccdf0051db..be9f7a0d6a 100644 --- a/packages/catalog-model/src/kinds/index.ts +++ b/packages/catalog-model/src/kinds/index.ts @@ -50,11 +50,6 @@ export type { SystemEntityV1alpha1 as SystemEntity, SystemEntityV1alpha1, } from './SystemEntityV1alpha1'; -export { templateEntityV1alpha1Validator } from './TemplateEntityV1alpha1'; -export type { - TemplateEntityV1alpha1 as TemplateEntity, - TemplateEntityV1alpha1, -} from './TemplateEntityV1alpha1'; export { templateEntityV1beta2Validator } from './TemplateEntityV1beta2'; export type { TemplateEntityV1beta2 } from './TemplateEntityV1beta2'; export type { KindValidator } from './types'; diff --git a/packages/catalog-model/src/schema/kinds/Template.v1alpha1.schema.json b/packages/catalog-model/src/schema/kinds/Template.v1alpha1.schema.json deleted file mode 100644 index 5a0c0efa73..0000000000 --- a/packages/catalog-model/src/schema/kinds/Template.v1alpha1.schema.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "TemplateV1alpha1", - "description": "A Template describes a skeleton for use with the Scaffolder. It is used for describing what templating library is supported, and also for documenting the variables that the template requires using JSON Forms Schema.", - "examples": [ - { - "apiVersion": "backstage.io/v1alpha1", - "kind": "Template", - "metadata": { - "name": "react-ssr-template", - "title": "React SSR Template", - "description": "Next.js application skeleton for creating isomorphic web applications.", - "tags": ["recommended", "react"] - }, - "spec": { - "owner": "artist-relations-team", - "templater": "cookiecutter", - "type": "website", - "path": ".", - "schema": { - "required": ["component-id", "description"], - "properties": { - "component_id": { - "title": "Name", - "type": "string", - "description": "Unique name of the component" - }, - "description": { - "title": "Description", - "type": "string", - "description": "Description of the component" - } - } - } - } - } - ], - "allOf": [ - { - "$ref": "Entity" - }, - { - "type": "object", - "required": ["spec"], - "properties": { - "apiVersion": { - "enum": ["backstage.io/v1alpha1", "backstage.io/v1beta1"] - }, - "kind": { - "enum": ["Template"] - }, - "metadata": { - "type": "object", - "properties": { - "title": { - "type": "string", - "description": "The nice display name for the template. This field is required as is used to reference the template to the user instead of the metadata.name field.", - "examples": ["React SSR Template"], - "minLength": 1 - } - } - }, - "spec": { - "type": "object", - "required": ["type", "templater", "schema"], - "properties": { - "type": { - "type": "string", - "description": "The type of component created by the template. The software catalog accepts any type value, but an organization should take great care to establish a proper taxonomy for these. Tools including Backstage itself may read this field and behave differently depending on its value. For example, a website type component may present tooling in the Backstage interface that is specific to just websites.", - "examples": ["service", "website", "library"], - "minLength": 1 - }, - "templater": { - "type": "string", - "description": "The templating library that is supported by the template skeleton.", - "examples": ["cookiecutter"], - "minLength": 1 - }, - "path": { - "type": "string", - "description": "The string location where the templater should be run if it is not on the same level as the template.yaml definition.", - "examples": ["./cookiecutter/skeleton"], - "minLength": 1 - }, - "schema": { - "type": "object", - "description": "The JSONSchema describing the inputs for the template." - }, - "owner": { - "type": "string", - "description": "The user (or group) owner of the template", - "minLength": 1 - } - } - } - } - } - ] -} diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 624c7aaa61..86a8ebb9f2 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -265,6 +265,7 @@ export class LocationReaders implements LocationReader { )}, ${e}`; emit(result.inputError(item.location, message)); logger.warn(message); + logger.debug(e.stack); return undefined; } } diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts index bdc6d463c2..33beffed0d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts @@ -21,7 +21,7 @@ import { GroupEntity, ResourceEntity, SystemEntity, - TemplateEntity, + TemplateEntityV1beta2, UserEntity, } from '@backstage/catalog-model'; import { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; @@ -522,22 +522,13 @@ describe('BuiltinKindsEntityProcessor', () => { }); }); it('generates relations for template entities', async () => { - const entity: TemplateEntity = { - apiVersion: 'backstage.io/v1alpha1', + const entity: TemplateEntityV1beta2 = { + apiVersion: 'backstage.io/v1beta2', kind: 'Template', metadata: { name: 'n' }, spec: { - schema: { - properties: { - description: { - title: 'd', - type: 'string', - description: 'des', - }, - }, - }, - templater: 'cookiecutter', - path: '.', + parameters: {}, + steps: [], type: 'service', owner: 'o', }, diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index b8805ab9ba..bcabc00f32 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -46,8 +46,7 @@ import { resourceEntityV1alpha1Validator, SystemEntity, systemEntityV1alpha1Validator, - TemplateEntity, - templateEntityV1alpha1Validator, + TemplateEntityV1beta2, templateEntityV1beta2Validator, UserEntity, userEntityV1alpha1Validator, @@ -62,7 +61,6 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { resourceEntityV1alpha1Validator, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, - templateEntityV1alpha1Validator, templateEntityV1beta2Validator, userEntityV1alpha1Validator, systemEntityV1alpha1Validator, @@ -136,7 +134,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { * Emit relations for the Template kind */ if (entity.kind === 'Template') { - const template = entity as TemplateEntity; + const template = entity as TemplateEntityV1beta2; doEmit( template.spec.owner, { defaultKind: 'Group', defaultNamespace: selfRef.namespace }, diff --git a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts index bb23b6afb3..235673db03 100644 --- a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts @@ -124,6 +124,7 @@ export class DefaultCatalogProcessingOrchestrator }; } catch (error) { this.options.logger.warn(error.message); + this.options.logger.debug(error.stack); return { ok: false, errors: collector.results().errors.concat(error), diff --git a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml index 7846a0994e..b89b21d5e3 100644 --- a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml @@ -6,4 +6,4 @@ metadata: spec: type: url targets: - - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + # - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml diff --git a/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts b/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts index 9f3a754485..430afa5b5d 100644 --- a/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts +++ b/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts @@ -14,10 +14,7 @@ * limitations under the License. */ -import { - TemplateEntityV1alpha1, - TemplateEntityV1beta2, -} from '@backstage/catalog-model'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { CatalogApi } from '@backstage/catalog-client'; import { ConflictError, NotFoundError } from '@backstage/errors'; @@ -35,7 +32,7 @@ export class CatalogEntityClient { async findTemplate( templateName: string, options?: { token?: string }, - ): Promise { + ): Promise { const { items: templates } = (await this.catalogClient.getEntities( { filter: { @@ -44,7 +41,7 @@ export class CatalogEntityClient { }, }, options, - )) as { items: (TemplateEntityV1alpha1 | TemplateEntityV1beta2)[] }; + )) as { items: TemplateEntityV1beta2[] }; if (templates.length !== 1) { if (templates.length > 1) { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index b3757571ff..08d7bbf824 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -14,15 +14,15 @@ * limitations under the License. */ -import { UrlReader } from '@backstage/backend-common'; +import { ContainerRunner, UrlReader } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; import { ScmIntegrations } from '@backstage/integration'; import { Config } from '@backstage/config'; -import { TemplaterBuilder } from '../../stages'; import { - createCatalogRegisterAction, createCatalogWriteAction, + createCatalogRegisterAction, } from './catalog'; + import { createDebugLogAction } from './debug'; import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch'; import { @@ -41,10 +41,16 @@ export const createBuiltinActions = (options: { reader: UrlReader; integrations: ScmIntegrations; catalogClient: CatalogApi; - templaters: TemplaterBuilder; + containerRunner: ContainerRunner; config: Config; }) => { - const { reader, integrations, templaters, catalogClient, config } = options; + const { + reader, + integrations, + containerRunner, + catalogClient, + config, + } = options; return [ createFetchPlainAction({ @@ -54,7 +60,7 @@ export const createBuiltinActions = (options: { createFetchCookiecutterAction({ reader, integrations, - templaters, + containerRunner, }), createPublishGithubAction({ integrations, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index 62dfc20125..e5720204c7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -14,22 +14,116 @@ * limitations under the License. */ -import { UrlReader, resolveSafeChildPath } from '@backstage/backend-common'; -import { JsonObject } from '@backstage/config'; +import { + ContainerRunner, + UrlReader, + resolveSafeChildPath, +} from '@backstage/backend-common'; +import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { ScmIntegrations } from '@backstage/integration'; +import commandExists from 'command-exists'; import fs from 'fs-extra'; -import { resolve as resolvePath } from 'path'; -import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater'; +import path, { resolve as resolvePath } from 'path'; +import { Writable } from 'stream'; +import { runCommand } from '../helpers'; import { createTemplateAction } from '../../createTemplateAction'; import { fetchContents } from './helpers'; +export class CookiecutterRunner { + private readonly containerRunner: ContainerRunner; + + constructor({ containerRunner }: { containerRunner: ContainerRunner }) { + this.containerRunner = containerRunner; + } + + private async fetchTemplateCookieCutter( + directory: string, + ): Promise> { + try { + return await fs.readJSON(path.join(directory, 'cookiecutter.json')); + } catch (ex) { + if (ex.code !== 'ENOENT') { + throw ex; + } + + return {}; + } + } + + public async run({ + workspacePath, + values, + logStream, + }: { + workspacePath: string; + values: JsonObject; + logStream: Writable; + }): Promise { + const templateDir = path.join(workspacePath, 'template'); + const intermediateDir = path.join(workspacePath, 'intermediate'); + await fs.ensureDir(intermediateDir); + const resultDir = path.join(workspacePath, 'result'); + + // First lets grab the default cookiecutter.json file + const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); + + const { imageName, ...valuesForCookieCutterJson } = values; + const cookieInfo = { + ...cookieCutterJson, + ...valuesForCookieCutterJson, + }; + + await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); + + // Directories to bind on container + const mountDirs = { + [templateDir]: '/input', + [intermediateDir]: '/output', + }; + + // the command-exists package returns `true` or throws an error + const cookieCutterInstalled = await commandExists('cookiecutter').catch( + () => false, + ); + if (cookieCutterInstalled) { + await runCommand({ + command: 'cookiecutter', + args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'], + logStream, + }); + } else { + await this.containerRunner.runContainer({ + imageName: (imageName as string) ?? 'spotify/backstage-cookiecutter', + command: 'cookiecutter', + args: ['--no-input', '-o', '/output', '/input', '--verbose'], + mountDirs, + workingDir: '/input', + // Set the home directory inside the container as something that applications can + // write to, otherwise they will just fail trying to write to / + envVars: { HOME: '/tmp' }, + logStream, + }); + } + + // if cookiecutter was successful, intermediateDir will contain + // exactly one directory. + const [generated] = await fs.readdir(intermediateDir); + + if (generated === undefined) { + throw new Error('No data generated by cookiecutter'); + } + + await fs.move(path.join(intermediateDir, generated), resultDir); + } +} + export function createFetchCookiecutterAction(options: { reader: UrlReader; integrations: ScmIntegrations; - templaters: TemplaterBuilder; + containerRunner: ContainerRunner; }) { - const { reader, templaters, integrations } = options; + const { reader, containerRunner, integrations } = options; return createTemplateAction<{ url: string; @@ -121,9 +215,9 @@ export function createFetchCookiecutterAction(options: { outputPath: templateContentsDir, }); - const cookiecutter = templaters.get('cookiecutter'); + const cookiecutter = new CookiecutterRunner({ containerRunner }); const values = { - ...(ctx.input.values as TemplaterValues), + ...ctx.input.values, _copy_without_render: ctx.input.copyWithoutRender, _extensions: ctx.input.extensions, imageName: ctx.input.imageName, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts similarity index 77% rename from plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts rename to plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts index c974c466d4..a024be4f67 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD:plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts * Copyright 2020 The Backstage Authors +======= + * Copyright 2021 The Backstage Authors +>>>>>>> breaking: removing alphav1 support for templates:plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +18,48 @@ * limitations under the License. */ +import { spawn } from 'child_process'; +import { PassThrough, Writable } from 'stream'; import globby from 'globby'; import { Logger } from 'winston'; import { Git } from '@backstage/backend-common'; import { Octokit } from '@octokit/rest'; +export type RunCommandOptions = { + command: string; + args: string[]; + logStream?: Writable; +}; + +export const runCommand = async ({ + command, + args, + logStream = new PassThrough(), +}: RunCommandOptions) => { + await new Promise((resolve, reject) => { + const process = spawn(command, args); + + process.stdout.on('data', stream => { + logStream.write(stream); + }); + + process.stderr.on('data', stream => { + logStream.write(stream); + }); + + process.on('error', error => { + return reject(error); + }); + + process.on('close', code => { + if (code !== 0) { + return reject(`Command ${command} failed, exit code: ${code}`); + } + return resolve(); + }); + }); +}; + export async function initRepoAndPush({ dir, remoteUrl, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts index 8b8f6ce3e5..c58dd69424 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts @@ -16,7 +16,7 @@ import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; import { getRepoSourceDirectory, parseRepoUrl } from './util'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 1dc80f370c..ffd2e4d6a7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -20,7 +20,7 @@ import { ScmIntegrationRegistry, } from '@backstage/integration'; import fetch from 'cross-fetch'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; import { createTemplateAction } from '../../createTemplateAction'; import { getRepoSourceDirectory, parseRepoUrl } from './util'; import { Config } from '@backstage/config'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts index 9f693a96d8..05301bda85 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.ts @@ -22,7 +22,7 @@ import { Octokit } from '@octokit/rest'; import { enableBranchProtectionOnDefaultRepoBranch, initRepoAndPush, -} from '../../../stages/publish/helpers'; +} from '../helpers'; import { getRepoSourceDirectory, parseRepoUrl } from './util'; import { createTemplateAction } from '../../createTemplateAction'; import { Config } from '@backstage/config'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts index 1cd77ebe2a..b99d40b1f7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.ts @@ -17,7 +17,7 @@ import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { Gitlab } from '@gitbeaker/node'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; import { getRepoSourceDirectory, parseRepoUrl } from './util'; import { createTemplateAction } from '../../createTemplateAction'; import { Config } from '@backstage/config'; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index 038873b42c..284e372b4c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -13,7 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { createLegacyActions } from './stages/legacy'; -export * from './stages'; -export * from './jobs'; export * from './actions'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts deleted file mode 100644 index 8ebffa2026..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export * from './processor'; -export * from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts deleted file mode 100644 index e5d298163b..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { makeLogStream } from './logger'; - -describe('Logger', () => { - const mockMeta = { test: 'blob' }; - - it('should return empty log lines by default', async () => { - const { log } = makeLogStream(mockMeta); - - expect(log).toEqual([]); - }); - it('should add lines to the log when using the logger that is returned', async () => { - const { logger, log } = makeLogStream(mockMeta); - - logger.info('TEST LINE'); - logger.warn('WARN LINE'); - - const [first, second] = log; - expect(log.length).toBe(2); - - expect(first).toContain('info'); - expect(first).toContain('TEST LINE'); - - expect(second).toContain('warn'); - expect(second).toContain('WARN LINE'); - }); - - it('should add lines from writing to the stream that is returned', async () => { - const { stream, log } = makeLogStream(mockMeta); - const textLine = 'SOMETHING'; - stream.write(textLine); - - expect(log).toContain(textLine); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts deleted file mode 100644 index 9ffdb33f37..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { PassThrough } from 'stream'; -import * as winston from 'winston'; -import { JsonValue } from '@backstage/config'; - -export const makeLogStream = (meta: Record) => { - const log: string[] = []; - - // Create an empty stream to collect all the log lines into - // one variable for the API. - const stream = new PassThrough(); - stream.on('data', chunk => { - const textValue = chunk.toString().trim(); - if (textValue?.length > 1) log.push(textValue); - }); - - const logger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: meta, - }); - - logger.add(new winston.transports.Stream({ stream })); - - return { - log, - stream, - logger, - }; -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts deleted file mode 100644 index 2bb2171899..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import parseGitUrl from 'git-url-parse'; -import mockFs from 'mock-fs'; -import os from 'os'; -import { RequiredTemplateValues } from '../stages/templater'; -import { makeLogStream } from './logger'; -import { JobProcessor } from './processor'; -import { StageInput } from './types'; - -describe('JobProcessor', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'example@email.com', - }, - }; - - const mockValues: RequiredTemplateValues = { - owner: 'blobby', - storePath: 'https://github.com/backstage/mock-repo', - destination: { - git: parseGitUrl('https://github.com/backstage/mock-repo'), - }, - }; - - const workingDirectory = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - - // NOTE(freben): Without this line, mock-fs makes winston/logform break. - // There are a number of reported issues with logform and its use of dynamic - // strings for imports. It confuses webpack. The basic fix is to trigger - // those imports before mock-fs runs. I wanted to add a mock dir - // 'node_modules': mockFs.passthrough(), but that doesn't seem to be a thing - // in mock-fs 4. - // Probable REAL fix: https://github.com/winstonjs/logform/pull/117 - makeLogStream({}); - - beforeEach(() => { - mockFs({ - [workingDirectory]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - - describe('create', () => { - it('creates should create a new job with a unique id', async () => { - const processor = new JobProcessor(workingDirectory); - - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages: [], - }); - - expect(job.id).toMatch( - /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i, - ); - }); - - it('should setup the correct context for the job', async () => { - const processor = new JobProcessor(workingDirectory); - - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages: [], - }); - - expect(job.context.entity).toBe(mockEntity); - expect(job.context.values).toBe(mockValues); - }); - - it('should set the status as pending', async () => { - const processor = new JobProcessor(workingDirectory); - - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages: [], - }); - - expect(job.status).toBe('PENDING'); - }); - - it('should create the correct stages', async () => { - const stages: StageInput[] = [ - { - name: 'Do something cool step 1', - handler: jest.fn(), - }, - { - name: 'Do something cool step 2', - handler: jest.fn(), - }, - ]; - - const processor = new JobProcessor(workingDirectory); - - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages, - }); - - expect(job.stages).toHaveLength(stages.length); - - for (let i = 0; i < job.stages.length; i++) { - expect(job.stages[i].name).toBe(stages[i].name); - expect(job.stages[i].status).toBe('PENDING'); - } - }); - }); - - describe('get', () => { - it('return undefined for when the job does not exist', () => { - const processor = new JobProcessor(workingDirectory); - expect(processor.get('123')).not.toBeDefined(); - }); - - it('should return the exact same instance of the job when one is created', async () => { - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages: [], - }); - - expect(processor.get(job.id)).toBe(job); - }); - }); - - describe('process', () => { - it('throws an error when the status of the job is not in pending state', async () => { - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages: [], - }); - - job.status = 'STARTED'; - - await expect(processor.run(job)).rejects.toThrow( - /Job is not in a 'PENDING' state/, - ); - }); - - it('will call each of the handlers in the stages', async () => { - const stages: StageInput[] = [ - { - name: 'c/o', - handler: jest.fn(), - }, - { - name: 'g/p', - handler: jest.fn(), - }, - ]; - - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages, - }); - - await processor.run(job); - - for (const stage of stages) { - expect(stage.handler).toHaveBeenCalled(); - } - }); - - it('should set all stages to complete and the job to complete when finishes without errors', async () => { - const stages: StageInput[] = [ - { - name: 'c/o', - handler: jest.fn(), - }, - { - name: 'g/p', - handler: jest.fn(), - }, - ]; - - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages, - }); - - await processor.run(job); - - for (const stage of job.stages) { - expect(stage.status).toBe('COMPLETED'); - } - - expect(job.status).toBe('COMPLETED'); - }); - - it('should merge the return value from previous steps into the context of the next step', async () => { - const stages: StageInput[] = [ - { - name: 'c/o', - handler: jest - .fn() - .mockResolvedValue({ first: 'ben', second: 'lambert' }), - }, - { - name: 'g/p', - handler: jest - .fn() - .mockResolvedValue({ second: 'linus', third: 'lambert' }), - }, - { - name: 'go', - handler: jest.fn(), - }, - ]; - - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages, - }); - - await processor.run(job); - - expect(stages[1].handler).toHaveBeenCalledWith( - expect.objectContaining({ first: 'ben', second: 'lambert' }), - ); - - expect(stages[2].handler).toHaveBeenCalledWith( - expect.objectContaining({ - first: 'ben', - second: 'linus', - third: 'lambert', - }), - ); - }); - - it('should fail the job and the step if one of them fails', async () => { - const fail = new Error('something went wrong here'); - const stages: StageInput[] = [ - { - name: 'c/o', - handler: jest.fn(), - }, - { - name: 'g/p', - handler: jest.fn().mockRejectedValue(fail), - }, - { - name: 'go', - handler: jest.fn(), - }, - ]; - - const processor = new JobProcessor(workingDirectory); - const job = processor.create({ - entity: mockEntity, - values: mockValues, - stages, - }); - - await processor.run(job); - - expect(job.status).toBe('FAILED'); - expect(job.stages[0].status).toBe('COMPLETED'); - expect(job.stages[1].status).toBe('FAILED'); - expect(job.stages[2].status).toBe('PENDING'); - expect(job.error?.message).toBe('something went wrong here'); - expect(job.stages[1].log.join()).toContain('something went wrong here'); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts deleted file mode 100644 index ef603477c1..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import os from 'os'; -import fs from 'fs-extra'; -import { Processor, Job, StageContext, StageInput } from './types'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import * as uuid from 'uuid'; -import path from 'path'; -import { TemplaterValues } from '../stages/templater'; -import { makeLogStream } from './logger'; -import { Logger } from 'winston'; -import { Config } from '@backstage/config'; - -export type JobAndDirectoryTuple = { - job: Job; - directory: string; -}; - -export class JobProcessor implements Processor { - private readonly workingDirectory: string; - private readonly jobs: Map; - - static async fromConfig({ - config, - logger, - }: { - config: Config; - logger: Logger; - }) { - let workingDirectory: string; - if (config.has('backend.workingDirectory')) { - workingDirectory = config.getString('backend.workingDirectory'); - try { - // Check if working directory exists and is writable - await fs.promises.access( - workingDirectory, - fs.constants.F_OK | fs.constants.W_OK, - ); - logger.info(`using working directory: ${workingDirectory}`); - } catch (err) { - logger.error( - `working directory ${workingDirectory} ${ - err.code === 'ENOENT' ? 'does not exist' : 'is not writable' - }`, - ); - throw err; - } - } else { - workingDirectory = os.tmpdir(); - } - - return new JobProcessor(workingDirectory); - } - - constructor(workingDirectory: string) { - this.workingDirectory = workingDirectory; - this.jobs = new Map(); - } - - create({ - entity, - values, - stages, - }: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - stages: StageInput[]; - }): Job { - const id = uuid.v4(); - const { logger, stream } = makeLogStream({ id }); - - const context: StageContext = { - entity, - values, - logger, - logStream: stream, - workspacePath: path.join(this.workingDirectory, id), - }; - - const job: Job = { - id, - context, - stages: stages.map(stage => ({ - handler: stage.handler, - log: [], - name: stage.name, - status: 'PENDING', - })), - status: 'PENDING', - }; - - this.jobs.set(job.id, job); - - return job; - } - - get(id: string): Job | undefined { - return this.jobs.get(id); - } - - async run(job: Job): Promise { - if (job.status !== 'PENDING') { - throw new Error("Job is not in a 'PENDING' state"); - } - - await fs.mkdir(job.context.workspacePath); - - job.status = 'STARTED'; - - try { - for (const stage of job.stages) { - // Create a logger for each stage so we can create separate - // Streams for each step. - const { logger, log, stream } = makeLogStream({ - id: job.id, - stage: stage.name, - }); - // Attach the logger to the stage, and setup some timestamps. - stage.log = log; - stage.startedAt = Date.now(); - - try { - // Run the handler with the context created for the Job and some - // Additional logging helpers. - stage.status = 'STARTED'; - const handlerResponse = await stage.handler({ - ...job.context, - logger, - logStream: stream, - }); - - // If the handler returns something, then let's merge this onto the - // context for the next stage to use as it might be relevant. - if (handlerResponse) { - job.context = { - ...job.context, - ...handlerResponse, - }; - } - - // Complete the current stage - stage.status = 'COMPLETED'; - } catch (error) { - // Log to the current stage the error that occurred and fail the stage. - stage.status = 'FAILED'; - logger.error(`Stage failed with error: ${error.message}`); - logger.debug(error.stack); - // Throw the error so the job can be failed too. - throw error; - } finally { - // Always set the stage end timestamp. - stage.endedAt = Date.now(); - } - } - - // If all went to plan, complete the job. - job.status = 'COMPLETED'; - } catch (error) { - // If something went wrong, fail the job, and set the error property on the job. - job.error = { name: error.name, message: error.message }; - job.status = 'FAILED'; - } finally { - await fs.remove(job.context.workspacePath); - } - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts deleted file mode 100644 index 84509a6c0f..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import type { Writable } from 'stream'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { TemplaterValues } from '../stages/templater'; -import { Logger } from 'winston'; - -// Context will be a mutable object which is passed between stages -// To share data, but also thinking that we can pass in functions here too -// To maybe create sub steps or fail the entire thing, or skip stages down the line. -export type StageContext = { - values: TemplaterValues; - entity: TemplateEntityV1alpha1; - logger: Logger; - logStream: Writable; - workspacePath: string; -} & T; - -export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; - -export interface StageResult extends StageInput { - log: string[]; - status: ProcessorStatus; - startedAt?: number; - endedAt?: number; -} - -export interface StageInput { - name: string; - handler(ctx: StageContext): Promise; -} - -export type Job = { - id: string; - context: StageContext; - status: ProcessorStatus; - stages: StageResult[]; - error?: Error; -}; - -export type Processor = { - create({ - entity, - values, - stages, - }: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - stages: StageInput[]; - }): Job; - - get(id: string): Job | undefined; - - run(job: Job): Promise; -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts deleted file mode 100644 index 8f21485f64..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - LOCATION_ANNOTATION, - TemplateEntityV1alpha1, -} from '@backstage/catalog-model'; -import { joinGitUrlPath, parseLocationAnnotation } from './helpers'; - -describe('Helpers', () => { - describe('parseLocationAnnotation', () => { - it('throws an exception when no annotation location', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: {}, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-d@example.com', - }, - }; - - expect(() => parseLocationAnnotation(mockEntity)).toThrow( - expect.objectContaining({ - name: 'InputError', - message: `No location annotation provided in entity: ${mockEntity.metadata.name}`, - }), - ); - }); - - it('should throw an error when the protocol part is not set in the location annotation', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - [LOCATION_ANNOTATION]: - ':https://github.com/o/r/blob/master/template.yaml', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-b@example.com', - }, - }; - - expect(() => parseLocationAnnotation(mockEntity)).toThrow( - expect.objectContaining({ - name: 'TypeError', - message: - "Unable to parse location reference ':https://github.com/o/r/blob/master/template.yaml', expected ':', e.g. 'url:https://host/path'", - }), - ); - }); - - it('should throw an error when the location part is not set in the location annotation', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - [LOCATION_ANNOTATION]: 'github:', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-a@example.com', - }, - }; - - expect(() => parseLocationAnnotation(mockEntity)).toThrow( - expect.objectContaining({ - name: 'TypeError', - message: `Unable to parse location reference 'github:', expected ':', e.g. 'url:https://host/path'`, - }), - ); - }); - - it('should parse the location and protocol correctly for simple locations', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - [LOCATION_ANNOTATION]: 'file:./path', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-b@example.com', - }, - }; - - expect(parseLocationAnnotation(mockEntity)).toEqual({ - protocol: 'file', - location: './path', - }); - }); - - it('should parse the location and protocol correctly for complex with unescaped locations', () => { - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - [LOCATION_ANNOTATION]: 'github:https://lol.com/:something/shello', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - owner: 'team-c@example.com', - }, - }; - - expect(parseLocationAnnotation(mockEntity)).toEqual({ - protocol: 'github', - location: 'https://lol.com/:something/shello', - }); - }); - }); - - describe('joinGitUrlPath', () => { - it.each([ - [ - 'https://github.com/o/r/blob/master/template.yaml', - 'template', - 'https://github.com/o/r/blob/master/template', - ], - [ - 'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml', - undefined, - 'https://dev.azure.com/o/p/_git/template-repo?path=%2F', - ], - [ - 'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml', - 'a', - 'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa', - ], - [ - 'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Ftemplate.yaml', - 'b', - 'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Fb', - ], - [ - 'https://github.com/o/r/blob/master/template.yaml', - undefined, - 'https://github.com/o/r/blob/master', - ], - [ - 'https://github.com/o/r/blob/master/template.yaml', - 'template', - 'https://github.com/o/r/blob/master/template', - ], - [ - 'https://github.com/o/r/blob/master/templates/graphql-starter/template.yaml', - 'template', - 'https://github.com/o/r/blob/master/templates/graphql-starter/template', - ], - [ - 'https://gitlab.com/o/r/-/blob/master/template.yaml', - undefined, - 'https://gitlab.com/o/r/-/blob/master', - ], - [ - 'https://gitlab.com/o/r/-/blob/master/template.yaml', - 'template', - 'https://gitlab.com/o/r/-/blob/master/template', - ], - [ - 'https://gitlab.com/o/r/-/blob/master/a/b/c/template.yaml', - '../../c', - 'https://gitlab.com/o/r/-/blob/master/a/c', - ], - [ - 'https://bitbucket.org/p/r/src/master/a/b/template.yaml', - undefined, - 'https://bitbucket.org/p/r/src/master/a/b', - ], - [ - 'https://bitbucket.org/p/r/src/master/a/b/template.yaml', - 'c', - 'https://bitbucket.org/p/r/src/master/a/b/c', - ], - [ - 'https://bitbucket.org/p/r/src/master/a/b/template.yaml', - '../c', - 'https://bitbucket.org/p/r/src/master/a/c', - ], - ])('should join git url %s with path %s', (url, path, result) => { - expect(joinGitUrlPath(url, path)).toBe(result); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts deleted file mode 100644 index 8085277030..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { InputError } from '@backstage/errors'; -import { - LOCATION_ANNOTATION, - parseLocationReference, - TemplateEntityV1alpha1, -} from '@backstage/catalog-model'; -import { posix as posixPath } from 'path'; - -export type ParsedLocationAnnotation = { - protocol: 'file' | 'url'; - location: string; -}; - -export const parseLocationAnnotation = ( - entity: TemplateEntityV1alpha1, -): ParsedLocationAnnotation => { - const annotation = entity.metadata.annotations?.[LOCATION_ANNOTATION]; - if (!annotation) { - throw new InputError( - `No location annotation provided in entity: ${entity.metadata.name}`, - ); - } - - const { type, target } = parseLocationReference(annotation); - return { - protocol: type as 'file' | 'url', - location: target, - }; -}; - -export function joinGitUrlPath(repoUrl: string, path?: string): string { - const parsed = new URL(repoUrl); - - if (parsed.hostname.endsWith('azure.com')) { - const templatePath = posixPath.normalize( - posixPath.join( - posixPath.dirname(parsed.searchParams.get('path') || '/'), - path || '.', - ), - ); - parsed.searchParams.set('path', templatePath); - return parsed.toString(); - } - - return new URL(path || '.', repoUrl).toString().replace(/\/$/, ''); -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts deleted file mode 100644 index c6f824a139..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export * from './prepare'; -export * from './publish'; -export * from './templater'; -export * from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts deleted file mode 100644 index efbb241470..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { createTemplateAction } from '../actions'; -import { FilePreparer, PreparerBuilder } from './prepare'; -import { PublisherBuilder } from './publish'; -import { TemplaterBuilder, TemplaterValues } from './templater'; - -type Options = { - preparers: PreparerBuilder; - templaters: TemplaterBuilder; - publishers: PublisherBuilder; -}; - -export function createLegacyActions(options: Options) { - const { preparers, templaters, publishers } = options; - - return [ - createTemplateAction({ - id: 'legacy:prepare', - async handler(ctx) { - ctx.logger.info('Preparing the skeleton'); - const { protocol, url } = ctx.input; - const preparer = - protocol === 'file' - ? new FilePreparer() - : preparers.get(url as string); - - await preparer.prepare({ - url: url as string, - logger: ctx.logger, - workspacePath: ctx.workspacePath, - }); - }, - }), - createTemplateAction({ - id: 'legacy:template', - async handler(ctx) { - ctx.logger.info('Running the templater'); - const templater = templaters.get(ctx.input.templater as string); - await templater.run({ - workspacePath: ctx.workspacePath, - logStream: ctx.logStream, - values: ctx.input.values as TemplaterValues, - }); - }, - }), - createTemplateAction({ - id: 'legacy:publish', - async handler(ctx) { - const { values } = ctx.input; - if ( - typeof values !== 'object' || - values === null || - Array.isArray(values) - ) { - throw new Error( - `Invalid values passed to publish, got ${typeof values}`, - ); - } - const storePath = values.storePath as unknown; - if (typeof storePath !== 'string') { - throw new Error( - `Invalid store path passed to publish, got ${typeof storePath}`, - ); - } - const owner = values.owner as unknown; - if (typeof owner !== 'string') { - throw new Error( - `Invalid owner passed to publish, got ${typeof owner}`, - ); - } - - const publisher = publishers.get(storePath); - ctx.logger.info('Will now store the template'); - const { remoteUrl, catalogInfoUrl } = await publisher.publish({ - values: { - ...values, - owner, - storePath, - }, - workspacePath: ctx.workspacePath, - logger: ctx.logger, - }); - ctx.output('remoteUrl', remoteUrl); - if (catalogInfoUrl) { - ctx.output('catalogInfoUrl', catalogInfoUrl); - } - }, - }), - ]; -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts deleted file mode 100644 index 74a21e0833..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -import { AzurePreparer } from './azure'; -import { getVoidLogger, Git } from '@backstage/backend-common'; - -jest.mock('fs-extra'); - -describe('AzurePreparer', () => { - const mockGitClient = { - clone: jest.fn(), - }; - - const logger = getVoidLogger(); - - jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); - - const preparer = AzurePreparer.fromConfig({ - host: 'dev.azure.com', - token: 'fake-azure-token', - }); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const checkoutPath = path.resolve(workspacePath, 'checkout'); - const templatePath = path.resolve(workspacePath, 'template'); - const prepareOptions = { - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - workspacePath, - logger, - }; - - it('calls the clone command with token from integrations config', async () => { - await preparer.prepare(prepareOptions); - expect(Git.fromAuth).toHaveBeenCalledWith({ - logger, - password: 'fake-azure-token', - username: 'notempty', - }); - expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath); - expect(fs.rmdir).toHaveBeenCalledWith(path.resolve(templatePath, '.git')); - }); - - it('calls the clone command with the correct arguments for a repository', async () => { - await preparer.prepare(prepareOptions); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - dir: checkoutPath, - }); - }); - - it('calls the clone command with the correct arguments for a repository with a specified branch', async () => { - await preparer.prepare({ - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster', - logger, - workspacePath, - }); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - dir: checkoutPath, - ref: 'master', - }); - }); - - it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { - await preparer.prepare({ - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - workspacePath, - logger, - }); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: - 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', - dir: checkoutPath, - }); - expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath); - }); - - it('moves the template from path if it is specified', async () => { - await preparer.prepare({ - url: `https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=${encodeURIComponent( - './subdir', - )}`, - logger, - workspacePath, - }); - - expect(fs.move).toHaveBeenCalledWith( - path.resolve(checkoutPath, 'subdir'), - templatePath, - ); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts deleted file mode 100644 index a405288f9f..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import fs from 'fs-extra'; -import path from 'path'; -import { Git } from '@backstage/backend-common'; -import { PreparerBase, PreparerOptions } from './types'; -import parseGitUrl from 'git-url-parse'; -import { AzureIntegrationConfig } from '@backstage/integration'; - -export class AzurePreparer implements PreparerBase { - static fromConfig(config: AzureIntegrationConfig) { - return new AzurePreparer({ token: config.token }); - } - - constructor(private readonly config: { token?: string }) {} - - async prepare({ url, workspacePath, logger }: PreparerOptions) { - const parsedGitUrl = parseGitUrl(url); - const checkoutPath = path.join(workspacePath, 'checkout'); - const targetPath = path.join(workspacePath, 'template'); - const fullPathToTemplate = path.resolve( - checkoutPath, - parsedGitUrl.filepath ?? '', - ); - - // Username can be anything but the empty string according to: - // https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat - const git = this.config.token - ? Git.fromAuth({ - password: this.config.token, - username: 'notempty', - logger, - }) - : Git.fromAuth({ logger }); - - await git.clone({ - url: parsedGitUrl.toString('https'), - ref: parsedGitUrl.ref, - dir: checkoutPath, - }); - - await fs.move(fullPathToTemplate, targetPath); - - try { - await fs.rmdir(path.join(targetPath, '.git')); - } catch { - // Ignore intentionally - } - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts deleted file mode 100644 index 2c4e24fed0..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import { BitbucketPreparer } from './bitbucket'; -import { getVoidLogger, Git } from '@backstage/backend-common'; -import path from 'path'; -import os from 'os'; - -jest.mock('fs-extra'); - -describe('BitbucketPreparer', () => { - const logger = getVoidLogger(); - const mockGitClient = { - clone: jest.fn(), - }; - - jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - const preparer = BitbucketPreparer.fromConfig({ - host: 'bitbucket.org', - username: 'fake-user', - appPassword: 'fake-password', - }); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const checkoutPath = path.resolve(workspacePath, 'checkout'); - const templatePath = path.resolve(workspacePath, 'template'); - - const prepareOptions = { - url: 'https://bitbucket.org/backstage-project/backstage-repo', - logger, - workspacePath, - }; - - it('calls the clone command with the correct arguments for a repository', async () => { - await preparer.prepare(prepareOptions); - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://bitbucket.org/backstage-project/backstage-repo', - dir: checkoutPath, - ref: expect.any(String), - }); - expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath); - expect(fs.rmdir).toHaveBeenCalledWith(path.resolve(templatePath, '.git')); - }); - - it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => { - const preparerCheck = BitbucketPreparer.fromConfig({ - host: 'bitbucket.org', - username: 'fake-user', - appPassword: 'fake-password', - }); - await preparerCheck.prepare(prepareOptions); - - expect(Git.fromAuth).toHaveBeenCalledWith({ - logger, - username: 'fake-user', - password: 'fake-password', - }); - }); - - it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { - await preparer.prepare(prepareOptions); - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://bitbucket.org/backstage-project/backstage-repo', - dir: checkoutPath, - ref: expect.any(String), - }); - }); - - it('moves a template subdirectory to checkout if specified', async () => { - await preparer.prepare({ - url: 'https://bitbucket.org/foo/bar/src/master/1/2/3', - logger, - workspacePath, - }); - expect(fs.move).toHaveBeenCalledWith( - path.resolve(checkoutPath, '1', '2', '3'), - templatePath, - ); - }); - - it('calls the clone command with with token for auth method', async () => { - const preparerCheck = BitbucketPreparer.fromConfig({ - host: 'bitbucket.org', - token: 'fake-token', - }); - await preparerCheck.prepare(prepareOptions); - - expect(Git.fromAuth).toHaveBeenCalledWith({ - logger, - username: 'x-token-auth', - password: 'fake-token', - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts deleted file mode 100644 index 36f6b07750..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import fs from 'fs-extra'; -import path from 'path'; -import { Git } from '@backstage/backend-common'; -import { PreparerBase, PreparerOptions } from './types'; -import { BitbucketIntegrationConfig } from '@backstage/integration'; -import parseGitUrl from 'git-url-parse'; - -export class BitbucketPreparer implements PreparerBase { - static fromConfig(config: BitbucketIntegrationConfig) { - return new BitbucketPreparer({ - username: config.username, - token: config.token, - appPassword: config.appPassword, - }); - } - - constructor( - private readonly config: { - username?: string; - token?: string; - appPassword?: string; - }, - ) {} - - async prepare({ url, workspacePath, logger }: PreparerOptions) { - const parsedGitUrl = parseGitUrl(url); - const checkoutPath = path.join(workspacePath, 'checkout'); - const targetPath = path.join(workspacePath, 'template'); - const fullPathToTemplate = path.resolve( - checkoutPath, - parsedGitUrl.filepath ?? '', - ); - - const git = Git.fromAuth({ logger, ...this.getAuth() }); - - await git.clone({ - url: parsedGitUrl.toString('https'), - dir: checkoutPath, - ref: parsedGitUrl.ref, - }); - - await fs.move(fullPathToTemplate, targetPath); - - try { - await fs.rmdir(path.join(targetPath, '.git')); - } catch { - // Ignore intentionally - } - } - - private getAuth(): { username: string; password: string } | undefined { - const { username, token, appPassword } = this.config; - - if (username && appPassword) { - return { username: username, password: appPassword }; - } - - if (token) { - return { - username: 'x-token-auth', - password: token! || appPassword!, - }; - } - - return undefined; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts deleted file mode 100644 index 73d944db67..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { getVoidLogger } from '@backstage/backend-common'; -import fs from 'fs-extra'; -import { FilePreparer } from './file'; -import os from 'os'; -import path from 'path'; - -jest.mock('fs-extra'); - -describe('File preparer', () => { - it('prepares templates from a file path', async () => { - const logger = getVoidLogger(); - const preparer = new FilePreparer(); - const root = os.platform() === 'win32' ? 'C:\\' : '/'; - const workspacePath = path.join(root, 'tmp'); - const targetPath = path.resolve(workspacePath, 'template'); - - await preparer.prepare({ - url: `file://${root}path/to/template`, - logger, - workspacePath, - }); - expect(fs.copy).toHaveBeenCalledWith( - path.join(root, 'path', 'to', 'template'), - targetPath, - { - recursive: true, - }, - ); - expect(fs.ensureDir).toHaveBeenCalledWith(targetPath); - - await expect( - preparer.prepare({ - url: 'http://not/file/path', - logger, - workspacePath, - }), - ).rejects.toThrow( - "Wrong location protocol, should be 'file', http://not/file/path", - ); - - if (os.platform() === 'win32') { - // eslint-disable-next-line jest/no-conditional-expect - await expect( - preparer.prepare({ - url: 'file:///unix/file/path', - logger, - workspacePath, - }), - ).rejects.toThrow('File URL path must be absolute'); - } else { - // eslint-disable-next-line jest/no-conditional-expect - await expect( - preparer.prepare({ - url: 'file://not/full/path', - logger, - workspacePath, - }), - ).rejects.toThrow( - `File URL host must be "localhost" or empty on ${os.platform()}`, - ); - } - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts deleted file mode 100644 index 92ee93511c..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import fs from 'fs-extra'; -import path from 'path'; -import { fileURLToPath } from 'url'; -import { InputError } from '@backstage/errors'; -import { PreparerBase, PreparerOptions } from './types'; - -export class FilePreparer implements PreparerBase { - async prepare({ url, workspacePath }: PreparerOptions) { - if (!url.startsWith('file://')) { - throw new InputError(`Wrong location protocol, should be 'file', ${url}`); - } - - const templatePath = fileURLToPath(url); - - const targetDir = path.join(workspacePath, 'template'); - await fs.ensureDir(targetDir); - - await fs.copy(templatePath, targetDir, { - recursive: true, - }); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts deleted file mode 100644 index 0438e28f01..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -import { GithubPreparer } from './github'; -import { getVoidLogger, Git } from '@backstage/backend-common'; - -jest.mock('fs-extra'); - -describe('GitHubPreparer', () => { - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const checkoutPath = path.resolve(workspacePath, 'checkout'); - const templatePath = path.resolve(workspacePath, 'template'); - - const mockGitClient = { - clone: jest.fn(), - }; - const logger = getVoidLogger(); - - jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - const preparer = GithubPreparer.fromConfig({ - host: 'github.com', - token: 'fake-token', - }); - - it('calls the clone command with the correct arguments for a repository', async () => { - await preparer.prepare({ - url: - 'https://github.com/benjdlambert/backstage-graphql-template/blob/master/templates/graphql-starter/template', - logger, - workspacePath, - }); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://github.com/benjdlambert/backstage-graphql-template', - dir: checkoutPath, - ref: expect.any(String), - }); - expect(fs.move).toHaveBeenCalledWith( - path.resolve(checkoutPath, 'templates', 'graphql-starter', 'template'), - templatePath, - ); - expect(fs.rmdir).toHaveBeenCalledWith(path.resolve(templatePath, '.git')); - }); - - it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { - await preparer.prepare({ - url: - 'https://github.com/benjdlambert/backstage-graphql-template/blob/master', - logger, - workspacePath, - }); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://github.com/benjdlambert/backstage-graphql-template', - dir: checkoutPath, - ref: 'master', - }); - expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath); - expect(fs.rmdir).toHaveBeenCalledWith(path.resolve(templatePath, '.git')); - }); - - it('calls the clone command with token', async () => { - await preparer.prepare({ - url: - 'https://github.com/benjdlambert/backstage-graphql-template/blob/master', - logger, - workspacePath, - }); - - expect(Git.fromAuth).toHaveBeenCalledWith({ - logger, - username: 'x-access-token', - password: 'fake-token', - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts deleted file mode 100644 index dec7fc06c2..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import fs from 'fs-extra'; -import path from 'path'; -import { Git } from '@backstage/backend-common'; -import { PreparerBase, PreparerOptions } from './types'; -import parseGitUrl from 'git-url-parse'; -import { - GitHubIntegrationConfig, - GithubCredentialsProvider, -} from '@backstage/integration'; - -export class GithubPreparer implements PreparerBase { - static fromConfig(config: GitHubIntegrationConfig) { - const credentialsProvider = GithubCredentialsProvider.create(config); - return new GithubPreparer({ credentialsProvider }); - } - - constructor( - private readonly config: { credentialsProvider: GithubCredentialsProvider }, - ) {} - - async prepare({ url, workspacePath, logger }: PreparerOptions) { - const parsedGitUrl = parseGitUrl(url); - const checkoutPath = path.join(workspacePath, 'checkout'); - const targetPath = path.join(workspacePath, 'template'); - const fullPathToTemplate = path.resolve( - checkoutPath, - parsedGitUrl.filepath ?? '', - ); - - const { token } = await this.config.credentialsProvider.getCredentials({ - url, - }); - - const git = token - ? Git.fromAuth({ - username: 'x-access-token', - password: token, - logger, - }) - : Git.fromAuth({ logger }); - - await git.clone({ - url: parsedGitUrl.toString('https'), - dir: checkoutPath, - ref: parsedGitUrl.ref, - }); - - await fs.move(fullPathToTemplate, targetPath); - - try { - await fs.rmdir(path.join(targetPath, '.git')); - } catch { - // Ignore intentionally - } - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts deleted file mode 100644 index fa76c74bd7..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -import { GitlabPreparer } from './gitlab'; -import { getVoidLogger, Git } from '@backstage/backend-common'; - -jest.mock('fs-extra'); - -describe('GitLabPreparer', () => { - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const checkoutPath = path.resolve(workspacePath, 'checkout'); - const templatePath = path.resolve(workspacePath, 'template'); - - const mockGitClient = { - clone: jest.fn(), - }; - const logger = getVoidLogger(); - - jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); - - beforeEach(() => { - jest.clearAllMocks(); - }); - const preparer = GitlabPreparer.fromConfig({ - host: '', - token: 'fake-token', - apiBaseUrl: '', - baseUrl: '', - }); - - it(`calls the clone command with the correct arguments for a repository`, async () => { - await preparer.prepare({ - url: - 'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master', - logger, - workspacePath, - }); - - expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git', - dir: checkoutPath, - ref: expect.any(String), - }); - - expect(Git.fromAuth).toHaveBeenCalledWith({ - logger, - username: 'oauth2', - password: 'fake-token', - }); - - expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath); - expect(fs.rmdir).toHaveBeenCalledWith(path.resolve(templatePath, '.git')); - }); - - it(`clones the template from a sub directory if specified`, async () => { - await preparer.prepare({ - url: - 'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/1/2/3', - logger, - workspacePath, - }); - expect(fs.move).toHaveBeenCalledWith( - path.resolve(checkoutPath, '1', '2', '3'), - templatePath, - ); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts deleted file mode 100644 index 582214606b..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Git } from '@backstage/backend-common'; -import { GitLabIntegrationConfig } from '@backstage/integration'; -import fs from 'fs-extra'; -import parseGitUrl from 'git-url-parse'; -import path from 'path'; -import { PreparerBase, PreparerOptions } from './types'; - -export class GitlabPreparer implements PreparerBase { - static fromConfig(config: GitLabIntegrationConfig) { - return new GitlabPreparer({ token: config.token }); - } - - constructor(private readonly config: { token?: string }) {} - - async prepare({ url, workspacePath, logger }: PreparerOptions) { - const parsedGitUrl = parseGitUrl(url); - const checkoutPath = path.join(workspacePath, 'checkout'); - const targetPath = path.join(workspacePath, 'template'); - const fullPathToTemplate = path.resolve( - checkoutPath, - parsedGitUrl.filepath ?? '', - ); - parsedGitUrl.git_suffix = true; - - const git = this.config.token - ? Git.fromAuth({ - password: this.config.token, - username: 'oauth2', - logger, - }) - : Git.fromAuth({ logger }); - - await git.clone({ - url: parsedGitUrl.toString('https'), - dir: checkoutPath, - ref: parsedGitUrl.ref, - }); - - await fs.move(fullPathToTemplate, targetPath); - - try { - await fs.rmdir(path.join(targetPath, '.git')); - } catch { - // Ignore intentionally - } - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts deleted file mode 100644 index 060b99b3a8..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export { AzurePreparer } from './azure'; -export { BitbucketPreparer } from './bitbucket'; -export { FilePreparer } from './file'; -export { GithubPreparer } from './github'; -export { GitlabPreparer } from './gitlab'; -export { Preparers } from './preparers'; -export type { PreparerBase, PreparerBuilder, PreparerOptions } from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts deleted file mode 100644 index 20dfad924b..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { GithubPreparer } from './github'; -import { Preparers } from './preparers'; - -describe('Preparers', () => { - it('should return the correct preparer based on the hostname', async () => { - const preparer = await GithubPreparer.fromConfig({ - host: 'github.com', - apiBaseUrl: 'lols', - token: 'something else yo', - }); - - const preparers = new Preparers(); - preparers.register('github.com', preparer); - - expect( - preparers.get('https://github.com/please/find/me/something/from/github'), - ).toBe(preparer); - }); - - it('should throw an error if there is nothing that will match the url provided', async () => { - const preparer = await GithubPreparer.fromConfig({ - host: 'github.com', - apiBaseUrl: 'lols', - token: 'something else yo', - }); - - const preparers = new Preparers(); - preparers.register('github.com', preparer); - - expect(() => preparers.get('https://404.com')).toThrow( - `Unable to find a preparer for URL: https://404.com. Please make sure to register this host under an integration in app-config`, - ); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts deleted file mode 100644 index 98ca984c17..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Config } from '@backstage/config'; -import { PreparerBase, PreparerBuilder } from './types'; -import { Logger } from 'winston'; - -import { GitlabPreparer } from './gitlab'; -import { AzurePreparer } from './azure'; -import { GithubPreparer } from './github'; -import { BitbucketPreparer } from './bitbucket'; -import { ScmIntegrations } from '@backstage/integration'; - -export class Preparers implements PreparerBuilder { - private preparerMap = new Map(); - - register(host: string, preparer: PreparerBase) { - this.preparerMap.set(host, preparer); - } - - get(url: string): PreparerBase { - const preparer = this.preparerMap.get(new URL(url).host); - if (!preparer) { - throw new Error( - `Unable to find a preparer for URL: ${url}. Please make sure to register this host under an integration in app-config`, - ); - } - return preparer; - } - - static async fromConfig( - config: Config, - // eslint-disable-next-line - _: { logger: Logger }, - ): Promise { - const preparers = new Preparers(); - const scm = ScmIntegrations.fromConfig(config); - for (const integration of scm.azure.list()) { - preparers.register( - integration.config.host, - AzurePreparer.fromConfig(integration.config), - ); - } - - for (const integration of scm.github.list()) { - preparers.register( - integration.config.host, - GithubPreparer.fromConfig(integration.config), - ); - } - - for (const integration of scm.gitlab.list()) { - preparers.register( - integration.config.host, - GitlabPreparer.fromConfig(integration.config), - ); - } - - for (const integration of scm.bitbucket.list()) { - preparers.register( - integration.config.host, - BitbucketPreparer.fromConfig(integration.config), - ); - } - - return preparers; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts deleted file mode 100644 index 1b6b4380f7..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Logger } from 'winston'; - -export type PreparerOptions = { - /** - * Full URL to the directory containg template data - */ - url: string; - /** - * The workspace path that will eventually be the the root of the new repo - */ - workspacePath: string; - logger: Logger; -}; - -export interface PreparerBase { - /** - * Prepare a directory with contents from the remote location - */ - prepare(opts: PreparerOptions): Promise; -} - -export type PreparerBuilder = { - register(host: string, preparer: PreparerBase): void; - get(url: string): PreparerBase; -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts deleted file mode 100644 index 3eecab33b5..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -jest.mock('./helpers'); - -jest.mock('azure-devops-node-api', () => ({ - WebApi: jest.fn(), - getPersonalAccessTokenHandler: jest.fn().mockReturnValue(() => {}), -})); - -import os from 'os'; -import { resolve } from 'path'; -import { AzurePublisher } from './azure'; -import { WebApi } from 'azure-devops-node-api'; -import * as helpers from './helpers'; -import { getVoidLogger } from '@backstage/backend-common'; - -describe('Azure Publisher', () => { - const logger = getVoidLogger(); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const resultPath = resolve(workspacePath, 'result'); - - describe('publish: createRemoteInAzure', () => { - it('should use azure-devops-node-api to create a repo in the given project', async () => { - const mockGitClient = { - createRepository: jest.fn(), - }; - const mockGitApi = { - getGitApi: jest.fn().mockReturnValue(mockGitClient), - }; - - ((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi); - - const publisher = await AzurePublisher.fromConfig({ - host: 'dev.azure.com', - token: 'fake-azure-token', - }); - - mockGitClient.createRepository.mockResolvedValue({ - remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', - } as { remoteUrl: string }); - - const result = await publisher!.publish({ - values: { - storePath: 'https://dev.azure.com/organisation/project/_git/repo', - owner: 'bob', - }, - workspacePath, - logger, - }); - - expect(WebApi).toHaveBeenCalledWith( - 'https://dev.azure.com/organisation', - expect.any(Function), - ); - - expect(result).toEqual({ - remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', - catalogInfoUrl: - 'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml', - }); - expect(mockGitClient.createRepository).toHaveBeenCalledWith( - { - name: 'repo', - }, - 'project', - ); - expect(helpers.initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', - auth: { username: 'notempty', password: 'fake-azure-token' }, - logger, - }); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts deleted file mode 100644 index 921c139a2e..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PublisherBase, PublisherOptions, PublisherResult } from './types'; -import { IGitApi } from 'azure-devops-node-api/GitApi'; -import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; -import { initRepoAndPush } from './helpers'; -import { AzureIntegrationConfig } from '@backstage/integration'; -import parseGitUrl from 'git-url-parse'; -import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; -import path from 'path'; - -export class AzurePublisher implements PublisherBase { - static async fromConfig(config: AzureIntegrationConfig) { - if (!config.token) { - return undefined; - } - return new AzurePublisher({ token: config.token }); - } - - constructor(private readonly config: { token: string }) {} - - async publish({ - values, - workspacePath, - logger, - }: PublisherOptions): Promise { - const { owner, name, organization, resource } = parseGitUrl( - values.storePath, - ); - const authHandler = getPersonalAccessTokenHandler(this.config.token); - const webApi = new WebApi( - `https://${resource}/${organization}`, - authHandler, - ); - const client = await webApi.getGitApi(); - - const remoteUrl = await this.createRemote({ - project: owner, - name, - client, - }); - - const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; - - await initRepoAndPush({ - dir: path.join(workspacePath, 'result'), - remoteUrl, - auth: { - username: 'notempty', - password: this.config.token, - }, - logger, - }); - - return { remoteUrl, catalogInfoUrl }; - } - - private async createRemote(opts: { - name: string; - project: string; - client: IGitApi; - }) { - const { name, project, client } = opts; - const createOptions: GitRepositoryCreateOptions = { name }; - const repo = await client.createRepository(createOptions, project); - - return repo.remoteUrl || ''; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts deleted file mode 100644 index a3f02e2c89..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -jest.mock('./helpers'); - -import os from 'os'; -import { resolve } from 'path'; -import { BitbucketPublisher } from './bitbucket'; -import { initRepoAndPush } from './helpers'; -import { getVoidLogger } from '@backstage/backend-common'; -import { rest } from 'msw'; -import { setupServer } from 'msw/node'; -import { msw } from '@backstage/test-utils'; - -describe('Bitbucket Publisher', () => { - const logger = getVoidLogger(); - const server = setupServer(); - msw.setupDefaultHandlers(server); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const resultPath = resolve(workspacePath, 'result'); - - describe('publish: createRemoteInBitbucketCloud', () => { - it('should create repo in bitbucket cloud', async () => { - server.use( - rest.post( - 'https://api.bitbucket.org/2.0/repositories/project/repo', - (_, res, ctx) => - res( - ctx.status(200), - ctx.set('Content-Type', 'application/json'), - ctx.json({ - links: { - html: { - href: 'https://bitbucket.org/project/repo', - }, - clone: [ - { - name: 'https', - href: 'https://bitbucket.org/project/repo', - }, - ], - }, - }), - ), - ), - ); - - const publisher = await BitbucketPublisher.fromConfig( - { - host: 'bitbucket.org', - username: 'fake-user', - appPassword: 'fake-token', - }, - { repoVisibility: 'private' }, - ); - - const result = await publisher.publish({ - values: { - storePath: 'https://bitbucket.org/project/repo', - owner: 'bob', - }, - workspacePath, - logger: logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://bitbucket.org/project/repo', - catalogInfoUrl: - 'https://bitbucket.org/project/repo/src/master/catalog-info.yaml', - }); - - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://bitbucket.org/project/repo', - auth: { username: 'fake-user', password: 'fake-token' }, - logger: logger, - }); - }); - }); - - describe('publish: createRemoteInBitbucketServer', () => { - it('should throw an error if no username present', async () => { - await expect( - BitbucketPublisher.fromConfig( - { - host: 'bitbucket.mycompany.com', - token: 'fake-token', - }, - { repoVisibility: 'private' }, - ), - ).rejects.toThrow( - 'Bitbucket server requires the username to be set in your config', - ); - }); - it('should create repo in bitbucket server', async () => { - server.use( - rest.post( - 'https://bitbucket.mycompany.com/rest/api/1.0/projects/project/repos', - (_, res, ctx) => - res( - ctx.status(201), - ctx.set('Content-Type', 'application/json'), - ctx.json({ - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/projects/project/repos/repo', - }, - ], - clone: [ - { - name: 'http', - href: 'https://bitbucket.mycompany.com/scm/project/repo', - }, - ], - }, - }), - ), - ), - ); - - const publisher = await BitbucketPublisher.fromConfig( - { - host: 'bitbucket.mycompany.com', - username: 'foo', - token: 'fake-token', - }, - { repoVisibility: 'private' }, - ); - - const result = await publisher.publish({ - values: { - storePath: 'https://bitbucket.mycompany.com/project/repo', - owner: 'bob', - }, - workspacePath, - logger: logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo', - catalogInfoUrl: - 'https://bitbucket.mycompany.com/projects/project/repos/repo/catalog-info.yaml', - }); - - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo', - auth: { username: 'foo', password: 'fake-token' }, - logger: logger, - }); - }); - }); - - it('should use apiBaseUrl to create the repository if it is set', async () => { - server.use( - rest.post( - 'https://bitbucket.mycompany.com/bitbucket/rest/api/1.0/projects/project/repos', - (_, res, ctx) => - res( - ctx.status(201), - ctx.set('Content-Type', 'application/json'), - ctx.json({ - links: { - self: [ - { - href: - 'https://bitbucket.mycompany.com/bitbucket/projects/project/repos/repo', - }, - ], - clone: [ - { - name: 'http', - href: - 'https://bitbucket.mycompany.com/bitbucket/scm/project/repo', - }, - ], - }, - }), - ), - ), - ); - - const publisher = await BitbucketPublisher.fromConfig( - { - host: 'bitbucket.mycompany.com', - username: 'foo', - token: 'fake-token', - apiBaseUrl: 'https://bitbucket.mycompany.com/bitbucket/rest/api/1.0', - }, - { repoVisibility: 'private' }, - ); - - const result = await publisher.publish({ - values: { - storePath: 'https://bitbucket.mycompany.com/project/repo', - owner: 'bob', - }, - workspacePath, - logger: logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://bitbucket.mycompany.com/bitbucket/scm/project/repo', - catalogInfoUrl: - 'https://bitbucket.mycompany.com/bitbucket/projects/project/repos/repo/catalog-info.yaml', - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts deleted file mode 100644 index a85518eeae..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PublisherBase, PublisherOptions, PublisherResult } from './types'; -import { initRepoAndPush } from './helpers'; -import fetch from 'cross-fetch'; -import { BitbucketIntegrationConfig } from '@backstage/integration'; -import parseGitUrl from 'git-url-parse'; -import path from 'path'; - -export type RepoVisibilityOptions = 'private' | 'public'; - -// TODO(blam): We should probably start to use a bitbucket client here that we can change -// the baseURL to point at on-prem or public bitbucket versions like we do for -// github and ghe. There's to much logic and not enough types here for us to say that this way is better than using -// a supported bitbucket client if one exists. -export class BitbucketPublisher implements PublisherBase { - static async fromConfig( - config: BitbucketIntegrationConfig, - { repoVisibility }: { repoVisibility: RepoVisibilityOptions }, - ) { - if (config.host !== 'bitbucket.org' && !config.username) - throw new Error( - 'Bitbucket server requires the username to be set in your config', - ); - - return new BitbucketPublisher({ - host: config.host, - token: config.token, - appPassword: config.appPassword, - username: config.username, - apiBaseUrl: config.apiBaseUrl, - repoVisibility, - }); - } - - constructor( - private readonly config: { - host: string; - token?: string; - appPassword?: string; - username?: string; - apiBaseUrl?: string; - repoVisibility: RepoVisibilityOptions; - }, - ) {} - - async publish({ - values, - workspacePath, - logger, - }: PublisherOptions): Promise { - const { owner: project, name } = parseGitUrl(values.storePath); - - const description = values.description as string; - const result = await this.createRemote({ - project, - name, - description, - }); - - await initRepoAndPush({ - dir: path.join(workspacePath, 'result'), - remoteUrl: result.remoteUrl, - auth: { - username: this.config.username ? this.config.username : 'x-token-auth', - password: this.config.appPassword - ? this.config.appPassword - : this.config.token ?? '', - }, - logger, - }); - return result; - } - - private async createRemote(opts: { - project: string; - name: string; - description: string; - }): Promise { - if (this.config.host === 'bitbucket.org') { - return this.createBitbucketCloudRepository(opts); - } - return this.createBitbucketServerRepository(opts); - } - - private async createBitbucketCloudRepository(opts: { - project: string; - name: string; - description: string; - }): Promise { - const { project, name, description } = opts; - - let response: Response; - - const options: RequestInit = { - method: 'POST', - body: JSON.stringify({ - scm: 'git', - description: description, - is_private: this.config.repoVisibility === 'private', - }), - headers: { - Authorization: this.getAuthorizationHeader(), - 'Content-Type': 'application/json', - }, - }; - try { - response = await fetch( - `https://api.bitbucket.org/2.0/repositories/${project}/${name}`, - options, - ); - } catch (e) { - throw new Error(`Unable to create repository, ${e}`); - } - if (response.status === 200) { - const r = await response.json(); - let remoteUrl = ''; - for (const link of r.links.clone) { - if (link.name === 'https') { - remoteUrl = link.href; - } - } - - // TODO use the urlReader to get the default branch - const catalogInfoUrl = `${r.links.html.href}/src/master/catalog-info.yaml`; - return { remoteUrl, catalogInfoUrl }; - } - throw new Error(`Not a valid response code ${await response.text()}`); - } - - private getAuthorizationHeader(): string { - if (this.config.username && this.config.appPassword) { - const buffer = Buffer.from( - `${this.config.username}:${this.config.appPassword}`, - 'utf8', - ); - - return `Basic ${buffer.toString('base64')}`; - } - - if (this.config.token) { - return `Bearer ${this.config.token}`; - } - - throw new Error( - `Authorization has not been provided for Bitbucket. Please add either username + appPassword or token to the Integrations config`, - ); - } - - private async createBitbucketServerRepository(opts: { - project: string; - name: string; - description: string; - }): Promise { - const { project, name, description } = opts; - - let response: Response; - const options: RequestInit = { - method: 'POST', - body: JSON.stringify({ - name: name, - description: description, - public: this.config.repoVisibility === 'public', - }), - headers: { - Authorization: this.getAuthorizationHeader(), - 'Content-Type': 'application/json', - }, - }; - - try { - const baseUrl = this.config.apiBaseUrl - ? this.config.apiBaseUrl - : `https://${this.config.host}/rest/api/1.0`; - - response = await fetch(`${baseUrl}/projects/${project}/repos`, options); - } catch (e) { - throw new Error(`Unable to create repository, ${e}`); - } - if (response.status === 201) { - const r = await response.json(); - let remoteUrl = ''; - for (const link of r.links.clone) { - if (link.name === 'http') { - remoteUrl = link.href; - } - } - const catalogInfoUrl = `${r.links.self[0].href}/catalog-info.yaml`; - return { remoteUrl, catalogInfoUrl }; - } - throw new Error(`Not a valid response code ${await response.text()}`); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts deleted file mode 100644 index fd4256dc86..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -jest.mock('@octokit/rest'); -jest.mock('./helpers'); - -import os from 'os'; -import { resolve } from 'path'; - -import { getVoidLogger } from '@backstage/backend-common'; -import { Octokit, RestEndpointMethodTypes } from '@octokit/rest'; -import { GithubPublisher } from './github'; -import { initRepoAndPush } from './helpers'; - -const { mockGithubClient } = require('@octokit/rest') as { - mockGithubClient: { - repos: jest.Mocked; - users: jest.Mocked; - teams: jest.Mocked; - }; -}; - -describe('GitHub Publisher', () => { - const logger = getVoidLogger(); - beforeEach(() => { - jest.clearAllMocks(); - }); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const resultPath = resolve(workspacePath, 'result'); - - describe('with public repo visibility', () => { - describe('publish: createRemoteInGithub', () => { - it('should use octokit to create a repo in an organisation if the organisation property is set', async () => { - const publisher = await GithubPublisher.fromConfig( - { - token: 'fake-token', - host: 'github.com', - }, - { repoVisibility: 'public' }, - ); - - mockGithubClient.repos.createInOrg.mockResolvedValue({ - data: { - clone_url: 'https://github.com/backstage/backstage.git', - }, - } as RestEndpointMethodTypes['repos']['createInOrg']['response']); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'Organization', - }, - } as RestEndpointMethodTypes['users']['getByUsername']['response']); - - const result = await publisher!.publish({ - values: { - storePath: 'https://github.com/blam/test', - owner: 'bob', - access: 'blam/team', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://github.com/backstage/backstage.git', - catalogInfoUrl: - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - }); - expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ - org: 'blam', - name: 'test', - private: false, - visibility: 'public', - }); - expect( - mockGithubClient.teams.addOrUpdateRepoPermissionsInOrg, - ).toHaveBeenCalledWith({ - org: 'blam', - team_slug: 'team', - owner: 'blam', - repo: 'test', - permission: 'admin', - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'x-access-token', password: 'fake-token' }, - logger, - }); - }); - - it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { - const publisher = await GithubPublisher.fromConfig( - { - token: 'fake-token', - host: 'github.com', - }, - { repoVisibility: 'public' }, - ); - - mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ - data: { - clone_url: 'https://github.com/backstage/backstage.git', - }, - } as RestEndpointMethodTypes['repos']['createForAuthenticatedUser']['response']); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'User', - }, - } as RestEndpointMethodTypes['users']['getByUsername']['response']); - - const result = await publisher!.publish({ - values: { - storePath: 'https://github.com/blam/test', - owner: 'bob', - access: 'blam', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://github.com/backstage/backstage.git', - catalogInfoUrl: - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - }); - expect( - mockGithubClient.repos.createForAuthenticatedUser, - ).toHaveBeenCalledWith({ - name: 'test', - private: false, - }); - expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled(); - - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'x-access-token', password: 'fake-token' }, - logger, - }); - }); - }); - - it('should invite other user in the authed user', async () => { - const publisher = await GithubPublisher.fromConfig( - { - token: 'fake-token', - host: 'github.com', - }, - { repoVisibility: 'public' }, - ); - - mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ - data: { - clone_url: 'https://github.com/backstage/backstage.git', - }, - } as RestEndpointMethodTypes['repos']['createForAuthenticatedUser']['response']); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'User', - }, - } as RestEndpointMethodTypes['users']['getByUsername']['response']); - - const result = await publisher!.publish({ - values: { - storePath: 'https://github.com/blam/test', - owner: 'bob', - access: 'bob', - description: 'description', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://github.com/backstage/backstage.git', - catalogInfoUrl: - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - }); - expect( - mockGithubClient.repos.createForAuthenticatedUser, - ).toHaveBeenCalledWith({ - description: 'description', - name: 'test', - private: false, - }); - expect(mockGithubClient.repos.addCollaborator).toHaveBeenCalledWith({ - owner: 'blam', - repo: 'test', - username: 'bob', - permission: 'admin', - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'x-access-token', password: 'fake-token' }, - logger, - }); - }); - }); - - describe('with internal repo visibility', () => { - it('creates a private repository in the organization with visibility set to internal', async () => { - const publisher = await GithubPublisher.fromConfig( - { - token: 'fake-token', - host: 'github.com', - }, - { repoVisibility: 'internal' }, - ); - - mockGithubClient.repos.createInOrg.mockResolvedValue({ - data: { - clone_url: 'https://github.com/backstage/backstage.git', - }, - } as RestEndpointMethodTypes['repos']['createInOrg']['response']); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'Organization', - }, - } as RestEndpointMethodTypes['users']['getByUsername']['response']); - - const result = await publisher!.publish({ - values: { - isOrg: true, - storePath: 'https://github.com/blam/test', - owner: 'bob', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://github.com/backstage/backstage.git', - catalogInfoUrl: - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - }); - expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ - org: 'blam', - name: 'test', - private: true, - visibility: 'internal', - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'x-access-token', password: 'fake-token' }, - logger, - }); - }); - }); - - describe('private visibility in a user account', () => { - it('creates a private repository', async () => { - const publisher = await GithubPublisher.fromConfig( - { - token: 'fake-token', - host: 'github.com', - }, - { repoVisibility: 'private' }, - ); - - mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ - data: { - clone_url: 'https://github.com/backstage/backstage.git', - }, - } as RestEndpointMethodTypes['repos']['createForAuthenticatedUser']['response']); - mockGithubClient.users.getByUsername.mockResolvedValue({ - data: { - type: 'User', - }, - } as RestEndpointMethodTypes['users']['getByUsername']['response']); - - const result = await publisher!.publish({ - values: { - storePath: 'https://github.com/blam/test', - owner: 'bob', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'https://github.com/backstage/backstage.git', - catalogInfoUrl: - 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - }); - expect( - mockGithubClient.repos.createForAuthenticatedUser, - ).toHaveBeenCalledWith({ - name: 'test', - private: true, - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'x-access-token', password: 'fake-token' }, - logger, - }); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts deleted file mode 100644 index 9b4b1d48f5..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PublisherBase, PublisherOptions, PublisherResult } from './types'; -import { - enableBranchProtectionOnDefaultRepoBranch, - initRepoAndPush, -} from './helpers'; -import { - GitHubIntegrationConfig, - GithubCredentialsProvider, -} from '@backstage/integration'; -import parseGitUrl from 'git-url-parse'; -import { Octokit } from '@octokit/rest'; -import path from 'path'; - -export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; - -/** @deprecated use createPublishGithubAction instead */ -export class GithubPublisher implements PublisherBase { - static async fromConfig( - config: GitHubIntegrationConfig, - { repoVisibility }: { repoVisibility: RepoVisibilityOptions }, - ) { - if (!config.token && !config.apps) { - return undefined; - } - - const credentialsProvider = GithubCredentialsProvider.create(config); - - return new GithubPublisher({ - credentialsProvider, - repoVisibility, - apiBaseUrl: config.apiBaseUrl, - }); - } - - constructor( - private readonly config: { - credentialsProvider: GithubCredentialsProvider; - repoVisibility: RepoVisibilityOptions; - apiBaseUrl: string | undefined; - }, - ) {} - - async publish({ - values, - workspacePath, - logger, - }: PublisherOptions): Promise { - const { owner, name } = parseGitUrl(values.storePath); - - const { token } = await this.config.credentialsProvider.getCredentials({ - url: values.storePath, - }); - - if (!token) { - throw new Error( - `No token could be acquired for URL: ${values.storePath}`, - ); - } - - const client = new Octokit({ - auth: token, - baseUrl: this.config.apiBaseUrl, - previews: ['nebula-preview'], - }); - - const description = values.description as string; - const access = values.access as string; - const remoteUrl = await this.createRemote({ - client, - description, - access, - name, - owner, - }); - - await initRepoAndPush({ - dir: path.join(workspacePath, 'result'), - remoteUrl, - auth: { - username: 'x-access-token', - password: token, - }, - logger, - }); - - const catalogInfoUrl = remoteUrl.replace( - /\.git$/, - '/blob/master/catalog-info.yaml', - ); - - try { - await enableBranchProtectionOnDefaultRepoBranch({ - owner, - client, - repoName: name, - logger, - }); - } catch (e) { - throw new Error(`Failed to add branch protection to '${name}', ${e}`); - } - - return { remoteUrl, catalogInfoUrl }; - } - - private async createRemote(opts: { - client: Octokit; - access: string; - name: string; - owner: string; - description: string; - }) { - const { client, access, description, owner, name } = opts; - - const user = await client.users.getByUsername({ - username: owner, - }); - - const repoCreationPromise = - user.data.type === 'Organization' - ? client.repos.createInOrg({ - name, - org: owner, - private: this.config.repoVisibility !== 'public', - visibility: this.config.repoVisibility, - description, - }) - : client.repos.createForAuthenticatedUser({ - name, - private: this.config.repoVisibility === 'private', - description, - }); - - const { data: newRepo } = await repoCreationPromise; - - try { - if (access?.startsWith(`${owner}/`)) { - const [, team] = access.split('/'); - await client.teams.addOrUpdateRepoPermissionsInOrg({ - org: owner, - team_slug: team, - owner, - repo: name, - permission: 'admin', - }); - // no need to add access if it's the person who owns the personal account - } else if (access && access !== owner) { - await client.repos.addCollaborator({ - owner, - repo: name, - username: access, - permission: 'admin', - }); - } - } catch (e) { - throw new Error( - `Failed to add access to '${access}'. Status ${e.status} ${e.message}`, - ); - } - - return newRepo.clone_url; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts deleted file mode 100644 index 5c6a01afc8..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -jest.mock('@gitbeaker/node', () => ({ - Gitlab: jest.fn(), -})); - -jest.mock('./helpers'); - -import os from 'os'; -import path from 'path'; -import { GitlabPublisher } from './gitlab'; -import { Gitlab } from '@gitbeaker/node'; -import { initRepoAndPush } from './helpers'; -import { getVoidLogger } from '@backstage/backend-common'; - -describe('GitLab Publisher', () => { - const logger = getVoidLogger(); - const mockGitlabClient = { - Namespaces: { - show: jest.fn(), - }, - Projects: { - create: jest.fn(), - }, - Users: { - current: jest.fn(), - }, - }; - - beforeEach(() => { - jest.clearAllMocks(); - - ((Gitlab as unknown) as jest.Mock).mockImplementation( - () => mockGitlabClient, - ); - }); - - const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - const resultPath = path.resolve(workspacePath, 'result'); - - describe('publish: createRemoteInGitLab', () => { - it('should use gitbeaker to create a repo in a namespace if the namespace property is set', async () => { - const publisher = await GitlabPublisher.fromConfig( - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: 'fake-token', - baseUrl: 'https://gitlab.hosted.com', - }, - { repoVisibility: 'public' }, - ); - - mockGitlabClient.Namespaces.show.mockResolvedValue({ - id: 42, - } as { id: number }); - mockGitlabClient.Projects.create.mockResolvedValue({ - http_url_to_repo: 'mockclone', - } as { http_url_to_repo: string }); - - const result = await publisher!.publish({ - values: { - isOrg: true, - storePath: 'https://gitlab.com/blam/test', - owner: 'bob', - }, - workspacePath, - logger, - }); - - expect(Gitlab).toHaveBeenCalledWith({ - token: 'fake-token', - host: 'https://gitlab.hosted.com', - }); - expect(result).toEqual({ - remoteUrl: 'mockclone', - catalogInfoUrl: 'mockclone', - }); - expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ - namespace_id: 42, - name: 'test', - visibility: 'public', - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'mockclone', - auth: { username: 'oauth2', password: 'fake-token' }, - logger, - }); - }); - - it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => { - const publisher = await GitlabPublisher.fromConfig( - { - host: 'gitlab.com', - apiBaseUrl: 'https://gitlab.com/api/v4', - token: 'fake-token', - baseUrl: 'https://gitlab.com', - }, - { repoVisibility: 'public' }, - ); - - mockGitlabClient.Namespaces.show.mockResolvedValue({}); - mockGitlabClient.Users.current.mockResolvedValue({ - id: 21, - } as { id: number }); - mockGitlabClient.Projects.create.mockResolvedValue({ - http_url_to_repo: 'mockclone', - } as { http_url_to_repo: string }); - - const result = await publisher!.publish({ - values: { - storePath: 'https://gitlab.com/blam/test', - owner: 'bob', - }, - workspacePath, - logger, - }); - - expect(result).toEqual({ - remoteUrl: 'mockclone', - catalogInfoUrl: 'mockclone', - }); - expect(mockGitlabClient.Users.current).toHaveBeenCalled(); - expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ - namespace_id: 21, - name: 'test', - visibility: 'public', - }); - expect(initRepoAndPush).toHaveBeenCalledWith({ - dir: resultPath, - remoteUrl: 'mockclone', - auth: { username: 'oauth2', password: 'fake-token' }, - logger, - }); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts deleted file mode 100644 index d2f58900a8..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PublisherBase, PublisherOptions, PublisherResult } from './types'; -import { Gitlab } from '@gitbeaker/node'; -import { Gitlab as GitlabClient } from '@gitbeaker/core'; -import { initRepoAndPush } from './helpers'; -import parseGitUrl from 'git-url-parse'; -import path from 'path'; -import { GitLabIntegrationConfig } from '@backstage/integration'; - -export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; - -export class GitlabPublisher implements PublisherBase { - static async fromConfig( - config: GitLabIntegrationConfig, - { repoVisibility }: { repoVisibility: RepoVisibilityOptions }, - ) { - if (!config.token) { - return undefined; - } - - const client = new Gitlab({ host: config.baseUrl, token: config.token }); - return new GitlabPublisher({ - token: config.token, - client, - repoVisibility, - }); - } - - constructor( - private readonly config: { - token: string; - client: GitlabClient; - repoVisibility: RepoVisibilityOptions; - }, - ) {} - - async publish({ - values, - workspacePath, - logger, - }: PublisherOptions): Promise { - const { owner, name } = parseGitUrl(values.storePath); - - const remoteUrl = await this.createRemote({ - owner, - name, - }); - - await initRepoAndPush({ - dir: path.join(workspacePath, 'result'), - remoteUrl, - auth: { - username: 'oauth2', - password: this.config.token, - }, - logger, - }); - - const catalogInfoUrl = remoteUrl.replace( - /\.git$/, - '/-/blob/master/catalog-info.yaml', - ); - return { remoteUrl, catalogInfoUrl }; - } - - private async createRemote(opts: { name: string; owner: string }) { - const { owner, name } = opts; - - // TODO(blam): this needs cleaning up to be nicer. The amount of brackets is too damn high! - // Shouldn't have to cast things now - let targetNamespace = ((await this.config.client.Namespaces.show( - owner, - )) as { - id: number; - }).id; - - if (!targetNamespace) { - targetNamespace = ((await this.config.client.Users.current()) as { - id: number; - }).id; - } - - const project = (await this.config.client.Projects.create({ - namespace_id: targetNamespace, - name: name, - visibility: this.config.repoVisibility, - })) as { http_url_to_repo: string }; - - return project?.http_url_to_repo; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts deleted file mode 100644 index 7a384f1657..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export { AzurePublisher } from './azure'; -export { BitbucketPublisher } from './bitbucket'; -export { GithubPublisher } from './github'; -export type { RepoVisibilityOptions } from './github'; -export { GitlabPublisher } from './gitlab'; -export { Publishers } from './publishers'; -export type { - PublisherBase, - PublisherBuilder, - PublisherOptions, - PublisherResult, -} from './types'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts deleted file mode 100644 index c7ca832752..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Publishers } from './publishers'; -import { GithubPublisher } from './github'; -import { getVoidLogger } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; -import { AzurePublisher } from './azure'; -import { GitlabPublisher } from './gitlab'; -import { BitbucketPublisher } from './bitbucket'; - -jest.mock('@octokit/rest'); -jest.mock('azure-devops-node-api'); - -describe('Publishers', () => { - const logger = getVoidLogger(); - - it('should throw an error when the publisher for the source location is not registered', () => { - const publishers = new Publishers(); - - expect(() => publishers.get('https://github.com/org/repo')).toThrow( - expect.objectContaining({ - message: - 'Unable to find a publisher for URL: https://github.com/org/repo. Please make sure to register this host under an integration in app-config', - }), - ); - }); - - it('should return the correct preparer when the source matches for github', async () => { - const publishers = await Publishers.fromConfig( - new ConfigReader({ - integrations: { - github: [{ host: 'github.com', token: 'blob' }], - }, - }), - { - logger, - }, - ); - - expect(publishers.get('https://github.com/org/repo')).toBeInstanceOf( - GithubPublisher, - ); - }); - - it('should return the correct preparer when the source matches for azure', async () => { - const publishers = await Publishers.fromConfig( - new ConfigReader({ - integrations: { - azure: [{ host: 'dev.azure.com', token: 'blob' }], - }, - }), - { - logger, - }, - ); - - expect( - publishers.get('https://dev.azure.com/org/project/_git/repo'), - ).toBeInstanceOf(AzurePublisher); - }); - - it('should return the correct preparer when the source matches for bitbucket', async () => { - const publishers = await Publishers.fromConfig( - new ConfigReader({ - integrations: { - bitbucket: [ - { host: 'bitbucket.com', username: 'foo', token: 'blob' }, - ], - }, - }), - { - logger, - }, - ); - expect(publishers.get('https://bitbucket.org/owner/repo')).toBeInstanceOf( - BitbucketPublisher, - ); - }); - - it('should return the correct preparer when the source matches for gitlab', async () => { - const publishers = await Publishers.fromConfig( - new ConfigReader({ - integrations: { - gitlab: [{ host: 'gitlab.com', token: 'blob' }], - }, - }), - { - logger, - }, - ); - expect(publishers.get('https://gitlab.com/owner/repo')).toBeInstanceOf( - GitlabPublisher, - ); - }); - - it('should respect registrations for custom URLs for providers using the integrations config', async () => { - const publishers = await Publishers.fromConfig( - new ConfigReader({ - integrations: { - github: [ - { host: 'my.special.github.enterprise.thing', token: 'lolghe' }, - ], - }, - }), - { - logger, - }, - ); - - expect( - publishers.get('https://my.special.github.enterprise.thing/org/repo'), - ).toBeInstanceOf(GithubPublisher); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts deleted file mode 100644 index 9c34ab5783..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Config } from '@backstage/config'; -import { PublisherBase, PublisherBuilder } from './types'; -import { - GithubPublisher, - RepoVisibilityOptions as GithubRepoVisibilityOptions, -} from './github'; -import { - GitlabPublisher, - RepoVisibilityOptions as GitlabRepoVisibilityOptions, -} from './gitlab'; -import { AzurePublisher } from './azure'; -import { - BitbucketPublisher, - RepoVisibilityOptions as BitbucketRepoVisibilityOptions, -} from './bitbucket'; -import { Logger } from 'winston'; -import { ScmIntegrations } from '@backstage/integration'; - -export class Publishers implements PublisherBuilder { - private publisherMap = new Map(); - - register(host: string, preparer: PublisherBase | undefined) { - this.publisherMap.set(host, preparer); - } - - get(url: string): PublisherBase { - const preparer = this.publisherMap.get(new URL(url).host); - if (!preparer) { - throw new Error( - `Unable to find a publisher for URL: ${url}. Please make sure to register this host under an integration in app-config`, - ); - } - return preparer; - } - - static async fromConfig( - config: Config, - _options: { logger: Logger }, - ): Promise { - const publishers = new Publishers(); - - const scm = ScmIntegrations.fromConfig(config); - - for (const integration of scm.azure.list()) { - const publisher = await AzurePublisher.fromConfig(integration.config); - if (publisher) { - publishers.register(integration.config.host, publisher); - } - } - - for (const integration of scm.github.list()) { - const repoVisibility = (config.getOptionalString( - 'scaffolder.github.visibility', - ) ?? 'public') as GithubRepoVisibilityOptions; - - const publisher = await GithubPublisher.fromConfig(integration.config, { - repoVisibility, - }); - if (publisher) { - publishers.register(integration.config.host, publisher); - } - } - - for (const integration of scm.gitlab.list()) { - const repoVisibility = (config.getOptionalString( - 'scaffolder.gitlab.visibility', - ) ?? 'public') as GitlabRepoVisibilityOptions; - - const publisher = await GitlabPublisher.fromConfig(integration.config, { - repoVisibility, - }); - - if (publisher) { - publishers.register(integration.config.host, publisher); - } - } - - for (const integration of scm.bitbucket.list()) { - const repoVisibility = (config.getOptionalString( - 'scaffolder.bitbucket.visibility', - ) ?? 'public') as BitbucketRepoVisibilityOptions; - - const publisher = await BitbucketPublisher.fromConfig( - integration.config, - { - repoVisibility, - }, - ); - - if (publisher) { - publishers.register(integration.config.host, publisher); - } - } - - return publishers; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts deleted file mode 100644 index 725816c918..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { TemplaterValues } from '../templater'; -import { Logger } from 'winston'; - -/** - * Publisher is in charge of taking a folder created by - * the templater, and pushing it to a remote storage - */ -export type PublisherBase = { - /** - * - * @param opts object containing the template entity from the service - * catalog, plus the values from the form and the directory that has - * been templated - */ - publish(opts: PublisherOptions): Promise; -}; - -export type PublisherOptions = { - values: TemplaterValues; - workspacePath: string; - logger: Logger; -}; - -export type PublisherResult = { - remoteUrl: string; - catalogInfoUrl?: string; -}; - -export type PublisherBuilder = { - register(host: string, publisher: PublisherBase): void; - get(storePath: string): PublisherBase; -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts deleted file mode 100644 index 1db5b52ffa..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const runCommand = jest.fn(); -const commandExists = jest.fn(); - -jest.mock('./helpers', () => ({ runCommand })); -jest.mock('command-exists', () => commandExists); -jest.mock('fs-extra'); - -import { ContainerRunner } from '@backstage/backend-common'; -import fs from 'fs-extra'; -import parseGitUrl from 'git-url-parse'; -import path from 'path'; -import { PassThrough } from 'stream'; -import { CookieCutter } from './cookiecutter'; - -describe('CookieCutter Templater', () => { - const containerRunner: jest.Mocked = { - runContainer: jest.fn(), - }; - - beforeEach(() => { - jest.clearAllMocks(); - commandExists.mockRejectedValue(null); - }); - - it('should write a cookiecutter.json file with the values from the entity', async () => { - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - description: 'description', - component_id: 'newthing', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - }); - - expect(fs.ensureDir).toBeCalledWith(path.join('tempdir', 'intermediate')); - expect(fs.writeJson).toBeCalledWith( - path.join('tempdir', 'template', 'cookiecutter.json'), - expect.objectContaining(values), - ); - }); - - it('should merge any value that is in the cookiecutter.json path already', async () => { - const existingJson = { - _copy_without_render: ['./github/workflows/*'], - }; - - jest - .spyOn(fs, 'readJSON') - .mockImplementationOnce(() => Promise.resolve(existingJson)); - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - component_id: 'something', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - }); - - expect(fs.writeJSON).toBeCalledWith( - path.join('tempdir', 'template', 'cookiecutter.json'), - { - ...existingJson, - ...values, - destination: { - git: expect.objectContaining({ organization: 'org', name: 'repo' }), - }, - }, - ); - }); - - it('should throw an error if the cookiecutter json is malformed and not missing', async () => { - jest.spyOn(fs, 'readJSON').mockImplementationOnce(() => { - throw new Error('BAM'); - }); - - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - const templater = new CookieCutter({ containerRunner }); - await expect( - templater.run({ - workspacePath: 'tempdir', - values, - }), - ).rejects.toThrow('BAM'); - }); - - it('should run the correct docker container with the correct bindings for the volumes', async () => { - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - component_id: 'newthing', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - jest - .spyOn(fs, 'realpath') - .mockImplementation(x => Promise.resolve(x.toString())); - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - }); - - expect(containerRunner.runContainer).toHaveBeenCalledWith({ - imageName: 'spotify/backstage-cookiecutter', - command: 'cookiecutter', - args: ['--no-input', '-o', '/output', '/input', '--verbose'], - envVars: { HOME: '/tmp' }, - mountDirs: { - [path.join('tempdir', 'template')]: '/input', - [path.join('tempdir', 'intermediate')]: '/output', - }, - workingDir: '/input', - logStream: undefined, - }); - }); - - it('should run the docker container mentioned in configs, overriding the default', async () => { - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - imageName: 'foo/cookiecutter-image-with-extensions', - }; - - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - }); - - expect(containerRunner.runContainer).toHaveBeenCalledWith( - expect.objectContaining({ - imageName: 'foo/cookiecutter-image-with-extensions', - }), - ); - }); - - it('should pass through the streamer to the run docker helper', async () => { - const stream = new PassThrough(); - - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - component_id: 'newthing', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - logStream: stream, - }); - - expect(containerRunner.runContainer).toHaveBeenCalledWith({ - imageName: 'spotify/backstage-cookiecutter', - command: 'cookiecutter', - args: ['--no-input', '-o', '/output', '/input', '--verbose'], - envVars: { HOME: '/tmp' }, - mountDirs: { - [path.join('tempdir', 'template')]: '/input', - [path.join('tempdir', 'intermediate')]: '/output', - }, - workingDir: '/input', - logStream: stream, - }); - }); - - describe('when cookiecutter is available', () => { - it('use the binary', async () => { - const stream = new PassThrough(); - - const values = { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - component_id: 'newthing', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }; - - jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any); - commandExists.mockResolvedValueOnce(true); - - const templater = new CookieCutter({ containerRunner }); - await templater.run({ - workspacePath: 'tempdir', - values, - logStream: stream, - }); - - expect(runCommand).toHaveBeenCalledWith({ - command: 'cookiecutter', - args: expect.arrayContaining([ - '--no-input', - '-o', - path.join('tempdir', 'intermediate'), - path.join('tempdir', 'template'), - '--verbose', - ]), - logStream: stream, - }); - }); - }); - - describe('when nothing was generated', () => { - it('throws an error', async () => { - const stream = new PassThrough(); - - jest - .spyOn(fs, 'readdir') - .mockImplementationOnce(() => Promise.resolve([])); - - const templater = new CookieCutter({ containerRunner }); - await expect( - templater.run({ - workspacePath: 'tempdir', - values: { - owner: 'blobby', - storePath: 'https://github.com/org/repo', - destination: { - git: parseGitUrl('https://github.com/org/repo'), - }, - }, - logStream: stream, - }), - ).rejects.toThrow(/No data generated by cookiecutter/); - }); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts deleted file mode 100644 index ef51fc20a9..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ContainerRunner } from '@backstage/backend-common'; -import { JsonValue } from '@backstage/config'; -import commandExists from 'command-exists'; -import fs from 'fs-extra'; -import path from 'path'; -import { runCommand } from './helpers'; -import { TemplaterBase, TemplaterRunOptions } from './types'; - -export class CookieCutter implements TemplaterBase { - private readonly containerRunner: ContainerRunner; - - constructor({ containerRunner }: { containerRunner: ContainerRunner }) { - this.containerRunner = containerRunner; - } - - private async fetchTemplateCookieCutter( - directory: string, - ): Promise> { - try { - return await fs.readJSON(path.join(directory, 'cookiecutter.json')); - } catch (ex) { - if (ex.code !== 'ENOENT') { - throw ex; - } - - return {}; - } - } - - public async run({ - workspacePath, - values, - logStream, - }: TemplaterRunOptions): Promise { - const templateDir = path.join(workspacePath, 'template'); - const intermediateDir = path.join(workspacePath, 'intermediate'); - await fs.ensureDir(intermediateDir); - const resultDir = path.join(workspacePath, 'result'); - - // First lets grab the default cookiecutter.json file - const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); - - const { imageName, ...valuesForCookieCutterJson } = values; - const cookieInfo = { - ...cookieCutterJson, - ...valuesForCookieCutterJson, - }; - - await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); - - // Directories to bind on container - const mountDirs = { - [templateDir]: '/input', - [intermediateDir]: '/output', - }; - - // the command-exists package returns `true` or throws an error - const cookieCutterInstalled = await commandExists('cookiecutter').catch( - () => false, - ); - if (cookieCutterInstalled) { - await runCommand({ - command: 'cookiecutter', - args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'], - logStream, - }); - } else { - await this.containerRunner.runContainer({ - imageName: imageName || 'spotify/backstage-cookiecutter', - command: 'cookiecutter', - args: ['--no-input', '-o', '/output', '/input', '--verbose'], - mountDirs, - workingDir: '/input', - // Set the home directory inside the container as something that applications can - // write to, otherwise they will just fail trying to write to / - envVars: { HOME: '/tmp' }, - logStream, - }); - } - - // if cookiecutter was successful, intermediateDir will contain - // exactly one directory. - const [generated] = await fs.readdir(intermediateDir); - - if (generated === undefined) { - throw new Error('No data generated by cookiecutter'); - } - - await fs.move(path.join(intermediateDir, generated), resultDir); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts deleted file mode 100644 index 46bd9094ac..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ContainerRunner } from '@backstage/backend-common'; -import fs from 'fs-extra'; -import path from 'path'; -import * as yaml from 'yaml'; -import { TemplaterBase, TemplaterRunOptions } from '../types'; - -// TODO(blam): Replace with the universal import from github-actions after a release -// As it will break the E2E without it -const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug'; - -export class CreateReactAppTemplater implements TemplaterBase { - private readonly containerRunner: ContainerRunner; - - constructor({ containerRunner }: { containerRunner: ContainerRunner }) { - this.containerRunner = containerRunner; - } - - public async run({ - workspacePath, - values, - logStream, - }: TemplaterRunOptions): Promise { - const { - component_id: componentName, - use_typescript: withTypescript, - use_github_actions: withGithubActions, - description, - owner, - } = values; - const intermediateDir = path.join(workspacePath, 'template'); - await fs.ensureDir(intermediateDir); - - const mountDirs = { - [intermediateDir]: '/template', - [intermediateDir]: '/result', - }; - - await this.containerRunner.runContainer({ - imageName: 'node:lts-alpine', - command: ['npx'], - args: [ - 'create-react-app', - componentName as string, - withTypescript ? ' --template typescript' : '', - ], - mountDirs, - workingDir: '/result', - logStream: logStream, - // Set the home directory inside the container as something that applications can - // write to, otherwise they will just fail trying to write to / - envVars: { HOME: '/tmp' }, - }); - - // if cookiecutter was successful, intermediateDir will contain - // exactly one directory. - const [generated] = await fs.readdir(intermediateDir); - - if (generated === undefined) { - throw new Error('No data generated by cookiecutter'); - } - - const resultDir = path.join(workspacePath, 'result'); - await fs.move(path.join(intermediateDir, generated), resultDir); - - const extraAnnotations: Record = {}; - if (withGithubActions) { - await fs.mkdir(`${resultDir}/.github`); - await fs.mkdir(`${resultDir}/.github/workflows`); - const githubActionsYaml = ` -name: CRA Build - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [12.x] - - steps: - - name: checkout code - uses: actions/checkout@v1 - - name: get yarn cache - id: yarn-cache - run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 - with: - path: \${{ steps.yarn-cache.outputs.dir }} - key: \${{ runner.os }}-yarn-\${{ hashFiles('**/yarn.lock') }} - restore-keys: | - \${{ runner.os }}-yarn- - - name: use node.js \${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: \${{ matrix.node-version }} - - name: yarn install, build, and test - working-directory: . - run: | - yarn install - yarn build - yarn test - env: - CI: true - `; - await fs.writeFile( - `${resultDir}/.github/workflows/main.yml`, - githubActionsYaml, - ); - - extraAnnotations[ - GITHUB_ACTIONS_ANNOTATION - ] = `${values?.destination?.git?.owner}/${values?.destination?.git?.name}`; - } - - const componentInfo = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: componentName, - description, - annotations: { - ...extraAnnotations, - }, - }, - spec: { - type: 'website', - lifecycle: 'experimental', - owner, - }, - }; - - await fs.writeFile( - `${resultDir}/catalog-info.yaml`, - yaml.stringify(componentInfo), - ); - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts deleted file mode 100644 index c2742428a5..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { InputError } from '@backstage/errors'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { spawn } from 'child_process'; -import { PassThrough, Writable } from 'stream'; - -export type RunCommandOptions = { - command: string; - args: string[]; - logStream?: Writable; -}; - -/** - * Gets the templater key to use for templating from the entity - * @param entity Template entity - */ -export const getTemplaterKey = (entity: TemplateEntityV1alpha1): string => { - const { templater } = entity.spec; - - if (!templater) { - throw new InputError('Template does not have a required templating key'); - } - - return templater; -}; - -/** - * - * @param options the options object - * @param options.command the command to run - * @param options.args the arguments to pass the command - * @param options.logStream the log streamer to capture log messages - */ -export const runCommand = async ({ - command, - args, - logStream = new PassThrough(), -}: RunCommandOptions) => { - await new Promise((resolve, reject) => { - const process = spawn(command, args); - - process.stdout.on('data', stream => { - logStream.write(stream); - }); - - process.stderr.on('data', stream => { - logStream.write(stream); - }); - - process.on('error', error => { - return reject(error); - }); - - process.on('close', code => { - if (code !== 0) { - return reject(`Command ${command} failed, exit code: ${code}`); - } - return resolve(); - }); - }); -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts deleted file mode 100644 index 4d80f86946..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -export * from './cookiecutter'; -export * from './types'; -export * from './helpers'; -export * from './templaters'; -export * from './cra'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts deleted file mode 100644 index ae3890c092..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ContainerRunner } from '@backstage/backend-common'; -import { CookieCutter } from './cookiecutter'; -import { Templaters } from './templaters'; - -describe('Templaters', () => { - const containerRunner: jest.Mocked = { - runContainer: jest.fn(), - }; - - it('should throw an error when the templater is not registered', () => { - const templaters = new Templaters(); - - expect(() => templaters.get('cookiecutter')).toThrow( - expect.objectContaining({ - message: 'No templater registered for template: "cookiecutter"', - }), - ); - }); - it('should return the correct templater when the templater matches', () => { - const templaters = new Templaters(); - const templater = new CookieCutter({ containerRunner }); - - templaters.register('cookiecutter', templater); - - expect(templaters.get('cookiecutter')).toBe(templater); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts deleted file mode 100644 index a977502a01..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - TemplaterBase, - SupportedTemplatingKey, - TemplaterBuilder, -} from './types'; - -export class Templaters implements TemplaterBuilder { - private templaterMap = new Map(); - - register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) { - this.templaterMap.set(templaterKey, templater); - } - - get(templaterId: string): TemplaterBase { - const templater = this.templaterMap.get(templaterId); - - if (!templater) { - throw new Error(`No templater registered for template: "${templaterId}"`); - } - - return templater; - } -} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts deleted file mode 100644 index 4d6b96c776..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import gitUrlParse from 'git-url-parse'; -import type { Writable } from 'stream'; - -/** - * Currently the required template values. The owner - * and where to store the result from templating - */ -export type RequiredTemplateValues = { - owner: string; - storePath: string; - destination?: { - git?: gitUrlParse.GitUrl; - }; -}; - -export type TemplaterValues = RequiredTemplateValues & Record; - -/** - * The returned directory from the templater which is ready - * to pass to the next stage of the scaffolder which is publishing - */ -export type TemplaterRunResult = { - resultDir: string; -}; - -/** - * The values that the templater will receive. The directory of the - * skeleton, with the values from the frontend. A dedicated log stream and a docker - * client to run any templater on top of your directory. - */ -export type TemplaterRunOptions = { - workspacePath: string; - values: TemplaterValues; - logStream?: Writable; -}; - -export type TemplaterBase = { - // runs the templating with the values and returns the directory to push the VCS - run(opts: TemplaterRunOptions): Promise; -}; - -export type TemplaterConfig = { - templater?: TemplaterBase; -}; - -/** - * List of supported templating options - */ -export type SupportedTemplatingKey = 'cookiecutter' | string; - -/** - * The templater builder holds the templaters ready for run time - */ -export type TemplaterBuilder = { - register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void; - get(templater: string): TemplaterBase; -}; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts deleted file mode 100644 index fef92d081f..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { resolve as resolvePath, dirname } from 'path'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import parseGitUrl from 'git-url-parse'; -import { TaskSpec } from './types'; -import { - getTemplaterKey, - joinGitUrlPath, - parseLocationAnnotation, - TemplaterValues, -} from '../stages'; - -export function templateEntityToSpec( - template: TemplateEntityV1alpha1, - inputValues: TemplaterValues, -): TaskSpec { - const steps: TaskSpec['steps'] = []; - - const { protocol, location } = parseLocationAnnotation(template); - - let url: string; - if (protocol === 'file') { - const path = resolvePath(dirname(location), template.spec.path || '.'); - - url = `file://${path}`; - } else { - url = joinGitUrlPath(location, template.spec.path); - } - const templater = getTemplaterKey(template); - - const values = { - ...inputValues, - destination: { - git: parseGitUrl(inputValues.storePath), - }, - } as TemplaterValues; - - steps.push({ - id: 'prepare', - name: 'Prepare', - action: 'legacy:prepare', - input: { - protocol, - url, - }, - }); - - steps.push({ - id: 'template', - name: 'Template', - action: 'legacy:template', - input: { - templater, - values, - }, - }); - - steps.push({ - id: 'publish', - name: 'Publish', - action: 'legacy:publish', - input: { - values, - }, - }); - - steps.push({ - id: 'register', - name: 'Register', - action: 'catalog:register', - input: { - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', - }, - }); - - return { - baseUrl: undefined, // not used by legacy actions - values: {}, - steps, - output: { - remoteUrl: '{{ steps.publish.output.remoteUrl }}', - catalogInfoUrl: '{{ steps.register.output.catalogInfoUrl }}', - entityRef: '{{ steps.register.output.entityRef }}', - }, - }; -} diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 17304aefb1..3ad4646b01 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -18,12 +18,6 @@ import { Config } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { - PreparerBuilder, - TemplaterBuilder, - TemplaterValues, - PublisherBuilder, -} from '../scaffolder'; import { CatalogEntityClient } from '../lib/catalog'; import { validate } from 'jsonschema'; import { @@ -31,27 +25,21 @@ import { StorageTaskBroker, TaskWorker, } from '../scaffolder/tasks'; -import { templateEntityToSpec } from '../scaffolder/tasks/TemplateConverter'; import { TemplateActionRegistry } from '../scaffolder/actions/TemplateActionRegistry'; -import { createLegacyActions } from '../scaffolder/stages/legacy'; import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; -import { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; +import { + ContainerRunner, + PluginDatabaseManager, + UrlReader, +} from '@backstage/backend-common'; import { InputError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; -import { - TemplateEntityV1alpha1, - TemplateEntityV1beta2, - Entity, -} from '@backstage/catalog-model'; +import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; import { ScmIntegrations } from '@backstage/integration'; import { TemplateAction } from '../scaffolder/actions'; import { createBuiltinActions } from '../scaffolder/actions/builtin/createBuiltinActions'; export interface RouterOptions { - preparers: PreparerBuilder; - templaters: TemplaterBuilder; - publishers: PublisherBuilder; - logger: Logger; config: Config; reader: UrlReader; @@ -59,19 +47,11 @@ export interface RouterOptions { catalogClient: CatalogApi; actions?: TemplateAction[]; taskWorkers?: number; -} - -function isAlpha1Template( - entity: TemplateEntityV1alpha1 | TemplateEntityV1beta2, -): entity is TemplateEntityV1alpha1 { - return ( - entity.apiVersion === 'backstage.io/v1alpha1' || - entity.apiVersion === 'backstage.io/v1beta1' - ); + containerRunner: ContainerRunner; } function isBeta2Template( - entity: TemplateEntityV1alpha1 | TemplateEntityV1beta2, + entity: TemplateEntityV1beta2, ): entity is TemplateEntityV1beta2 { return entity.apiVersion === 'backstage.io/v1beta2'; } @@ -83,15 +63,13 @@ export async function createRouter( router.use(express.json()); const { - preparers, - templaters, - publishers, logger: parentLogger, config, reader, database, catalogClient, actions, + containerRunner, taskWorkers, } = options; @@ -118,20 +96,13 @@ export async function createRouter( const actionsToRegister = Array.isArray(actions) ? actions - : [ - ...createLegacyActions({ - preparers, - publishers, - templaters, - }), - ...createBuiltinActions({ - integrations, - catalogClient, - templaters, - reader, - config, - }), - ]; + : createBuiltinActions({ + integrations, + catalogClient, + containerRunner, + reader, + config, + }); actionsToRegister.forEach(action => actionRegistry.register(action)); workers.forEach(worker => worker.start()); @@ -165,42 +136,6 @@ export async function createRouter( schema, })), }); - } else if (isAlpha1Template(template)) { - res.json({ - title: template.metadata.title ?? template.metadata.name, - steps: [ - { - title: 'Fill in template parameters', - schema: template.spec.schema, - }, - { - title: 'Choose owner and repo', - schema: { - type: 'object', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: - 'A full URL to the repository that should be created. e.g https://github.com/backstage/new-repo', - }, - access: { - type: 'string', - title: 'Access', - description: - 'Who should have access, in org/team or user format', - }, - }, - }, - }, - ], - }); } else { throw new InputError( `Unsupported apiVersion field in schema entity, ${ @@ -222,26 +157,14 @@ export async function createRouter( }) .post('/v2/tasks', async (req, res) => { const templateName: string = req.body.templateName; - const values: TemplaterValues = req.body.values; + const values = req.body.values; const token = getBearerToken(req.headers.authorization); const template = await entityClient.findTemplate(templateName, { token, }); let taskSpec; - if (isAlpha1Template(template)) { - logger.warn( - `[DEPRECATION] - Template: ${template.metadata.name} has version ${template.apiVersion} which is going to be deprecated. Please refer to https://backstage.io/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2 for help on migrating`, - ); - - const result = validate(values, template.spec.schema); - if (!result.valid) { - res.status(400).json({ errors: result.errors }); - return; - } - - taskSpec = templateEntityToSpec(template, values); - } else if (isBeta2Template(template)) { + if (isBeta2Template(template)) { for (const parameters of [template.spec.parameters ?? []].flat()) { const result = validate(values, parameters); From a02ea1e4e121c1d2f26b534488cc44d0c87f7efa Mon Sep 17 00:00:00 2001 From: blam Date: Sat, 19 Jun 2021 02:39:44 +0200 Subject: [PATCH 02/18] breaking: some more changes in the frontend. removing support for v1alpha1 Signed-off-by: blam --- .../sample-templates/remote-templates.yaml | 2 +- .../ScaffolderPage/ScaffolderPage.tsx | 32 ++++++-------- .../components/TemplatePage/TemplatePage.tsx | 43 ------------------- .../src/filter/EntityFilterGroupsProvider.tsx | 14 +++--- plugins/scaffolder/src/filter/context.ts | 4 +- 5 files changed, 23 insertions(+), 72 deletions(-) diff --git a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml index b89b21d5e3..7846a0994e 100644 --- a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml @@ -6,4 +6,4 @@ metadata: spec: type: url targets: - # - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + - https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 1d16e50568..29758f4ac3 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -14,7 +14,17 @@ * limitations under the License. */ -import { EntityMeta, TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { EntityMeta, TemplateEntityV1beta2 } from '@backstage/catalog-model'; +import { + Content, + ContentHeader, + Header, + ItemCardGrid, + Lifecycle, + Page, + Progress, + SupportButton, +} from '@backstage/core-components'; import { useStarredEntities } from '@backstage/plugin-catalog-react'; import { Button, Link, makeStyles, Typography } from '@material-ui/core'; import StarIcon from '@material-ui/icons/Star'; @@ -30,18 +40,6 @@ import { TemplateCard } from '../TemplateCard'; import { configApiRef, useApi, useRouteRef } from '@backstage/core-plugin-api'; -import { - Content, - ContentHeader, - Header, - ItemCardGrid, - Lifecycle, - Page, - Progress, - SupportButton, - WarningPanel, -} from '@backstage/core-components'; - const useStyles = makeStyles(theme => ({ contentWrapper: { display: 'grid', @@ -90,7 +88,7 @@ export const ScaffolderPageContents = () => { ); const [search, setSearch] = useState(''); const [matchingEntities, setMatchingEntities] = useState( - [] as TemplateEntityV1alpha1[], + [] as TemplateEntityV1beta2[], ); const matchesQuery = (metadata: EntityMeta, query: string) => @@ -175,11 +173,7 @@ export const ScaffolderPageContents = () => { {matchingEntities && matchingEntities?.length > 0 && matchingEntities.map((template, i) => ( - + ))} diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 3caece4878..b6cab5b620 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -16,7 +16,6 @@ import { JsonObject, JsonValue } from '@backstage/config'; import { LinearProgress } from '@material-ui/core'; import { FieldValidation, FormValidation, IChangeEvent } from '@rjsf/core'; -import parseGitUrl from 'git-url-parse'; import React, { useCallback, useState } from 'react'; import { generatePath, Navigate, useNavigate } from 'react-router'; import { useParams } from 'react-router-dom'; @@ -99,39 +98,6 @@ export const createValidator = ( }; }; -const storePathValidator = ( - formData: { storePath?: string }, - errors: FormValidation, -) => { - const { storePath } = formData; - if (!storePath) { - errors.storePath.addError('Store path is required and not present'); - return errors; - } - - try { - const parsedUrl = parseGitUrl(storePath); - - if (!parsedUrl.resource || !parsedUrl.owner || !parsedUrl.name) { - if (parsedUrl.resource === 'dev.azure.com') { - errors.storePath.addError( - "The store path should be formatted like https://dev.azure.com/{org}/{project}/_git/{repo} for Azure URL's", - ); - } else { - errors.storePath.addError( - 'The store path should be a complete Git URL to the new repository location. For example: https://github.com/{owner}/{repo}', - ); - } - } - } catch (ex) { - errors.storePath.addError( - `Failed validation of the store path with message ${ex.message}`, - ); - } - - return errors; -}; - export const TemplatePage = ({ customFieldExtensions = [], }: { @@ -203,15 +169,6 @@ export const TemplatePage = ({ onReset={handleFormReset} onFinish={handleCreate} steps={schema.steps.map(step => { - // TODO: Can delete this function when the migration from v1 to v2 beta is completed - // And just have the default validator for all fields. - if ((step.schema as any)?.properties?.storePath) { - return { - ...step, - validate: (a, b) => storePathValidator(a, b), - }; - } - return { ...step, validate: createValidator(step.schema, customFieldValidators), diff --git a/plugins/scaffolder/src/filter/EntityFilterGroupsProvider.tsx b/plugins/scaffolder/src/filter/EntityFilterGroupsProvider.tsx index 7aa1a21643..a5be11380f 100644 --- a/plugins/scaffolder/src/filter/EntityFilterGroupsProvider.tsx +++ b/plugins/scaffolder/src/filter/EntityFilterGroupsProvider.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import React, { useCallback, useEffect, useRef, useState } from 'react'; import { useAsyncFn } from 'react-use'; @@ -50,7 +50,7 @@ function useProvideEntityFilters(): FilterGroupsContext { const response = await catalogApi.getEntities({ filter: { kind: 'Template' }, }); - return response.items as TemplateEntityV1alpha1[]; + return response.items as TemplateEntityV1beta2[]; }); const filterGroups = useRef<{ @@ -64,7 +64,7 @@ function useProvideEntityFilters(): FilterGroupsContext { [filterGroupId: string]: FilterGroupStates; }>({}); const [filteredEntities, setFilteredEntities] = useState< - TemplateEntityV1alpha1[] + TemplateEntityV1beta2[] >([]); const [availableCategories, setAvailableCategories] = useState([]); const [isCatalogEmpty, setCatalogEmpty] = useState(false); @@ -161,7 +161,7 @@ function buildStates( filterGroups: { [filterGroupId: string]: FilterGroup }, selectedFilterKeys: { [filterGroupId: string]: Set }, selectedCategories: string[], - entities?: TemplateEntityV1alpha1[], + entities?: TemplateEntityV1beta2[], error?: Error, ): { [filterGroupId: string]: FilterGroupStates } { // On error - all entries are an error state @@ -208,7 +208,7 @@ function buildStates( } // Given all entites, find all possible categories and provide them in a sorted list. -function collectCategories(entities?: TemplateEntityV1alpha1[]): string[] { +function collectCategories(entities?: TemplateEntityV1beta2[]): string[] { const categories = new Set(); (entities || []).forEach(e => { if (e.spec?.type) { @@ -224,9 +224,9 @@ function buildMatchingEntities( filterGroups: { [filterGroupId: string]: FilterGroup }, selectedFilterKeys: { [filterGroupId: string]: Set }, selectedCategories: string[], - entities?: TemplateEntityV1alpha1[], + entities?: TemplateEntityV1beta2[], excludeFilterGroupId?: string, -): TemplateEntityV1alpha1[] { +): TemplateEntityV1beta2[] { // Build one filter fn per filter group const allFilters: EntityFilterFn[] = []; for (const [filterGroupId, filterGroup] of Object.entries(filterGroups)) { diff --git a/plugins/scaffolder/src/filter/context.ts b/plugins/scaffolder/src/filter/context.ts index ee66e3d9da..385d3f6eb5 100644 --- a/plugins/scaffolder/src/filter/context.ts +++ b/plugins/scaffolder/src/filter/context.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { createContext } from 'react'; import { FilterGroup, FilterGroupStates } from './types'; @@ -32,7 +32,7 @@ export type FilterGroupsContext = { loading: boolean; error?: Error; filterGroupStates: { [filterGroupId: string]: FilterGroupStates }; - filteredEntities: TemplateEntityV1alpha1[]; + filteredEntities: TemplateEntityV1beta2[]; availableCategories: string[]; isCatalogEmpty: boolean; }; From 18660e933765f2b93c328bc15f04af7c637cd050 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 30 Jun 2021 20:53:50 +0200 Subject: [PATCH 03/18] chore: fixing merge conflict Signed-off-by: blam --- .../src/scaffolder/actions/builtin/helpers.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts index a024be4f67..e2459a8ca7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts @@ -1,9 +1,5 @@ /* -<<<<<<< HEAD:plugins/scaffolder-backend/src/scaffolder/stages/publish/helpers.ts - * Copyright 2020 The Backstage Authors -======= * Copyright 2021 The Backstage Authors ->>>>>>> breaking: removing alphav1 support for templates:plugins/scaffolder-backend/src/scaffolder/actions/builtin/helpers.ts * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From a41b024a0f092276104b92fdfcc84faf68aa8d87 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 30 Jun 2021 21:18:39 +0200 Subject: [PATCH 04/18] feat: removing the last parts of v1alpha1 and fixing up TypeScript Signed-off-by: blam --- .../builtin/fetch/cookiecutter.test.ts | 186 ------------------ .../actions/builtin/publish/azure.test.ts | 7 +- .../actions/builtin/publish/bitbucket.test.ts | 5 +- .../actions/builtin/publish/github.test.ts | 7 +- .../actions/builtin/publish/gitlab.test.ts | 6 +- .../src/service/router.test.ts | 36 +--- .../ScaffolderPage/ScaffolderPage.tsx | 1 + .../components/TemplateCard/TemplateCard.tsx | 6 +- 8 files changed, 25 insertions(+), 229 deletions(-) delete mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts deleted file mode 100644 index 3ce3052956..0000000000 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -jest.mock('./helpers'); - -import { getVoidLogger, UrlReader } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; -import { ScmIntegrations } from '@backstage/integration'; -import mock from 'mock-fs'; -import os from 'os'; -import { resolve as resolvePath } from 'path'; -import { PassThrough } from 'stream'; -import { Templaters } from '../../../stages/templater'; -import { createFetchCookiecutterAction } from './cookiecutter'; -import { fetchContents } from './helpers'; - -describe('fetch:cookiecutter', () => { - const integrations = ScmIntegrations.fromConfig( - new ConfigReader({ - integrations: { - azure: [ - { host: 'dev.azure.com', token: 'tokenlols' }, - { host: 'myazurehostnotoken.com' }, - ], - }, - }), - ); - - const templaters = new Templaters(); - const cookiecutterTemplater = { run: jest.fn() }; - const mockTmpDir = os.tmpdir(); - const mockContext = { - input: { - url: 'https://google.com/cookie/cutter', - targetPath: 'something', - values: { - help: 'me', - }, - }, - baseUrl: 'somebase', - workspacePath: mockTmpDir, - logger: getVoidLogger(), - logStream: new PassThrough(), - output: jest.fn(), - createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), - }; - - const mockReader: UrlReader = { - read: jest.fn(), - readTree: jest.fn(), - search: jest.fn(), - }; - - const action = createFetchCookiecutterAction({ - integrations, - templaters, - reader: mockReader, - }); - - templaters.register('cookiecutter', cookiecutterTemplater); - - beforeEach(() => { - mock({ [`${mockContext.workspacePath}/result`]: {} }); - jest.restoreAllMocks(); - }); - - afterEach(() => { - mock.restore(); - }); - - it('should call fetchContents with the correct values', async () => { - await action.handler(mockContext); - - expect(fetchContents).toHaveBeenCalledWith({ - reader: mockReader, - integrations, - baseUrl: mockContext.baseUrl, - fetchUrl: mockContext.input.url, - outputPath: resolvePath( - mockContext.workspacePath, - `template/{{cookiecutter and 'contents'}}`, - ), - }); - }); - - it('should execute the cookiecutter templater with the correct values', async () => { - await action.handler(mockContext); - - expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ - workspacePath: mockTmpDir, - logStream: mockContext.logStream, - values: mockContext.input.values, - }); - }); - - it('should execute the cookiecutter templater with optional inputs if they are present and valid', async () => { - await action.handler({ - ...mockContext, - input: { - ...mockContext.input, - copyWithoutRender: ['goreleaser.yml'], - extensions: [ - 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', - ], - imageName: 'foo/cookiecutter-image-with-extensions', - }, - }); - - expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ - workspacePath: mockTmpDir, - logStream: mockContext.logStream, - values: { - ...mockContext.input.values, - _copy_without_render: ['goreleaser.yml'], - _extensions: [ - 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', - ], - imageName: 'foo/cookiecutter-image-with-extensions', - }, - }); - }); - - it('should throw if copyWithoutRender is not an Array', async () => { - await expect( - action.handler({ - ...mockContext, - input: { - ...mockContext.input, - copyWithoutRender: 'xyz', - }, - }), - ).rejects.toThrow(/copyWithoutRender must be an Array/); - }); - - it('should throw if extensions is not an Array', async () => { - await expect( - action.handler({ - ...mockContext, - input: { - ...mockContext.input, - extensions: 'xyz', - }, - }), - ).rejects.toThrow(/extensions must be an Array/); - }); - - it('should throw if there is no cookiecutter templater initialized', async () => { - const templatersWithoutCookiecutter = new Templaters(); - - const newAction = createFetchCookiecutterAction({ - integrations, - templaters: templatersWithoutCookiecutter, - reader: mockReader, - }); - - await expect(newAction.handler(mockContext)).rejects.toThrow( - /No templater registered/, - ); - }); - - it('should throw if the target directory is outside of the workspace path', async () => { - await expect( - action.handler({ - ...mockContext, - input: { - ...mockContext.input, - targetPath: '/foo', - }, - }), - ).rejects.toThrow( - /Relative path is not allowed to refer to a directory outside its parent/, - ); - }); -}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts index 1dc29ea763..30995dee38 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,19 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -jest.mock('../../../stages/publish/helpers'); jest.mock('azure-devops-node-api', () => ({ WebApi: jest.fn(), getPersonalAccessTokenHandler: jest.fn().mockReturnValue(() => {}), })); +jest.mock('../helpers'); + import { createPublishAzureAction } from './azure'; import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { WebApi } from 'azure-devops-node-api'; import { PassThrough } from 'stream'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; describe('publish:azure', () => { const config = new ConfigReader({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts index 0e5f8df72b..9fe245935b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.test.ts @@ -13,7 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -jest.mock('../../../stages/publish/helpers'); + +jest.mock('../helpers'); import { createPublishBitbucketAction } from './bitbucket'; import { rest } from 'msw'; @@ -23,7 +24,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; describe('publish:bitbucket', () => { const config = new ConfigReader({ diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts index 0b277c4739..f0725d6e8e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/github.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -jest.mock('../../../stages/publish/helpers'); + +jest.mock('../helpers'); jest.mock('@octokit/rest'); import { createPublishGithubAction } from './github'; @@ -21,7 +22,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; import { when } from 'jest-when'; describe('publish:github', () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts index 25a129d905..5150205a08 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/gitlab.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -jest.mock('../../../stages/publish/helpers'); +jest.mock('../helpers'); jest.mock('@gitbeaker/node'); import { createPublishGitlabAction } from './gitlab'; @@ -21,7 +21,7 @@ import { ScmIntegrations } from '@backstage/integration'; import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { PassThrough } from 'stream'; -import { initRepoAndPush } from '../../../stages/publish/helpers'; +import { initRepoAndPush } from '../helpers'; describe('publish:gitlab', () => { const config = new ConfigReader({ diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index b43b62a413..d91ec2b7d0 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -33,12 +33,13 @@ import { PluginDatabaseManager, DatabaseManager, UrlReaders, + DockerContainerRunner, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; -import { Preparers, Publishers, Templaters } from '../scaffolder'; +import { TemplateEntityV1beta2 } from '../../../../packages/catalog-model/src'; import { createRouter } from './router'; const createCatalogClient = (templates: any[] = []) => @@ -66,8 +67,8 @@ const mockUrlReader = UrlReaders.default({ describe('createRouter', () => { let app: express.Express; - const template = { - apiVersion: 'backstage.io/v1alpha1', + const template: TemplateEntityV1beta2 = { + apiVersion: 'backstage.io/v1beta2', kind: 'Template', metadata: { description: 'Create a new CRA website project', @@ -80,42 +81,19 @@ describe('createRouter', () => { }, spec: { owner: 'web@example.com', - path: '.', - schema: { - properties: { - component_id: { - description: 'Unique name of the component', - title: 'Name', - type: 'string', - }, - description: { - description: 'Description of the component', - title: 'Description', - type: 'string', - }, - use_typescript: { - default: true, - description: 'Include TypeScript', - title: 'Use TypeScript', - type: 'boolean', - }, - }, - required: ['component_id', 'use_typescript'], - }, - templater: 'cra', type: 'website', + steps: [], + parameters: [], }, }; beforeAll(async () => { const router = await createRouter({ logger: getVoidLogger(), - preparers: new Preparers(), - templaters: new Templaters(), - publishers: new Publishers(), config: new ConfigReader({}), database: createDatabase(), catalogClient: createCatalogClient([template]), + containerRunner: new DockerContainerRunner({} as any), reader: mockUrlReader, }); app = express().use(router); diff --git a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx index 29758f4ac3..65386b60b3 100644 --- a/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx +++ b/plugins/scaffolder/src/components/ScaffolderPage/ScaffolderPage.tsx @@ -21,6 +21,7 @@ import { Header, ItemCardGrid, Lifecycle, + WarningPanel, Page, Progress, SupportButton, diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index ad91fc6cfd..9d2e827be9 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -16,7 +16,7 @@ import { Entity, RELATION_OWNED_BY, - TemplateEntityV1alpha1, + TemplateEntityV1beta2, } from '@backstage/catalog-model'; import { ScmIntegrationIcon, @@ -93,7 +93,7 @@ const useDeprecationStyles = makeStyles(theme => ({ })); export type TemplateCardProps = { - template: TemplateEntityV1alpha1; + template: TemplateEntityV1beta2; deprecated?: boolean; }; @@ -106,7 +106,7 @@ type TemplateProps = { }; const getTemplateCardProps = ( - template: TemplateEntityV1alpha1, + template: TemplateEntityV1beta2, ): TemplateProps & { key: string } => { return { key: template.metadata.uid!, From 7632873678f0360cabd0744614a5a37ec50f4ee8 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 11:02:31 +0200 Subject: [PATCH 05/18] chore: updating api-reports Signed-off-by: blam --- packages/catalog-model/api-report.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 68644bef87..5d055566be 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -492,29 +492,6 @@ export { SystemEntityV1alpha1 } // @public (undocumented) export const systemEntityV1alpha1Validator: KindValidator; -// @public (undocumented) -interface TemplateEntityV1alpha1 extends Entity { - // (undocumented) - apiVersion: 'backstage.io/v1alpha1' | 'backstage.io/v1beta1'; - // (undocumented) - kind: 'Template'; - // (undocumented) - spec: { - type: string; - templater: string; - path?: string; - schema: JSONSchema; - owner?: string; - }; -} - -export { TemplateEntityV1alpha1 as TemplateEntity } - -export { TemplateEntityV1alpha1 } - -// @public (undocumented) -export const templateEntityV1alpha1Validator: KindValidator; - // @public (undocumented) export interface TemplateEntityV1beta2 extends Entity { // (undocumented) From 70bb2a711f04b0bdc213add414556cd6776b6aff Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 14:26:07 +0200 Subject: [PATCH 06/18] chore: fix up some code review comments Signed-off-by: blam --- plugins/catalog-backend/src/ingestion/LocationReaders.ts | 1 - .../src/next/processing/DefaultCatalogProcessingOrchestrator.ts | 1 - plugins/scaffolder-backend/src/service/router.test.ts | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 86a8ebb9f2..624c7aaa61 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -265,7 +265,6 @@ export class LocationReaders implements LocationReader { )}, ${e}`; emit(result.inputError(item.location, message)); logger.warn(message); - logger.debug(e.stack); return undefined; } } diff --git a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts index 235673db03..bb23b6afb3 100644 --- a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts @@ -124,7 +124,6 @@ export class DefaultCatalogProcessingOrchestrator }; } catch (error) { this.options.logger.warn(error.message); - this.options.logger.debug(error.stack); return { ok: false, errors: collector.results().errors.concat(error), diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index d91ec2b7d0..c693573c4f 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -39,7 +39,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; -import { TemplateEntityV1beta2 } from '../../../../packages/catalog-model/src'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { createRouter } from './router'; const createCatalogClient = (templates: any[] = []) => From f0928d674fdd0f0336c13f0d6420a43cb7050d88 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 14:50:39 +0200 Subject: [PATCH 07/18] chore: fix e2e tests as they no longer need all of the Templaters and Publishers Signed-off-by: blam --- .../packages/backend/src/plugins/scaffolder.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index 333ffa11df..ee40b06320 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -24,23 +24,11 @@ export default async function createPlugin({ const dockerClient = new Docker(); const containerRunner = new DockerContainerRunner({ dockerClient }); - const cookiecutterTemplater = new CookieCutter({ containerRunner }); - const craTemplater = new CreateReactAppTemplater({ containerRunner }); - const templaters = new Templaters(); - - templaters.register('cookiecutter', cookiecutterTemplater); - templaters.register('cra', craTemplater); - - const preparers = await Preparers.fromConfig(config, { logger }); - const publishers = await Publishers.fromConfig(config, { logger }); - const discovery = SingleHostDiscovery.fromConfig(config); const catalogClient = new CatalogClient({ discoveryApi: discovery }); return await createRouter({ - preparers, - templaters, - publishers, + containerRunner, logger, config, database, From 350555727fcb85c6ace5728b0977e590d6fb2f4c Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 14:56:18 +0200 Subject: [PATCH 08/18] chore: dont need to export this dep any more Signed-off-by: blam --- plugins/scaffolder-backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 0a0ccd36b1..7173f865c5 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -39,7 +39,6 @@ "@gitbeaker/node": "^30.2.0", "@octokit/rest": "^18.5.3", "@types/express": "^4.17.6", - "@types/git-url-parse": "^9.0.0", "azure-devops-node-api": "^10.2.2", "command-exists": "^1.2.9", "compression": "^1.7.4", @@ -68,6 +67,7 @@ "@backstage/test-utils": "^0.1.14", "@types/command-exists": "^1.2.0", "@types/fs-extra": "^9.0.1", + "@types/git-url-parse": "^9.0.0", "@types/mock-fs": "^4.13.0", "@types/supertest": "^2.0.8", "jest-when": "^3.1.0", From b9cba53a05c85d442e6a6961b7ca1a857e8577d3 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 15:08:36 +0200 Subject: [PATCH 09/18] chore: removing the extra deps Signed-off-by: blam --- .../default-app/packages/backend/src/plugins/scaffolder.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index ee40b06320..a0201cec01 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -4,12 +4,7 @@ import { } from '@backstage/backend-common'; import { CatalogClient } from '@backstage/catalog-client'; import { - CookieCutter, - CreateReactAppTemplater, createRouter, - Preparers, - Publishers, - Templaters, } from '@backstage/plugin-scaffolder-backend'; import Docker from 'dockerode'; import { Router } from 'express'; From 4c689b0fae738935be3b5659e7524f244bc68dd7 Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 16:18:22 +0200 Subject: [PATCH 10/18] chore: making the failing test pass because of parameter validation Signed-off-by: blam --- .../scaffolder-backend/src/service/router.test.ts | 13 +++++++++++-- plugins/scaffolder-backend/src/service/router.ts | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index c693573c4f..227f6d3c88 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -83,7 +83,16 @@ describe('createRouter', () => { owner: 'web@example.com', type: 'website', steps: [], - parameters: [], + parameters: { + type: 'object', + required: ['required'], + properties: { + required: { + type: 'string', + description: 'Required parameter', + }, + }, + }, }, }; @@ -117,7 +126,7 @@ describe('createRouter', () => { const response = await request(app) .post('/v2/tasks') .send({ - templateName: '', + templateName: 'create-react-app-template', values: { storePath: 'https://github.com/backstage/backstage', }, diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 3ad4646b01..2b5bdb3bf8 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -164,6 +164,7 @@ export async function createRouter( }); let taskSpec; + if (isBeta2Template(template)) { for (const parameters of [template.spec.parameters ?? []].flat()) { const result = validate(values, parameters); From 2529ec28a6bad75a9c8e617873af491edea9a2ec Mon Sep 17 00:00:00 2001 From: blam Date: Fri, 2 Jul 2021 16:21:49 +0200 Subject: [PATCH 11/18] chore: fixing all tests Signed-off-by: blam --- plugins/scaffolder-backend/src/service/router.test.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 227f6d3c88..c0fdcb2ef7 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -141,10 +141,7 @@ describe('createRouter', () => { .send({ templateName: 'create-react-app-template', values: { - storePath: 'https://github.com/backstage/backstage', - component_id: '123', - name: 'test', - use_typescript: false, + required: 'required-value', }, }); @@ -161,10 +158,7 @@ describe('createRouter', () => { .send({ templateName: 'create-react-app-template', values: { - storePath: 'https://github.com/backstage/backstage', - component_id: '123', - name: 'test', - use_typescript: false, + required: 'required-value', }, }); From b658d339442f30ac3ee209c8118fcdcb3139183f Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 5 Jul 2021 20:38:27 +0200 Subject: [PATCH 12/18] chore: added some tests for cookiecutter but they need re-writing Signed-off-by: blam --- .../builtin/fetch/cookiecutter.test.ts | 194 ++++++++++++++++++ .../actions/builtin/fetch/cookiecutter.ts | 1 + 2 files changed, 195 insertions(+) create mode 100644 plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts new file mode 100644 index 0000000000..53f255a639 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -0,0 +1,194 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const runCommand = jest.fn(); +const commandExists = jest.fn(); +const fetchContents = jest.fn(); +jest.mock('./helpers', () => ({ runCommand, fetchContents })); +jest.mock('command-exists', () => commandExists); +jest.mock('./helpers'); +jest.mock('fs-extra'); + +import fs from 'fs-extra'; +import { + getVoidLogger, + UrlReader, + ContainerRunner, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import mock from 'mock-fs'; +import os from 'os'; +import { resolve as resolvePath } from 'path'; +import { PassThrough } from 'stream'; +import { createFetchCookiecutterAction } from './cookiecutter'; + +describe('fetch:cookiecutter', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azure: [ + { host: 'dev.azure.com', token: 'tokenlols' }, + { host: 'myazurehostnotoken.com' }, + ], + }, + }), + ); + + const mockTmpDir = os.tmpdir(); + const mockContext = { + input: { + url: 'https://google.com/cookie/cutter', + targetPath: 'something', + values: { + help: 'me', + }, + }, + baseUrl: 'somebase', + workspacePath: mockTmpDir, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), + }; + + const containerRunner: jest.Mocked = { + runContainer: jest.fn(), + }; + + const mockReader: UrlReader = { + read: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + const action = createFetchCookiecutterAction({ + integrations, + containerRunner, + reader: mockReader, + }); + + beforeEach(() => { + (fs.readdir as jest.Mock).mockReturnValueOnce(['cookiecutter.json']); + + mock({ + [`${mockContext.workspacePath}/template`]: { 'cookiecutter.json': '{}' }, + }); + mock({ [`${mockContext.workspacePath}/result`]: {} }); + mock({ + [`${mockContext.workspacePath}/intermediate`]: {}, + }); + jest.restoreAllMocks(); + commandExists.mockRejectedValue(null); + }); + + afterEach(() => { + mock.restore(); + }); + + it('should call fetchContents with the correct values', async () => { + await action.handler(mockContext); + + expect(fetchContents).toHaveBeenCalledWith({ + reader: mockReader, + integrations, + baseUrl: mockContext.baseUrl, + fetchUrl: mockContext.input.url, + outputPath: resolvePath( + mockContext.workspacePath, + `template/{{cookiecutter and 'contents'}}`, + ), + }); + }); + + it('should execute the cookiecutter templater with optional inputs if they are present and valid', async () => { + await action.handler({ + ...mockContext, + input: { + ...mockContext.input, + copyWithoutRender: ['goreleaser.yml'], + extensions: [ + 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', + ], + imageName: 'foo/cookiecutter-image-with-extensions', + }, + }); + + expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ + workspacePath: mockTmpDir, + logStream: mockContext.logStream, + values: { + ...mockContext.input.values, + _copy_without_render: ['goreleaser.yml'], + _extensions: [ + 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', + ], + imageName: 'foo/cookiecutter-image-with-extensions', + }, + }); + }); + + // it('should throw if copyWithoutRender is not an Array', async () => { + // await expect( + // action.handler({ + // ...mockContext, + // input: { + // ...mockContext.input, + // copyWithoutRender: 'xyz', + // }, + // }), + // ).rejects.toThrow(/copyWithoutRender must be an Array/); + // }); + + // it('should throw if extensions is not an Array', async () => { + // await expect( + // action.handler({ + // ...mockContext, + // input: { + // ...mockContext.input, + // extensions: 'xyz', + // }, + // }), + // ).rejects.toThrow(/extensions must be an Array/); + // }); + + // it('should throw if there is no cookiecutter templater initialized', async () => { + // const templatersWithoutCookiecutter = new Templaters(); + + // const newAction = createFetchCookiecutterAction({ + // integrations, + // templaters: templatersWithoutCookiecutter, + // reader: mockReader, + // }); + + // await expect(newAction.handler(mockContext)).rejects.toThrow( + // /No templater registered/, + // ); + // }); + + // it('should throw if the target directory is outside of the workspace path', async () => { + // await expect( + // action.handler({ + // ...mockContext, + // input: { + // ...mockContext.input, + // targetPath: '/foo', + // }, + // }), + // ).rejects.toThrow( + // /Relative path is not allowed to refer to a directory outside its parent/, + // ); + // }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts index e5720204c7..2bf22ce4fd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.ts @@ -108,6 +108,7 @@ export class CookiecutterRunner { // if cookiecutter was successful, intermediateDir will contain // exactly one directory. + const [generated] = await fs.readdir(intermediateDir); if (generated === undefined) { From 2ed57f31c76a22358da80d5b5e9959d26d8e2036 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 6 Jul 2021 20:20:49 +0200 Subject: [PATCH 13/18] chore: write some tests for the cookiecutter templater Signed-off-by: blam --- .../builtin/fetch/cookiecutter.test.ts | 252 ++++++++++-------- 1 file changed, 138 insertions(+), 114 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 53f255a639..8bd205be1d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -16,24 +16,26 @@ const runCommand = jest.fn(); const commandExists = jest.fn(); const fetchContents = jest.fn(); -jest.mock('./helpers', () => ({ runCommand, fetchContents })); -jest.mock('command-exists', () => commandExists); -jest.mock('./helpers'); -jest.mock('fs-extra'); -import fs from 'fs-extra'; +jest.mock('./helpers', () => ({ fetchContents })); +jest.mock('command-exists', () => commandExists); +jest.mock('../helpers', () => ({ runCommand })); + import { getVoidLogger, UrlReader, ContainerRunner, } from '@backstage/backend-common'; -import { ConfigReader } from '@backstage/config'; +import { ConfigReader, JsonObject } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; import mock from 'mock-fs'; import os from 'os'; -import { resolve as resolvePath } from 'path'; import { PassThrough } from 'stream'; import { createFetchCookiecutterAction } from './cookiecutter'; +import { join } from 'path'; +import { ActionContext } from '../../types'; + +import fs from 'fs-extra'; describe('fetch:cookiecutter', () => { const integrations = ScmIntegrations.fromConfig( @@ -48,21 +50,15 @@ describe('fetch:cookiecutter', () => { ); const mockTmpDir = os.tmpdir(); - const mockContext = { - input: { - url: 'https://google.com/cookie/cutter', - targetPath: 'something', - values: { - help: 'me', - }, - }, - baseUrl: 'somebase', - workspacePath: mockTmpDir, - logger: getVoidLogger(), - logStream: new PassThrough(), - output: jest.fn(), - createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), - }; + + let mockContext: ActionContext<{ + url: string; + targetPath?: string; + values: JsonObject; + copyWithoutRender?: string[]; + extensions?: string[]; + imageName?: string; + }>; const containerRunner: jest.Mocked = { runContainer: jest.fn(), @@ -81,114 +77,142 @@ describe('fetch:cookiecutter', () => { }); beforeEach(() => { - (fs.readdir as jest.Mock).mockReturnValueOnce(['cookiecutter.json']); + jest.resetAllMocks(); - mock({ - [`${mockContext.workspacePath}/template`]: { 'cookiecutter.json': '{}' }, + mockContext = { + input: { + url: 'https://google.com/cookie/cutter', + targetPath: 'something', + values: { + help: 'me', + }, + }, + baseUrl: 'somebase', + workspacePath: mockTmpDir, + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir), + }; + + // mock the temp directory + mock({ [mockTmpDir]: {} }); + mock({ [`${join(mockTmpDir, 'template')}`]: {} }); + + commandExists.mockResolvedValue(null); + + // Mock when run container is called it creates some new files in the mock filesystem + containerRunner.runContainer.mockImplementation(async () => { + mock({ + [`${join(mockTmpDir, 'intermediate')}`]: { + 'testfile.json': '{}', + }, + }); }); - mock({ [`${mockContext.workspacePath}/result`]: {} }); - mock({ - [`${mockContext.workspacePath}/intermediate`]: {}, + + // Mock when runCommand is called it creats some new files in the mock filesystem + runCommand.mockImplementation(async () => { + mock({ + [`${join(mockTmpDir, 'intermediate')}`]: { + 'testfile.json': '{}', + }, + }); }); - jest.restoreAllMocks(); - commandExists.mockRejectedValue(null); }); afterEach(() => { mock.restore(); }); - it('should call fetchContents with the correct values', async () => { + it('should throw an error when copyWithoutRender is not an array', async () => { + (mockContext.input as any).copyWithoutRender = 'not an array'; + + await expect( + action.handler(mockContext), + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Fetch action input copyWithoutRender must be an Array"`, + ); + }); + + it('should throw an error when extensions is not an array', async () => { + (mockContext.input as any).extensions = 'not an array'; + + await expect( + action.handler(mockContext), + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Fetch action input extensions must be an Array"`, + ); + }); + + it('should call fetchContents with the correct variables', async () => { + fetchContents.mockImplementation(() => Promise.resolve()); + await action.handler(mockContext); + expect(fetchContents).toHaveBeenCalledWith( + expect.objectContaining({ + reader: mockReader, + integrations, + baseUrl: mockContext.baseUrl, + fetchUrl: mockContext.input.url, + outputPath: join( + mockTmpDir, + 'template', + "{{cookiecutter and 'contents'}}", + ), + }), + ); + }); + + it('should call out to cookiecutter using runCommand when cookiecutter is installed', async () => { + commandExists.mockResolvedValue(true); + await action.handler(mockContext); - expect(fetchContents).toHaveBeenCalledWith({ - reader: mockReader, - integrations, - baseUrl: mockContext.baseUrl, - fetchUrl: mockContext.input.url, - outputPath: resolvePath( - mockContext.workspacePath, - `template/{{cookiecutter and 'contents'}}`, - ), - }); + expect(runCommand).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'cookiecutter', + args: [ + '--no-input', + '-o', + join(mockTmpDir, 'intermediate'), + join(mockTmpDir, 'template'), + '--verbose', + ], + logStream: mockContext.logStream, + }), + ); }); - it('should execute the cookiecutter templater with optional inputs if they are present and valid', async () => { - await action.handler({ - ...mockContext, - input: { - ...mockContext.input, - copyWithoutRender: ['goreleaser.yml'], - extensions: [ - 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', - ], - imageName: 'foo/cookiecutter-image-with-extensions', - }, - }); + it('should call out to the containerRunner when there is no cookiecutter installed', async () => { + commandExists.mockResolvedValue(false); - expect(cookiecutterTemplater.run).toHaveBeenCalledWith({ - workspacePath: mockTmpDir, - logStream: mockContext.logStream, - values: { - ...mockContext.input.values, - _copy_without_render: ['goreleaser.yml'], - _extensions: [ - 'jinja2_custom_filters_extension.string_filters_extension.StringFilterExtension', - ], - imageName: 'foo/cookiecutter-image-with-extensions', - }, - }); + await action.handler(mockContext); + + expect(containerRunner.runContainer).toHaveBeenCalledWith( + expect.objectContaining({ + imageName: 'spotify/backstage-cookiecutter', + command: 'cookiecutter', + args: ['--no-input', '-o', '/output', '/input', '--verbose'], + mountDirs: { + [join(mockTmpDir, 'intermediate')]: '/output', + [join(mockTmpDir, 'template')]: '/input', + }, + workingDir: '/input', + envVars: { HOME: '/tmp' }, + logStream: mockContext.logStream, + }), + ); }); - // it('should throw if copyWithoutRender is not an Array', async () => { - // await expect( - // action.handler({ - // ...mockContext, - // input: { - // ...mockContext.input, - // copyWithoutRender: 'xyz', - // }, - // }), - // ).rejects.toThrow(/copyWithoutRender must be an Array/); - // }); + it('should use a custom imageName when there is an image supplied to the context', async () => { + const imageName = 'test-image'; + mockContext.input.imageName = imageName; - // it('should throw if extensions is not an Array', async () => { - // await expect( - // action.handler({ - // ...mockContext, - // input: { - // ...mockContext.input, - // extensions: 'xyz', - // }, - // }), - // ).rejects.toThrow(/extensions must be an Array/); - // }); + await action.handler(mockContext); - // it('should throw if there is no cookiecutter templater initialized', async () => { - // const templatersWithoutCookiecutter = new Templaters(); - - // const newAction = createFetchCookiecutterAction({ - // integrations, - // templaters: templatersWithoutCookiecutter, - // reader: mockReader, - // }); - - // await expect(newAction.handler(mockContext)).rejects.toThrow( - // /No templater registered/, - // ); - // }); - - // it('should throw if the target directory is outside of the workspace path', async () => { - // await expect( - // action.handler({ - // ...mockContext, - // input: { - // ...mockContext.input, - // targetPath: '/foo', - // }, - // }), - // ).rejects.toThrow( - // /Relative path is not allowed to refer to a directory outside its parent/, - // ); - // }); + expect(containerRunner.runContainer).toHaveBeenCalledWith( + expect.objectContaining({ + imageName, + }), + ); + }); }); From fd5ec6d304d3f4f6bc0846a8ae8e6e6866402786 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 6 Jul 2021 20:29:58 +0200 Subject: [PATCH 14/18] chore: some api-docs changed Signed-off-by: blam --- .../src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts | 2 -- plugins/sentry/api-report.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 8bd205be1d..1ef1e1a940 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -35,8 +35,6 @@ import { createFetchCookiecutterAction } from './cookiecutter'; import { join } from 'path'; import { ActionContext } from '../../types'; -import fs from 'fs-extra'; - describe('fetch:cookiecutter', () => { const integrations = ScmIntegrations.fromConfig( new ConfigReader({ diff --git a/plugins/sentry/api-report.md b/plugins/sentry/api-report.md index c3cce9ed7c..f140fc8b60 100644 --- a/plugins/sentry/api-report.md +++ b/plugins/sentry/api-report.md @@ -81,7 +81,7 @@ export type SentryIssue = { // @public (undocumented) export const SentryIssuesWidget: ({ entity, statsFor, variant, }: { entity: Entity; - statsFor?: "12h" | "24h" | undefined; + statsFor?: "24h" | "12h" | undefined; variant?: InfoCardVariants | undefined; }) => JSX.Element; From 4d9264f0fcacff87095e9cb4b81ccb719c5580dd Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 6 Jul 2021 20:37:39 +0200 Subject: [PATCH 15/18] chore: fixing api-reports again Signed-off-by: blam --- plugins/sentry/api-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/sentry/api-report.md b/plugins/sentry/api-report.md index f140fc8b60..c3cce9ed7c 100644 --- a/plugins/sentry/api-report.md +++ b/plugins/sentry/api-report.md @@ -81,7 +81,7 @@ export type SentryIssue = { // @public (undocumented) export const SentryIssuesWidget: ({ entity, statsFor, variant, }: { entity: Entity; - statsFor?: "24h" | "12h" | undefined; + statsFor?: "12h" | "24h" | undefined; variant?: InfoCardVariants | undefined; }) => JSX.Element; From 8e183acc9cee145f665b4c0c9d2ca997e2171b50 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 7 Jul 2021 11:02:31 +0200 Subject: [PATCH 16/18] chore: code review comments Signed-off-by: blam --- .../builtin/fetch/cookiecutter.test.ts | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts index 1ef1e1a940..86f953a6a5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/cookiecutter.test.ts @@ -28,7 +28,7 @@ import { } from '@backstage/backend-common'; import { ConfigReader, JsonObject } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; -import mock from 'mock-fs'; +import mockFs from 'mock-fs'; import os from 'os'; import { PassThrough } from 'stream'; import { createFetchCookiecutterAction } from './cookiecutter'; @@ -94,14 +94,14 @@ describe('fetch:cookiecutter', () => { }; // mock the temp directory - mock({ [mockTmpDir]: {} }); - mock({ [`${join(mockTmpDir, 'template')}`]: {} }); + mockFs({ [mockTmpDir]: {} }); + mockFs({ [`${join(mockTmpDir, 'template')}`]: {} }); commandExists.mockResolvedValue(null); // Mock when run container is called it creates some new files in the mock filesystem containerRunner.runContainer.mockImplementation(async () => { - mock({ + mockFs({ [`${join(mockTmpDir, 'intermediate')}`]: { 'testfile.json': '{}', }, @@ -110,7 +110,7 @@ describe('fetch:cookiecutter', () => { // Mock when runCommand is called it creats some new files in the mock filesystem runCommand.mockImplementation(async () => { - mock({ + mockFs({ [`${join(mockTmpDir, 'intermediate')}`]: { 'testfile.json': '{}', }, @@ -119,26 +119,22 @@ describe('fetch:cookiecutter', () => { }); afterEach(() => { - mock.restore(); + mockFs.restore(); }); it('should throw an error when copyWithoutRender is not an array', async () => { (mockContext.input as any).copyWithoutRender = 'not an array'; - await expect( - action.handler(mockContext), - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Fetch action input copyWithoutRender must be an Array"`, + await expect(action.handler(mockContext)).rejects.toThrowError( + /Fetch action input copyWithoutRender must be an Array/, ); }); it('should throw an error when extensions is not an array', async () => { (mockContext.input as any).extensions = 'not an array'; - await expect( - action.handler(mockContext), - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Fetch action input extensions must be an Array"`, + await expect(action.handler(mockContext)).rejects.toThrowError( + /Fetch action input extensions must be an Array/, ); }); From 60e8302222987ae3832efbcb777cba105bb6934c Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 7 Jul 2021 11:21:46 +0200 Subject: [PATCH 17/18] chore: made a lovely changeset Signed-off-by: blam --- .changeset/fresh-elephants-tease.md | 68 +++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .changeset/fresh-elephants-tease.md diff --git a/.changeset/fresh-elephants-tease.md b/.changeset/fresh-elephants-tease.md new file mode 100644 index 0000000000..5141f96be7 --- /dev/null +++ b/.changeset/fresh-elephants-tease.md @@ -0,0 +1,68 @@ +--- +'@backstage/catalog-model': minor +'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-scaffolder': minor +'@backstage/plugin-scaffolder-backend': minor +'@backstage/create-app': patch +--- + +Support for `Template` kinds with version `backstage.io/v1alpha1` has now been removed. This means that the old method of running templates with `Preparers`, `Templaters` and `Publishers` has also been removed. If you had any logic in these abstractions, they should now be moved to `actions` instead, and you can find out more about those in the [documentation](https://backstage.io/docs/features/software-templates/writing-custom-actions) + +If you need any help migrating existing templates, there's a [migration guide](https://backstage.io/docs/features/software-templates/migrating-from-v1alpha1-to-v1beta2). Reach out to us on Discord in the #support channel if you're having problems. + +The `scaffolder-backend` now no longer requires these `Preparers`, `Templaters`, and `Publishers` to be passed in, now all it needs is the `containerRunner`. + +Please update your `packages/backend/src/plugins/scaffolder.ts` like the following + +```diff +- import { +- DockerContainerRunner, +- SingleHostDiscovery, +- } from '@backstage/backend-common'; ++ import { DockerContainerRunner } from '@backstage/backend-common'; + import { CatalogClient } from '@backstage/catalog-client'; +- import { +- CookieCutter, +- CreateReactAppTemplater, +- createRouter, +- Preparers, +- Publishers, +- Templaters, +- } from '@backstage/plugin-scaffolder-backend'; ++ import { createRouter } from '@backstage/plugin-scaffolder-backend'; + import Docker from 'dockerode'; + import { Router } from 'express'; + import type { PluginEnvironment } from '../types'; + + export default async function createPlugin({ + config, + database, + reader, ++ discovery, + }: PluginEnvironment): Promise { + const dockerClient = new Docker(); + const containerRunner = new DockerContainerRunner({ dockerClient }); + +- const cookiecutterTemplater = new CookieCutter({ containerRunner }); +- const craTemplater = new CreateReactAppTemplater({ containerRunner }); +- const templaters = new Templaters(); + +- templaters.register('cookiecutter', cookiecutterTemplater); +- templaters.register('cra', craTemplater); +- +- const preparers = await Preparers.fromConfig(config, { logger }); +- const publishers = await Publishers.fromConfig(config, { logger }); + +- const discovery = SingleHostDiscovery.fromConfig(config); + const catalogClient = new CatalogClient({ discoveryApi: discovery }); + + return await createRouter({ +- preparers, +- templaters, +- publishers, ++ containerRunner, + logger, + config, + database, + +``` From e922a9952830474e1431330bf46d0a1c601df93d Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 7 Jul 2021 11:27:55 +0200 Subject: [PATCH 18/18] chore: updating api-docs Signed-off-by: blam --- plugins/scaffolder-backend/api-report.md | 377 +---------------------- 1 file changed, 5 insertions(+), 372 deletions(-) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 21c6e03b0f..ab3ea433c6 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -4,18 +4,11 @@ ```ts -import { AzureIntegrationConfig } from '@backstage/integration'; -import { BitbucketIntegrationConfig } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; import { ContainerRunner } from '@backstage/backend-common'; import { createPullRequest } from 'octokit-plugin-create-pull-request'; import express from 'express'; -import { GithubCredentialsProvider } from '@backstage/integration'; -import { GitHubIntegrationConfig } from '@backstage/integration'; -import { Gitlab } from '@gitbeaker/core'; -import { GitLabIntegrationConfig } from '@backstage/integration'; -import gitUrlParse from 'git-url-parse'; import { JsonObject } from '@backstage/config'; import { JsonValue } from '@backstage/config'; import { Logger } from 'winston'; @@ -23,7 +16,6 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { Schema } from 'jsonschema'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { ScmIntegrations } from '@backstage/integration'; -import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; import { Writable } from 'stream'; @@ -40,74 +32,12 @@ export type ActionContext = { createTemporaryDirectory(): Promise; }; -// @public (undocumented) -export class AzurePreparer implements PreparerBase { - constructor(config: { - token?: string; - }); - // (undocumented) - static fromConfig(config: AzureIntegrationConfig): AzurePreparer; - // (undocumented) - prepare({ url, workspacePath, logger }: PreparerOptions): Promise; -} - -// @public (undocumented) -export class AzurePublisher implements PublisherBase { - constructor(config: { - token: string; - }); - // (undocumented) - static fromConfig(config: AzureIntegrationConfig): Promise; - // (undocumented) - publish({ values, workspacePath, logger, }: PublisherOptions): Promise; -} - -// @public (undocumented) -export class BitbucketPreparer implements PreparerBase { - constructor(config: { - username?: string; - token?: string; - appPassword?: string; - }); - // (undocumented) - static fromConfig(config: BitbucketIntegrationConfig): BitbucketPreparer; - // (undocumented) - prepare({ url, workspacePath, logger }: PreparerOptions): Promise; -} - -// @public (undocumented) -export class BitbucketPublisher implements PublisherBase { - constructor(config: { - host: string; - token?: string; - appPassword?: string; - username?: string; - apiBaseUrl?: string; - repoVisibility: RepoVisibilityOptions_2; - }); - // (undocumented) - static fromConfig(config: BitbucketIntegrationConfig, { repoVisibility }: { - repoVisibility: RepoVisibilityOptions_2; - }): Promise; - // (undocumented) - publish({ values, workspacePath, logger, }: PublisherOptions): Promise; -} - // @public export class CatalogEntityClient { constructor(catalogClient: CatalogApi); findTemplate(templateName: string, options?: { token?: string; - }): Promise; -} - -// @public (undocumented) -export class CookieCutter implements TemplaterBase { - constructor({ containerRunner }: { - containerRunner: ContainerRunner; - }); - // (undocumented) - run({ workspacePath, values, logStream, }: TemplaterRunOptions): Promise; + }): Promise; } // @public (undocumented) @@ -115,7 +45,7 @@ export const createBuiltinActions: (options: { reader: UrlReader; integrations: ScmIntegrations; catalogClient: CatalogApi; - templaters: TemplaterBuilder; + containerRunner: ContainerRunner; config: Config; }) => TemplateAction[]; @@ -135,7 +65,7 @@ export function createDebugLogAction(): TemplateAction; export function createFetchCookiecutterAction(options: { reader: UrlReader; integrations: ScmIntegrations; - templaters: TemplaterBuilder; + containerRunner: ContainerRunner; }): TemplateAction; // @public (undocumented) @@ -150,9 +80,6 @@ export const createFilesystemDeleteAction: () => TemplateAction; // @public (undocumented) export const createFilesystemRenameAction: () => TemplateAction; -// @public (undocumented) -export function createLegacyActions(options: Options): TemplateAction[]; - // @public (undocumented) export function createPublishAzureAction(options: { integrations: ScmIntegrationRegistry; @@ -183,15 +110,6 @@ export function createPublishGitlabAction(options: { config: Config; }): TemplateAction; -// @public (undocumented) -export class CreateReactAppTemplater implements TemplaterBase { - constructor({ containerRunner }: { - containerRunner: ContainerRunner; - }); - // (undocumented) - run({ workspacePath, values, logStream, }: TemplaterRunOptions): Promise; -} - // @public (undocumented) export function createRouter(options: RouterOptions): Promise; @@ -200,206 +118,6 @@ export const createTemplateAction: | undefined; }>>(templateAction: TemplateAction) => TemplateAction; -// @public (undocumented) -export class FilePreparer implements PreparerBase { - // (undocumented) - prepare({ url, workspacePath }: PreparerOptions): Promise; -} - -// @public -export const getTemplaterKey: (entity: TemplateEntityV1alpha1) => string; - -// @public (undocumented) -export class GithubPreparer implements PreparerBase { - constructor(config: { - credentialsProvider: GithubCredentialsProvider; - }); - // (undocumented) - static fromConfig(config: GitHubIntegrationConfig): GithubPreparer; - // (undocumented) - prepare({ url, workspacePath, logger }: PreparerOptions): Promise; -} - -// @public @deprecated (undocumented) -export class GithubPublisher implements PublisherBase { - constructor(config: { - credentialsProvider: GithubCredentialsProvider; - repoVisibility: RepoVisibilityOptions; - apiBaseUrl: string | undefined; - }); - // (undocumented) - static fromConfig(config: GitHubIntegrationConfig, { repoVisibility }: { - repoVisibility: RepoVisibilityOptions; - }): Promise; - // (undocumented) - publish({ values, workspacePath, logger, }: PublisherOptions): Promise; -} - -// @public (undocumented) -export class GitlabPreparer implements PreparerBase { - constructor(config: { - token?: string; - }); - // (undocumented) - static fromConfig(config: GitLabIntegrationConfig): GitlabPreparer; - // (undocumented) - prepare({ url, workspacePath, logger }: PreparerOptions): Promise; -} - -// @public (undocumented) -export class GitlabPublisher implements PublisherBase { - constructor(config: { - token: string; - client: Gitlab; - repoVisibility: RepoVisibilityOptions_3; - }); - // (undocumented) - static fromConfig(config: GitLabIntegrationConfig, { repoVisibility }: { - repoVisibility: RepoVisibilityOptions_3; - }): Promise; - // (undocumented) - publish({ values, workspacePath, logger, }: PublisherOptions): Promise; -} - -// @public (undocumented) -export type Job = { - id: string; - context: StageContext; - status: ProcessorStatus; - stages: StageResult[]; - error?: Error; -}; - -// @public (undocumented) -export type JobAndDirectoryTuple = { - job: Job; - directory: string; -}; - -// @public (undocumented) -export class JobProcessor implements Processor { - constructor(workingDirectory: string); - // (undocumented) - create({ entity, values, stages, }: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - stages: StageInput[]; - }): Job; - // (undocumented) - static fromConfig({ config, logger, }: { - config: Config; - logger: Logger; - }): Promise; - // (undocumented) - get(id: string): Job | undefined; - // (undocumented) - run(job: Job): Promise; - } - -// @public (undocumented) -export function joinGitUrlPath(repoUrl: string, path?: string): string; - -// @public (undocumented) -export type ParsedLocationAnnotation = { - protocol: 'file' | 'url'; - location: string; -}; - -// @public (undocumented) -export const parseLocationAnnotation: (entity: TemplateEntityV1alpha1) => ParsedLocationAnnotation; - -// @public (undocumented) -export interface PreparerBase { - prepare(opts: PreparerOptions): Promise; -} - -// @public (undocumented) -export type PreparerBuilder = { - register(host: string, preparer: PreparerBase): void; - get(url: string): PreparerBase; -}; - -// @public (undocumented) -export type PreparerOptions = { - url: string; - workspacePath: string; - logger: Logger; -}; - -// @public (undocumented) -export class Preparers implements PreparerBuilder { - // (undocumented) - static fromConfig(config: Config, _: { - logger: Logger; - }): Promise; - // (undocumented) - get(url: string): PreparerBase; - // (undocumented) - register(host: string, preparer: PreparerBase): void; -} - -// @public (undocumented) -export type Processor = { - create({ entity, values, stages, }: { - entity: TemplateEntityV1alpha1; - values: TemplaterValues; - stages: StageInput[]; - }): Job; - get(id: string): Job | undefined; - run(job: Job): Promise; -}; - -// @public (undocumented) -export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; - -// @public -export type PublisherBase = { - publish(opts: PublisherOptions): Promise; -}; - -// @public (undocumented) -export type PublisherBuilder = { - register(host: string, publisher: PublisherBase): void; - get(storePath: string): PublisherBase; -}; - -// @public (undocumented) -export type PublisherOptions = { - values: TemplaterValues; - workspacePath: string; - logger: Logger; -}; - -// @public (undocumented) -export type PublisherResult = { - remoteUrl: string; - catalogInfoUrl?: string; -}; - -// @public (undocumented) -export class Publishers implements PublisherBuilder { - // (undocumented) - static fromConfig(config: Config, _options: { - logger: Logger; - }): Promise; - // (undocumented) - get(url: string): PublisherBase; - // (undocumented) - register(host: string, preparer: PublisherBase | undefined): void; -} - -// @public (undocumented) -export type RepoVisibilityOptions = 'private' | 'internal' | 'public'; - -// @public -export type RequiredTemplateValues = { - owner: string; - storePath: string; - destination?: { - git?: gitUrlParse.GitUrl; - }; -}; - // @public (undocumented) export interface RouterOptions { // (undocumented) @@ -409,63 +127,17 @@ export interface RouterOptions { // (undocumented) config: Config; // (undocumented) + containerRunner: ContainerRunner; + // (undocumented) database: PluginDatabaseManager; // (undocumented) logger: Logger; // (undocumented) - preparers: PreparerBuilder; - // (undocumented) - publishers: PublisherBuilder; - // (undocumented) reader: UrlReader; // (undocumented) taskWorkers?: number; - // (undocumented) - templaters: TemplaterBuilder; } -// @public (undocumented) -export const runCommand: ({ command, args, logStream, }: RunCommandOptions) => Promise; - -// @public (undocumented) -export type RunCommandOptions = { - command: string; - args: string[]; - logStream?: Writable; -}; - -// @public (undocumented) -export type StageContext = { - values: TemplaterValues; - entity: TemplateEntityV1alpha1; - logger: Logger; - logStream: Writable; - workspacePath: string; -} & T; - -// @public (undocumented) -export interface StageInput { - // (undocumented) - handler(ctx: StageContext): Promise; - // (undocumented) - name: string; -} - -// @public (undocumented) -export interface StageResult extends StageInput { - // (undocumented) - endedAt?: number; - // (undocumented) - log: string[]; - // (undocumented) - startedAt?: number; - // (undocumented) - status: ProcessorStatus; -} - -// @public -export type SupportedTemplatingKey = 'cookiecutter' | string; - // @public (undocumented) export type TemplateAction = { id: string; @@ -487,45 +159,6 @@ export class TemplateActionRegistry { register(action: TemplateAction): void; } -// @public (undocumented) -export type TemplaterBase = { - run(opts: TemplaterRunOptions): Promise; -}; - -// @public -export type TemplaterBuilder = { - register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void; - get(templater: string): TemplaterBase; -}; - -// @public (undocumented) -export type TemplaterConfig = { - templater?: TemplaterBase; -}; - -// @public -export type TemplaterRunOptions = { - workspacePath: string; - values: TemplaterValues; - logStream?: Writable; -}; - -// @public -export type TemplaterRunResult = { - resultDir: string; -}; - -// @public (undocumented) -export class Templaters implements TemplaterBuilder { - // (undocumented) - get(templaterId: string): TemplaterBase; - // (undocumented) - register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase): void; - } - -// @public (undocumented) -export type TemplaterValues = RequiredTemplateValues & Record; - // (No @packageDocumentation comment for this package)