diff --git a/.changeset/bitbucket-scaffolder.md b/.changeset/bitbucket-scaffolder.md new file mode 100644 index 0000000000..e1e6f17b29 --- /dev/null +++ b/.changeset/bitbucket-scaffolder.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Add scaffolding support for Bitbucket Cloud and Server. diff --git a/app-config.yaml b/app-config.yaml index f6f71478d0..66e69462aa 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -206,6 +206,13 @@ scaffolder: api: token: $env: AZURE_TOKEN + bitbucket: + api: + host: https://bitbucket.org + username: + $env: BITBUCKET_USERNAME + token: + $env: BITBUCKET_TOKEN auth: environment: development ### Providing an auth.session.secret will enable session support in the auth-backend diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 1a443ceb7c..c290a94b19 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -59,12 +59,14 @@ }, "devDependencies": { "@backstage/cli": "^0.4.3", + "@backstage/test-utils": "^0.1.5", "@octokit/types": "^5.4.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", "@types/supertest": "^2.0.8", "supertest": "^4.0.2", - "yaml": "^1.10.0" + "yaml": "^1.10.0", + "msw": "^0.21.2" }, "files": [ "dist", diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts index e54e796b3b..21b63609e8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts @@ -85,7 +85,9 @@ export function makeDeprecatedLocationTypeDetector( config.getOptionalConfigArray('integrations.azure')?.forEach(sub => { hostMap.set(sub.getString('host'), 'azure/api'); }); - + config.getOptionalConfigArray('integrations.bitbucket')?.forEach(sub => { + hostMap.set(sub.getString('host'), 'bitbucket'); + }); return (url: string): string | undefined => { const parsed = new URL(url); return hostMap.get(parsed.hostname); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts new file mode 100644 index 0000000000..809d18c808 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts @@ -0,0 +1,145 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.doMock('fs-extra', () => ({ + promises: { + mkdtemp: jest.fn(dir => `${dir}-static`), + }, +})); + +import { BitbucketPreparer } from './bitbucket'; +import { + TemplateEntityV1alpha1, + LOCATION_ANNOTATION, +} from '@backstage/catalog-model'; +import { getVoidLogger, Git } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; + +describe('BitbucketPreparer', () => { + let mockEntity: TemplateEntityV1alpha1; + const mockGitClient = { + clone: jest.fn(), + }; + + jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any); + + beforeEach(() => { + jest.clearAllMocks(); + mockEntity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Template', + metadata: { + annotations: { + [LOCATION_ANNOTATION]: + 'bitbucket:https://bitbucket.org/backstage-project/backstage-repo', + }, + name: 'graphql-starter', + title: 'GraphQL Service', + description: + 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', + uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', + etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', + generation: 1, + }, + spec: { + type: 'website', + templater: 'cookiecutter', + path: './template', + schema: { + $schema: 'http://json-schema.org/draft-07/schema#', + required: ['storePath', 'owner'], + properties: { + owner: { + type: 'string', + title: 'Owner', + description: 'Who is going to own this component', + }, + storePath: { + type: 'string', + title: 'Store path', + description: 'GitHub store path in org/repo format', + }, + }, + }, + }, + }; + }); + + it('calls the clone command with the correct arguments for a repository', async () => { + const preparer = new BitbucketPreparer(new ConfigReader({})); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://bitbucket.org/backstage-project/backstage-repo', + dir: expect.any(String), + }); + }); + + it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => { + const preparer = new BitbucketPreparer( + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: 'fake-user', + appPassword: 'fake-password', + }, + ], + }, + }), + ); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://bitbucket.org/backstage-project/backstage-repo', + dir: expect.any(String), + }); + }); + + it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { + const preparer = new BitbucketPreparer(new ConfigReader({})); + delete mockEntity.spec.path; + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); + expect(mockGitClient.clone).toHaveBeenCalledWith({ + url: 'https://bitbucket.org/backstage-project/backstage-repo', + dir: expect.any(String), + }); + }); + + it('return the temp directory with the path to the folder if it is specified', async () => { + const preparer = new BitbucketPreparer(new ConfigReader({})); + mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + }); + + expect(response.split('\\').join('/')).toMatch( + /\/template\/test\/1\/2\/3$/, + ); + }); + + it('return the working directory with the path to the folder if it is specified', async () => { + const preparer = new BitbucketPreparer(new ConfigReader({})); + mockEntity.spec.path = './template/test/1/2/3'; + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + workingDirectory: '/workDir', + }); + + expect(response.split('\\').join('/')).toMatch( + /\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/, + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts new file mode 100644 index 0000000000..5b244d24ce --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import os from 'os'; +import fs from 'fs-extra'; +import path from 'path'; +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { parseLocationAnnotation } from '../helpers'; +import { InputError, Git } from '@backstage/backend-common'; +import { PreparerBase, PreparerOptions } from './types'; +import GitUriParser from 'git-url-parse'; +import { Config } from '@backstage/config'; + +export class BitbucketPreparer implements PreparerBase { + private readonly privateToken: string; + private readonly username: string; + + constructor(config: Config) { + this.username = + config.getOptionalString('scaffolder.bitbucket.api.username') ?? ''; + this.privateToken = + config.getOptionalString('scaffolder.bitbucket.api.token') ?? ''; + } + + async prepare( + template: TemplateEntityV1alpha1, + opts: PreparerOptions, + ): Promise { + const { protocol, location } = parseLocationAnnotation(template); + const workingDirectory = opts?.workingDirectory ?? os.tmpdir(); + const { logger } = opts; + + if (!['bitbucket', 'url'].includes(protocol)) { + throw new InputError( + `Wrong location protocol: ${protocol}, should be 'url'`, + ); + } + const templateId = template.metadata.name; + + const repo = GitUriParser(location); + const repositoryCheckoutUrl = repo.toString('https'); + + const tempDir = await fs.promises.mkdtemp( + path.join(workingDirectory, templateId), + ); + + const templateDirectory = path.join( + `${path.dirname(repo.filepath)}`, + template.spec.path ?? '.', + ); + + const checkoutLocation = path.resolve(tempDir, templateDirectory); + + const git = this.privateToken + ? Git.fromAuth({ + username: this.username, + password: this.privateToken, + logger, + }) + : Git.fromAuth({ logger }); + + await git.clone({ + url: repositoryCheckoutUrl, + dir: tempDir, + }); + + return checkoutLocation; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts index 84be130b79..906fc0f9fd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts @@ -28,6 +28,7 @@ import { FilePreparer } from './file'; import { GitlabPreparer } from './gitlab'; import { AzurePreparer } from './azure'; import { GithubPreparer } from './github'; +import { BitbucketPreparer } from './bitbucket'; export class Preparers implements PreparerBuilder { private preparerMap = new Map(); @@ -79,11 +80,13 @@ export class Preparers implements PreparerBuilder { const filePreparer = new FilePreparer(); const gitlabPreparer = new GitlabPreparer(config); const azurePreparer = new AzurePreparer(config); + const bitbucketPreparer = new BitbucketPreparer(config); preparers.register('file', filePreparer); preparers.register('gitlab', gitlabPreparer); preparers.register('gitlab/api', gitlabPreparer); preparers.register('azure/api', azurePreparer); + preparers.register('bitbucket', bitbucketPreparer); const githubConfig = config.getOptionalConfig('scaffolder.github'); if (githubConfig) { diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts new file mode 100644 index 0000000000..3b084f1bf9 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts @@ -0,0 +1,148 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('./helpers'); + +import { BitbucketPublisher } from './bitbucket'; +import { initRepoAndPush } from './helpers'; +import { getVoidLogger } from '@backstage/backend-common'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; + +describe('Bitbucket Publisher', () => { + const logger = getVoidLogger(); + const server = setupServer(); + msw.setupDefaultHandlers(server); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('publish: createRemoteInBitbucketCloud', () => { + it('should create repo in bitbucket cloud', async () => { + server.use( + rest.post( + 'https://api.bitbucket.org/2.0/repositories/project/repo', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + html: { + href: 'https://bitbucket.org/project/repo', + }, + clone: [ + { + name: 'https', + href: 'https://bitbucket.org/project/repo', + }, + ], + }, + }), + ), + ), + ); + + const publisher = new BitbucketPublisher( + 'https://bitbucket.org', + 'fake-user', + 'fake-token', + ); + + const result = await publisher.publish({ + values: { + storePath: 'project/repo', + owner: 'bob', + }, + directory: '/tmp/test', + logger: logger, + }); + + expect(result).toEqual({ + remoteUrl: 'https://bitbucket.org/project/repo', + catalogInfoUrl: + 'https://bitbucket.org/project/repo/src/master/catalog-info.yaml', + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://bitbucket.org/project/repo', + auth: { username: 'fake-user', password: 'fake-token' }, + logger: logger, + }); + }); + }); + describe('publish: createRemoteInBitbucketServer', () => { + it('should create repo in bitbucket server', async () => { + server.use( + rest.post( + 'https://bitbucket.mycompany.com/rest/api/1.0/projects/project/repos', + (_, res, ctx) => + res( + ctx.status(201), + ctx.set('Content-Type', 'application/json'), + ctx.json({ + links: { + self: [ + { + href: + 'https://bitbucket.mycompany.com/projects/project/repos/repo', + }, + ], + clone: [ + { + name: 'http', + href: 'https://bitbucket.mycompany.com/scm/project/repo', + }, + ], + }, + }), + ), + ), + ); + + const publisher = new BitbucketPublisher( + 'https://bitbucket.mycompany.com', + 'fake-user', + 'fake-token', + ); + + const result = await publisher.publish({ + values: { + storePath: 'project/repo', + owner: 'bob', + }, + directory: '/tmp/test', + logger: logger, + }); + + expect(result).toEqual({ + remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo', + catalogInfoUrl: + 'https://bitbucket.mycompany.com/projects/project/repos/repo/catalog-info.yaml', + }); + + expect(initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo', + auth: { username: 'fake-user', password: 'fake-token' }, + logger: logger, + }); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts new file mode 100644 index 0000000000..ad83ad54e8 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts @@ -0,0 +1,143 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; +import { initRepoAndPush } from './helpers'; +import { RequiredTemplateValues } from '../templater'; +import { JsonValue } from '../../../../../../packages/config/src'; +import fetch from 'cross-fetch'; + +export class BitbucketPublisher implements PublisherBase { + private readonly host: string; + private readonly username: string; + private readonly token: string; + + constructor(host: string, username: string, token: string) { + this.host = host; + this.username = username; + this.token = token; + } + + async publish({ + values, + directory, + logger, + }: PublisherOptions): Promise { + const result = await this.createRemote(values); + + await initRepoAndPush({ + dir: directory, + remoteUrl: result.remoteUrl, + auth: { + username: this.username, + password: this.token, + }, + logger, + }); + return result; + } + + private async createRemote( + values: RequiredTemplateValues & Record, + ): Promise { + if (this.host === 'https://bitbucket.org') { + return this.createBitbucketCloudRepository(values); + } + return this.createBitbucketServerRepository(values); + } + + private async createBitbucketCloudRepository( + values: RequiredTemplateValues & Record, + ): Promise { + const [project, name] = values.storePath.split('/'); + + let response: Response; + const buffer = Buffer.from(`${this.username}:${this.token}`, 'utf8'); + + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + scm: 'git', + description: values.description, + }), + headers: { + Authorization: `Basic ${buffer.toString('base64')}`, + 'Content-Type': 'application/json', + }, + }; + try { + response = await fetch( + `https://api.bitbucket.org/2.0/repositories/${project}/${name}`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + if (response.status === 200) { + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'https') { + remoteUrl = link.href; + } + } + + // TODO use the urlReader to get the defautl branch + const catalogInfoUrl = `${r.links.html.href}/src/master/catalog-info.yaml`; + return { remoteUrl, catalogInfoUrl }; + } + throw new Error(`Not a valid response code ${await response.text()}`); + } + + private async createBitbucketServerRepository( + values: RequiredTemplateValues & Record, + ): Promise { + const [project, name] = values.storePath.split('/'); + + let response: Response; + const options: RequestInit = { + method: 'POST', + body: JSON.stringify({ + name: name, + description: values.description, + }), + headers: { + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json', + }, + }; + try { + response = await fetch( + `${this.host}/rest/api/1.0/projects/${project}/repos`, + options, + ); + } catch (e) { + throw new Error(`Unable to create repository, ${e}`); + } + if (response.status === 201) { + const r = await response.json(); + let remoteUrl = ''; + for (const link of r.links.clone) { + if (link.name === 'http') { + remoteUrl = link.href; + } + } + const catalogInfoUrl = `${r.links.self[0].href}/catalog-info.yaml`; + return { remoteUrl, catalogInfoUrl }; + } + throw new Error(`Not a valid response code ${await response.text()}`); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts index 03a4e884ba..75243aa610 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts @@ -30,6 +30,7 @@ import { RemoteProtocol } from '../types'; import { GithubPublisher, RepoVisibilityOptions } from './github'; import { GitlabPublisher } from './gitlab'; import { AzurePublisher } from './azure'; +import { BitbucketPublisher } from './bitbucket'; export class Publishers implements PublisherBuilder { private publisherMap = new Map(); @@ -163,6 +164,34 @@ export class Publishers implements PublisherBuilder { } } + const bitbucketConfig = config.getOptionalConfig( + 'scaffolder.bitbucket.api', + ); + if (bitbucketConfig) { + try { + const baseUrl = bitbucketConfig.getString('host'); + const bitbucketUsername = bitbucketConfig.getString('username'); + const bitbucketToken = bitbucketConfig.getString('token'); + + const bitbucketPublisher = new BitbucketPublisher( + baseUrl, + bitbucketUsername, + bitbucketToken, + ); + publishers.register('bitbucket', bitbucketPublisher); + } catch (e) { + const providerName = 'bitbucket'; + if (process.env.NODE_ENV !== 'development') { + throw new Error( + `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, + ); + } + + logger.warn( + `Skipping ${providerName} scaffolding provider, ${e.message}`, + ); + } + } return publishers; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/types.ts index 938d906857..56111337fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/types.ts @@ -18,4 +18,5 @@ export type RemoteProtocol = | 'github' | 'gitlab' | 'gitlab/api' - | 'azure/api'; + | 'azure/api' + | 'bitbucket';