Merge pull request #3669 from goober/feature/bitbucket-scaffolding

Add initial support for Bitbucket Server scaffolding

.changeset/bitbucket-scaffolder.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': patch
+---
+
+Add scaffolding support for Bitbucket Cloud and Server.

app-config.yaml

@@ -206,6 +206,13 @@ scaffolder:
     api:
       token:
         $env: AZURE_TOKEN
+  bitbucket:
+    api:
+      host: https://bitbucket.org
+      username:
+        $env: BITBUCKET_USERNAME
+      token:
+        $env: BITBUCKET_TOKEN
 auth:
   environment: development
   ### Providing an auth.session.secret will enable session support in the auth-backend

plugins/scaffolder-backend/package.json

@@ -59,12 +59,14 @@
   },
   "devDependencies": {
     "@backstage/cli": "^0.4.3",
+    "@backstage/test-utils": "^0.1.5",
     "@octokit/types": "^5.4.1",
     "@types/fs-extra": "^9.0.1",
     "@types/git-url-parse": "^9.0.0",
     "@types/supertest": "^2.0.8",
     "supertest": "^4.0.2",
-    "yaml": "^1.10.0"
+    "yaml": "^1.10.0",
+    "msw": "^0.21.2"
   },
   "files": [
     "dist",

plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts

@@ -85,7 +85,9 @@ export function makeDeprecatedLocationTypeDetector(
   config.getOptionalConfigArray('integrations.azure')?.forEach(sub => {
     hostMap.set(sub.getString('host'), 'azure/api');
   });
-
+  config.getOptionalConfigArray('integrations.bitbucket')?.forEach(sub => {
+    hostMap.set(sub.getString('host'), 'bitbucket');
+  });
   return (url: string): string | undefined => {
     const parsed = new URL(url);
     return hostMap.get(parsed.hostname);

plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.doMock('fs-extra', () => ({
+  promises: {
+    mkdtemp: jest.fn(dir => `${dir}-static`),
+  },
+}));
+
+import { BitbucketPreparer } from './bitbucket';
+import {
+  TemplateEntityV1alpha1,
+  LOCATION_ANNOTATION,
+} from '@backstage/catalog-model';
+import { getVoidLogger, Git } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+
+describe('BitbucketPreparer', () => {
+  let mockEntity: TemplateEntityV1alpha1;
+  const mockGitClient = {
+    clone: jest.fn(),
+  };
+
+  jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    mockEntity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Template',
+      metadata: {
+        annotations: {
+          [LOCATION_ANNOTATION]:
+            'bitbucket:https://bitbucket.org/backstage-project/backstage-repo',
+        },
+        name: 'graphql-starter',
+        title: 'GraphQL Service',
+        description:
+          'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
+        uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
+        etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
+        generation: 1,
+      },
+      spec: {
+        type: 'website',
+        templater: 'cookiecutter',
+        path: './template',
+        schema: {
+          $schema: 'http://json-schema.org/draft-07/schema#',
+          required: ['storePath', 'owner'],
+          properties: {
+            owner: {
+              type: 'string',
+              title: 'Owner',
+              description: 'Who is going to own this component',
+            },
+            storePath: {
+              type: 'string',
+              title: 'Store path',
+              description: 'GitHub store path in org/repo format',
+            },
+          },
+        },
+      },
+    };
+  });
+
+  it('calls the clone command with the correct arguments for a repository', async () => {
+    const preparer = new BitbucketPreparer(new ConfigReader({}));
+    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+    expect(mockGitClient.clone).toHaveBeenCalledWith({
+      url: 'https://bitbucket.org/backstage-project/backstage-repo',
+      dir: expect.any(String),
+    });
+  });
+
+  it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => {
+    const preparer = new BitbucketPreparer(
+      new ConfigReader({
+        integrations: {
+          bitbucket: [
+            {
+              host: 'bitbucket.org',
+              username: 'fake-user',
+              appPassword: 'fake-password',
+            },
+          ],
+        },
+      }),
+    );
+    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+    expect(mockGitClient.clone).toHaveBeenCalledWith({
+      url: 'https://bitbucket.org/backstage-project/backstage-repo',
+      dir: expect.any(String),
+    });
+  });
+
+  it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
+    const preparer = new BitbucketPreparer(new ConfigReader({}));
+    delete mockEntity.spec.path;
+    await preparer.prepare(mockEntity, { logger: getVoidLogger() });
+    expect(mockGitClient.clone).toHaveBeenCalledWith({
+      url: 'https://bitbucket.org/backstage-project/backstage-repo',
+      dir: expect.any(String),
+    });
+  });
+
+  it('return the temp directory with the path to the folder if it is specified', async () => {
+    const preparer = new BitbucketPreparer(new ConfigReader({}));
+    mockEntity.spec.path = './template/test/1/2/3';
+    const response = await preparer.prepare(mockEntity, {
+      logger: getVoidLogger(),
+    });
+
+    expect(response.split('\\').join('/')).toMatch(
+      /\/template\/test\/1\/2\/3$/,
+    );
+  });
+
+  it('return the working directory with the path to the folder if it is specified', async () => {
+    const preparer = new BitbucketPreparer(new ConfigReader({}));
+    mockEntity.spec.path = './template/test/1/2/3';
+    const response = await preparer.prepare(mockEntity, {
+      logger: getVoidLogger(),
+      workingDirectory: '/workDir',
+    });
+
+    expect(response.split('\\').join('/')).toMatch(
+      /\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
+    );
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import os from 'os';
+import fs from 'fs-extra';
+import path from 'path';
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { parseLocationAnnotation } from '../helpers';
+import { InputError, Git } from '@backstage/backend-common';
+import { PreparerBase, PreparerOptions } from './types';
+import GitUriParser from 'git-url-parse';
+import { Config } from '@backstage/config';
+
+export class BitbucketPreparer implements PreparerBase {
+  private readonly privateToken: string;
+  private readonly username: string;
+
+  constructor(config: Config) {
+    this.username =
+      config.getOptionalString('scaffolder.bitbucket.api.username') ?? '';
+    this.privateToken =
+      config.getOptionalString('scaffolder.bitbucket.api.token') ?? '';
+  }
+
+  async prepare(
+    template: TemplateEntityV1alpha1,
+    opts: PreparerOptions,
+  ): Promise<string> {
+    const { protocol, location } = parseLocationAnnotation(template);
+    const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
+    const { logger } = opts;
+
+    if (!['bitbucket', 'url'].includes(protocol)) {
+      throw new InputError(
+        `Wrong location protocol: ${protocol}, should be 'url'`,
+      );
+    }
+    const templateId = template.metadata.name;
+
+    const repo = GitUriParser(location);
+    const repositoryCheckoutUrl = repo.toString('https');
+
+    const tempDir = await fs.promises.mkdtemp(
+      path.join(workingDirectory, templateId),
+    );
+
+    const templateDirectory = path.join(
+      `${path.dirname(repo.filepath)}`,
+      template.spec.path ?? '.',
+    );
+
+    const checkoutLocation = path.resolve(tempDir, templateDirectory);
+
+    const git = this.privateToken
+      ? Git.fromAuth({
+          username: this.username,
+          password: this.privateToken,
+          logger,
+        })
+      : Git.fromAuth({ logger });
+
+    await git.clone({
+      url: repositoryCheckoutUrl,
+      dir: tempDir,
+    });
+
+    return checkoutLocation;
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts

@@ -28,6 +28,7 @@ import { FilePreparer } from './file';
 import { GitlabPreparer } from './gitlab';
 import { AzurePreparer } from './azure';
 import { GithubPreparer } from './github';
+import { BitbucketPreparer } from './bitbucket';
 
 export class Preparers implements PreparerBuilder {
   private preparerMap = new Map<RemoteProtocol, PreparerBase>();
@@ -79,11 +80,13 @@ export class Preparers implements PreparerBuilder {
     const filePreparer = new FilePreparer();
     const gitlabPreparer = new GitlabPreparer(config);
     const azurePreparer = new AzurePreparer(config);
+    const bitbucketPreparer = new BitbucketPreparer(config);
 
     preparers.register('file', filePreparer);
     preparers.register('gitlab', gitlabPreparer);
     preparers.register('gitlab/api', gitlabPreparer);
     preparers.register('azure/api', azurePreparer);
+    preparers.register('bitbucket', bitbucketPreparer);
 
     const githubConfig = config.getOptionalConfig('scaffolder.github');
     if (githubConfig) {

plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('./helpers');
+
+import { BitbucketPublisher } from './bitbucket';
+import { initRepoAndPush } from './helpers';
+import { getVoidLogger } from '@backstage/backend-common';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import { msw } from '@backstage/test-utils';
+
+describe('Bitbucket Publisher', () => {
+  const logger = getVoidLogger();
+  const server = setupServer();
+  msw.setupDefaultHandlers(server);
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('publish: createRemoteInBitbucketCloud', () => {
+    it('should create repo in bitbucket cloud', async () => {
+      server.use(
+        rest.post(
+          'https://api.bitbucket.org/2.0/repositories/project/repo',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/json'),
+              ctx.json({
+                links: {
+                  html: {
+                    href: 'https://bitbucket.org/project/repo',
+                  },
+                  clone: [
+                    {
+                      name: 'https',
+                      href: 'https://bitbucket.org/project/repo',
+                    },
+                  ],
+                },
+              }),
+            ),
+        ),
+      );
+
+      const publisher = new BitbucketPublisher(
+        'https://bitbucket.org',
+        'fake-user',
+        'fake-token',
+      );
+
+      const result = await publisher.publish({
+        values: {
+          storePath: 'project/repo',
+          owner: 'bob',
+        },
+        directory: '/tmp/test',
+        logger: logger,
+      });
+
+      expect(result).toEqual({
+        remoteUrl: 'https://bitbucket.org/project/repo',
+        catalogInfoUrl:
+          'https://bitbucket.org/project/repo/src/master/catalog-info.yaml',
+      });
+
+      expect(initRepoAndPush).toHaveBeenCalledWith({
+        dir: '/tmp/test',
+        remoteUrl: 'https://bitbucket.org/project/repo',
+        auth: { username: 'fake-user', password: 'fake-token' },
+        logger: logger,
+      });
+    });
+  });
+  describe('publish: createRemoteInBitbucketServer', () => {
+    it('should create repo in bitbucket server', async () => {
+      server.use(
+        rest.post(
+          'https://bitbucket.mycompany.com/rest/api/1.0/projects/project/repos',
+          (_, res, ctx) =>
+            res(
+              ctx.status(201),
+              ctx.set('Content-Type', 'application/json'),
+              ctx.json({
+                links: {
+                  self: [
+                    {
+                      href:
+                        'https://bitbucket.mycompany.com/projects/project/repos/repo',
+                    },
+                  ],
+                  clone: [
+                    {
+                      name: 'http',
+                      href: 'https://bitbucket.mycompany.com/scm/project/repo',
+                    },
+                  ],
+                },
+              }),
+            ),
+        ),
+      );
+
+      const publisher = new BitbucketPublisher(
+        'https://bitbucket.mycompany.com',
+        'fake-user',
+        'fake-token',
+      );
+
+      const result = await publisher.publish({
+        values: {
+          storePath: 'project/repo',
+          owner: 'bob',
+        },
+        directory: '/tmp/test',
+        logger: logger,
+      });
+
+      expect(result).toEqual({
+        remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo',
+        catalogInfoUrl:
+          'https://bitbucket.mycompany.com/projects/project/repos/repo/catalog-info.yaml',
+      });
+
+      expect(initRepoAndPush).toHaveBeenCalledWith({
+        dir: '/tmp/test',
+        remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo',
+        auth: { username: 'fake-user', password: 'fake-token' },
+        logger: logger,
+      });
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { PublisherBase, PublisherOptions, PublisherResult } from './types';
+import { initRepoAndPush } from './helpers';
+import { RequiredTemplateValues } from '../templater';
+import { JsonValue } from '../../../../../../packages/config/src';
+import fetch from 'cross-fetch';
+
+export class BitbucketPublisher implements PublisherBase {
+  private readonly host: string;
+  private readonly username: string;
+  private readonly token: string;
+
+  constructor(host: string, username: string, token: string) {
+    this.host = host;
+    this.username = username;
+    this.token = token;
+  }
+
+  async publish({
+    values,
+    directory,
+    logger,
+  }: PublisherOptions): Promise<PublisherResult> {
+    const result = await this.createRemote(values);
+
+    await initRepoAndPush({
+      dir: directory,
+      remoteUrl: result.remoteUrl,
+      auth: {
+        username: this.username,
+        password: this.token,
+      },
+      logger,
+    });
+    return result;
+  }
+
+  private async createRemote(
+    values: RequiredTemplateValues & Record<string, JsonValue>,
+  ): Promise<PublisherResult> {
+    if (this.host === 'https://bitbucket.org') {
+      return this.createBitbucketCloudRepository(values);
+    }
+    return this.createBitbucketServerRepository(values);
+  }
+
+  private async createBitbucketCloudRepository(
+    values: RequiredTemplateValues & Record<string, JsonValue>,
+  ): Promise<PublisherResult> {
+    const [project, name] = values.storePath.split('/');
+
+    let response: Response;
+    const buffer = Buffer.from(`${this.username}:${this.token}`, 'utf8');
+
+    const options: RequestInit = {
+      method: 'POST',
+      body: JSON.stringify({
+        scm: 'git',
+        description: values.description,
+      }),
+      headers: {
+        Authorization: `Basic ${buffer.toString('base64')}`,
+        'Content-Type': 'application/json',
+      },
+    };
+    try {
+      response = await fetch(
+        `https://api.bitbucket.org/2.0/repositories/${project}/${name}`,
+        options,
+      );
+    } catch (e) {
+      throw new Error(`Unable to create repository, ${e}`);
+    }
+    if (response.status === 200) {
+      const r = await response.json();
+      let remoteUrl = '';
+      for (const link of r.links.clone) {
+        if (link.name === 'https') {
+          remoteUrl = link.href;
+        }
+      }
+
+      // TODO use the urlReader to get the defautl branch
+      const catalogInfoUrl = `${r.links.html.href}/src/master/catalog-info.yaml`;
+      return { remoteUrl, catalogInfoUrl };
+    }
+    throw new Error(`Not a valid response code ${await response.text()}`);
+  }
+
+  private async createBitbucketServerRepository(
+    values: RequiredTemplateValues & Record<string, JsonValue>,
+  ): Promise<PublisherResult> {
+    const [project, name] = values.storePath.split('/');
+
+    let response: Response;
+    const options: RequestInit = {
+      method: 'POST',
+      body: JSON.stringify({
+        name: name,
+        description: values.description,
+      }),
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+        'Content-Type': 'application/json',
+      },
+    };
+    try {
+      response = await fetch(
+        `${this.host}/rest/api/1.0/projects/${project}/repos`,
+        options,
+      );
+    } catch (e) {
+      throw new Error(`Unable to create repository, ${e}`);
+    }
+    if (response.status === 201) {
+      const r = await response.json();
+      let remoteUrl = '';
+      for (const link of r.links.clone) {
+        if (link.name === 'http') {
+          remoteUrl = link.href;
+        }
+      }
+      const catalogInfoUrl = `${r.links.self[0].href}/catalog-info.yaml`;
+      return { remoteUrl, catalogInfoUrl };
+    }
+    throw new Error(`Not a valid response code ${await response.text()}`);
+  }
+}

plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts

@@ -30,6 +30,7 @@ import { RemoteProtocol } from '../types';
 import { GithubPublisher, RepoVisibilityOptions } from './github';
 import { GitlabPublisher } from './gitlab';
 import { AzurePublisher } from './azure';
+import { BitbucketPublisher } from './bitbucket';
 
 export class Publishers implements PublisherBuilder {
   private publisherMap = new Map<RemoteProtocol, PublisherBase>();
@@ -163,6 +164,34 @@ export class Publishers implements PublisherBuilder {
       }
     }
 
+    const bitbucketConfig = config.getOptionalConfig(
+      'scaffolder.bitbucket.api',
+    );
+    if (bitbucketConfig) {
+      try {
+        const baseUrl = bitbucketConfig.getString('host');
+        const bitbucketUsername = bitbucketConfig.getString('username');
+        const bitbucketToken = bitbucketConfig.getString('token');
+
+        const bitbucketPublisher = new BitbucketPublisher(
+          baseUrl,
+          bitbucketUsername,
+          bitbucketToken,
+        );
+        publishers.register('bitbucket', bitbucketPublisher);
+      } catch (e) {
+        const providerName = 'bitbucket';
+        if (process.env.NODE_ENV !== 'development') {
+          throw new Error(
+            `Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
+          );
+        }
+
+        logger.warn(
+          `Skipping ${providerName} scaffolding provider, ${e.message}`,
+        );
+      }
+    }
     return publishers;
   }
 }

plugins/scaffolder-backend/src/scaffolder/stages/types.ts

@@ -18,4 +18,5 @@ export type RemoteProtocol =
   | 'github'
   | 'gitlab'
   | 'gitlab/api'
-  | 'azure/api';
+  | 'azure/api'
+  | 'bitbucket';