From 56a9f8389b3a40fd5d1870969a10e18b55af7838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Mon, 21 Sep 2020 13:50:31 +0000 Subject: [PATCH] Add Azure DevOps publisher to scaffolder --- app-config.yaml | 3 + .../software-templates/installation.md | 9 +- packages/backend/package.json | 1 + packages/backend/src/plugins/scaffolder.ts | 29 +++ plugins/scaffolder-backend/package.json | 1 + .../azure-devops-node-api/GitApi/index.ts | 25 +++ .../scaffolder/stages/publish/azure.test.ts | 179 ++++++++++++++++++ .../src/scaffolder/stages/publish/azure.ts | 82 ++++++++ .../src/scaffolder/stages/publish/index.ts | 1 + yarn.lock | 30 ++- 10 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 plugins/scaffolder-backend/src/scaffolder/__mocks__/azure-devops-node-api/GitApi/index.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts create mode 100644 plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts diff --git a/app-config.yaml b/app-config.yaml index c7c5101b55..b9bdd94192 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -134,6 +134,9 @@ scaffolder: env: GITLAB_ACCESS_TOKEN azure: api: + organization: + $secret: + env: AZURE_ORGANIZATION token: $secret: env: AZURE_PRIVATE_TOKEN diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index b804c84e52..c374ab8d72 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -245,13 +245,18 @@ scaffolder: #### Azure DevOps -For Azure DevOps we currently support the preparer stage with the configuration -of a private access token (PAT) when needed: +For Azure DevOps we support both the preparer and publisher stage with the +configuration of a private access token (PAT) when needed. For the publisher +it's also required to define the organization that you want to create the +repository in. ```yaml scaffolder: azure: api: + organization: + $secret: + env: AZURE_ORGANIZATION token: $secret: env: AZURE_PRIVATE_TOKEN diff --git a/packages/backend/package.json b/packages/backend/package.json index 3c40d03def..8f70aa1fc3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -33,6 +33,7 @@ "@backstage/plugin-techdocs-backend": "^0.1.1-alpha.23", "@gitbeaker/node": "^23.5.0", "@octokit/rest": "^18.0.0", + "azure-devops-node-api": "^10.1.1", "dockerode": "^3.2.0", "example-app": "^0.1.1-alpha.23", "express": "^4.17.1", diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 17cd7ba10e..f3dde321f3 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -25,12 +25,14 @@ import { Publishers, GithubPublisher, GitlabPublisher, + AzurePublisher, CreateReactAppTemplater, Templaters, RepoVisibilityOptions, } from '@backstage/plugin-scaffolder-backend'; import { Octokit } from '@octokit/rest'; import { Gitlab } from '@gitbeaker/node'; +import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api'; import type { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -114,6 +116,33 @@ export default async function createPlugin({ } } + const azureConfig = config.getOptionalConfig('scaffolder.azure.api'); + if (azureConfig) { + try { + const organization = azureConfig.getString('organization'); + const azureToken = azureConfig.getString('token'); + + const serverUrl = `https://dev.azure.com/${organization}`; + const authHandler = getPersonalAccessTokenHandler(azureToken); + const webApi = new WebApi(serverUrl, authHandler); + const azureClient = await webApi.getGitApi(); + + const azurePublisher = new AzurePublisher(azureClient, azureToken); + publishers.register('azure/api', azurePublisher); + } catch (e) { + const providerName = 'azure'; + if (process.env.NODE_ENV !== 'development') { + throw new Error( + `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, + ); + } + + logger.warn( + `Skipping ${providerName} scaffolding provider, ${e.message}`, + ); + } + } + const dockerClient = new Docker(); return await createRouter({ preparers, diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 90d1af4b5f..7a7938a483 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -28,6 +28,7 @@ "@octokit/rest": "^18.0.0", "@types/dockerode": "^2.5.32", "@types/express": "^4.17.6", + "azure-devops-node-api": "^10.1.1", "command-exists-promise": "^2.0.2", "compression": "^1.7.4", "cors": "^2.8.5", diff --git a/plugins/scaffolder-backend/src/scaffolder/__mocks__/azure-devops-node-api/GitApi/index.ts b/plugins/scaffolder-backend/src/scaffolder/__mocks__/azure-devops-node-api/GitApi/index.ts new file mode 100644 index 0000000000..116f092866 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/__mocks__/azure-devops-node-api/GitApi/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const mockGitApi = { + createRepository: jest.fn(), +}; + +export class GitApi { + constructor() { + return mockGitApi; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts new file mode 100644 index 0000000000..8b84eb0edf --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -0,0 +1,179 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('nodegit'); +jest.mock('azure-devops-node-api/GitApi'); +jest.mock('azure-devops-node-api/interfaces/GitInterfaces'); + +import { AzurePublisher } from './azure'; +import { GitApi } from 'azure-devops-node-api/GitApi'; +import * as NodeGit from 'nodegit'; + +const { mockGitApi } = require('azure-devops-node-api/GitApi') as { + mockGitApi: { + createRepository: jest.MockedFunction; + }; +}; + +const { + Repository, + mockRepo, + mockIndex, + Signature, + Remote, + mockRemote, + Cred, +} = require('nodegit') as { + Repository: jest.Mocked<{ init: any }>; + Signature: jest.Mocked<{ now: any }>; + Cred: jest.Mocked<{ userpassPlaintextNew: any }>; + Remote: jest.Mocked<{ create: any }>; + + mockIndex: jest.Mocked; + mockRepo: jest.Mocked; + mockRemote: jest.Mocked; +}; + +describe('Azure Publisher', () => { + const publisher = new AzurePublisher(new GitApi('', []), 'fake-token'); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('publish: createRemoteInAzure', () => { + it('should use azure-devops-node-api to create a repo in the given project', async () => { + mockGitApi.createRepository.mockResolvedValue({ + remoteUrl: 'mockclone', + } as { remoteUrl: string }); + + await publisher.publish({ + values: { + storePath: 'project/repo', + owner: 'bob', + }, + directory: '/tmp/test', + }); + + expect(mockGitApi.createRepository).toHaveBeenCalledWith( + { + name: 'repo', + }, + 'project', + ); + }); + }); + + describe('publish: createGitDirectory', () => { + const values = { + isOrg: true, + storePath: 'blam/test', + owner: 'lols', + }; + + const mockDir = '/tmp/test/dir'; + + mockGitApi.createRepository.mockResolvedValue({ + remoteUrl: 'mockclone', + } as { remoteUrl: string }); + + it('should call init on the repo with the directory', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Repository.init).toHaveBeenCalledWith(mockDir, 0); + }); + + it('should call refresh index on the index and write the new files', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(mockRepo.refreshIndex).toHaveBeenCalled(); + }); + + it('should call add all files and write', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(mockIndex.addAll).toHaveBeenCalled(); + expect(mockIndex.write).toHaveBeenCalled(); + expect(mockIndex.writeTree).toHaveBeenCalled(); + }); + + it('should create a commit with on head with the right name and commiter', async () => { + const mockSignature = { mockSignature: 'bloblly' }; + Signature.now.mockReturnValue(mockSignature); + + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Signature.now).toHaveBeenCalledTimes(2); + expect(Signature.now).toHaveBeenCalledWith( + 'Scaffolder', + 'scaffolder@backstage.io', + ); + + expect(mockRepo.createCommit).toHaveBeenCalledWith( + 'HEAD', + mockSignature, + mockSignature, + 'initial commit', + 'mockoid', + [], + ); + }); + + it('creates a remote with the repo and remote', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Remote.create).toHaveBeenCalledWith( + mockRepo, + 'origin', + 'mockclone', + ); + }); + + it('shoud push to the remote repo', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + const [remotes, { callbacks }] = mockRemote.push.mock + .calls[0] as NodeGit.PushOptions[]; + + expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); + + callbacks?.credentials?.(); + + expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( + 'notempty', + 'fake-token', + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts new file mode 100644 index 0000000000..9dfc7b5b66 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PublisherBase } from './types'; +import { GitApi } from 'azure-devops-node-api/GitApi'; +import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; + +import { JsonValue } from '@backstage/config'; +import { RequiredTemplateValues } from '../templater'; +import { Repository, Remote, Signature, Cred } from 'nodegit'; + +export class AzurePublisher implements PublisherBase { + private readonly client: GitApi; + private readonly token: string; + + constructor(client: GitApi, token: string) { + this.client = client; + this.token = token; + } + + async publish({ + values, + directory, + }: { + values: RequiredTemplateValues & Record; + directory: string; + }): Promise<{ remoteUrl: string }> { + const remoteUrl = await this.createRemote(values); + await this.pushToRemote(directory, remoteUrl); + + return { remoteUrl }; + } + + private async createRemote( + values: RequiredTemplateValues & Record, + ) { + const [project, name] = values.storePath.split('/'); + + const createOptions: GitRepositoryCreateOptions = { name }; + const repo = await this.client.createRepository(createOptions, project); + + return repo.remoteUrl || ''; + } + + private async pushToRemote(directory: string, remote: string): Promise { + const repo = await Repository.init(directory, 0); + const index = await repo.refreshIndex(); + await index.addAll(); + await index.write(); + const oid = await index.writeTree(); + await repo.createCommit( + 'HEAD', + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + 'initial commit', + oid, + [], + ); + + const remoteRepo = await Remote.create(repo, 'origin', remote); + + await remoteRepo.push(['refs/heads/master:refs/heads/master'], { + callbacks: { + // Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat + credentials: () => Cred.userpassPlaintextNew('notempty', this.token), + }, + }); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts index 1d89cd37f0..b37baa3246 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts @@ -16,4 +16,5 @@ export * from './publishers'; export * from './github'; export * from './gitlab'; +export * from './azure'; export * from './types'; diff --git a/yarn.lock b/yarn.lock index 804eb9dc10..73863f110f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6822,6 +6822,15 @@ axobject-query@^2.0.2: resolved "https://registry.npmjs.org/axobject-query/-/axobject-query-2.1.2.tgz#2bdffc0371e643e5f03ba99065d5179b9ca79799" integrity sha512-ICt34ZmrVt8UQnvPl6TVyDTkmhXmAyAT4Jh5ugfGUX4MOrZ+U/ZY6/sdylRw3qGNr9Ub5AJsaHeDMzNLehRdOQ== +azure-devops-node-api@^10.1.1: + version "10.1.1" + resolved "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-10.1.1.tgz#9016d8935316fff260f5f8fafd81d0caff90a19e" + integrity sha512-P4Hyrh/+Nzc2KXQk73z72/GsenSWIH5o8uiyELqykJYs9TWTVCxVwghoR7lPeiY6QVoXkq2S2KtvAgi5fyjl9w== + dependencies: + tunnel "0.0.6" + typed-rest-client "^1.7.3" + underscore "1.8.3" + babel-code-frame@^6.22.0: version "6.26.0" resolved "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" @@ -18545,7 +18554,7 @@ qs@6.7.0: resolved "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== -qs@^6.5.1, qs@^6.6.0, qs@^6.9.4: +qs@^6.5.1, qs@^6.6.0, qs@^6.9.1, qs@^6.9.4: version "6.9.4" resolved "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687" integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ== @@ -22226,6 +22235,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tunnel@0.0.6: + version "0.0.6" + resolved "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -22293,6 +22307,15 @@ type@^2.0.0: resolved "https://registry.npmjs.org/type/-/type-2.0.0.tgz#5f16ff6ef2eb44f260494dae271033b29c09a9c3" integrity sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow== +typed-rest-client@^1.7.3: + version "1.7.3" + resolved "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.7.3.tgz#1beb263b86b14d34596f6127c6172dd5fd652e7b" + integrity sha512-CwTpx/TkRHGZoHkJhBcp4X8K3/WtlzSHVQR0OIFnt10j4tgy4ypgq/SrrgVpA1s6tAL49Q6J3R5C0Cgfh2ddqA== + dependencies: + qs "^6.9.1" + tunnel "0.0.6" + underscore "1.8.3" + typed-styles@^0.0.7: version "0.0.7" resolved "https://registry.npmjs.org/typed-styles/-/typed-styles-0.0.7.tgz#93392a008794c4595119ff62dde6809dbc40a3d9" @@ -22360,6 +22383,11 @@ undefsafe@^2.0.2: dependencies: debug "^2.2.0" +underscore@1.8.3: + version "1.8.3" + resolved "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022" + integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI= + underscore@^1.9.1: version "1.11.0" resolved "https://registry.npmjs.org/underscore/-/underscore-1.11.0.tgz#dd7c23a195db34267186044649870ff1bab5929e"