Merge pull request #2543 from mfrinnstrom/azure-scaffolder-prepare

Add Azure DevOps support to the scaffolder
This commit is contained in:
Ben Lambert
2020-09-28 23:44:19 +02:00
committed by GitHub
14 changed files with 598 additions and 6 deletions
+6
View File
@@ -138,6 +138,12 @@ scaffolder:
token:
$secret:
env: GITLAB_ACCESS_TOKEN
azure:
baseUrl: https://dev.azure.com/{your-organization}
api:
token:
$secret:
env: AZURE_PRIVATE_TOKEN
auth:
providers:
@@ -217,6 +217,11 @@ The Github access token is retrieved from environment variables via the config.
The config file needs to specify what environment variable the token is
retrieved from. Your config should have the following objects.
You can configure who can see the new repositories that the scaffolder creates
by specifying `visibility` option. Valid options are `public`, `private` and
`internal`. `internal` options is for GitHub Enterprise clients, which means
public within the organization.
#### Gitlab
For Gitlab, we currently support the configuration of the GitLab publisher and
@@ -238,10 +243,23 @@ scaffolder:
env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN
```
You can configure who can see the new repositories that the scaffolder creates
by specifying `visibility` option. Valid options are `public`, `private` and
`internal`. `internal` option is for GitHub Enterprise clients, which means
public within the organization.
#### Azure DevOps
For Azure DevOps we support both the preparer and publisher stage with the
configuration of a private access token (PAT). For the publisher it's also
required to define the base URL for the client to connect to the service. This
will hopefully support on-prem installations as well but that has not been
verified.
```yaml
scaffolder:
azure:
baseUrl: https://dev.azure.com/{your-organization}
api:
token:
$secret:
env: AZURE_PRIVATE_TOKEN
```
### Running the Backend
+1
View File
@@ -33,6 +33,7 @@
"@backstage/plugin-techdocs-backend": "^0.1.1-alpha.23",
"@gitbeaker/node": "^23.5.0",
"@octokit/rest": "^18.0.0",
"azure-devops-node-api": "^10.1.1",
"dockerode": "^3.2.0",
"example-app": "^0.1.1-alpha.23",
"express": "^4.17.1",
@@ -20,16 +20,19 @@ import {
FilePreparer,
GithubPreparer,
GitlabPreparer,
AzurePreparer,
Preparers,
Publishers,
GithubPublisher,
GitlabPublisher,
AzurePublisher,
CreateReactAppTemplater,
Templaters,
RepoVisibilityOptions,
} from '@backstage/plugin-scaffolder-backend';
import { Octokit } from '@octokit/rest';
import { Gitlab } from '@gitbeaker/node';
import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
import type { PluginEnvironment } from '../types';
import Docker from 'dockerode';
@@ -46,12 +49,14 @@ export default async function createPlugin({
const filePreparer = new FilePreparer();
const githubPreparer = new GithubPreparer();
const gitlabPreparer = new GitlabPreparer(config);
const azurePreparer = new AzurePreparer(config);
const preparers = new Preparers();
preparers.register('file', filePreparer);
preparers.register('github', githubPreparer);
preparers.register('gitlab', gitlabPreparer);
preparers.register('gitlab/api', gitlabPreparer);
preparers.register('azure/api', azurePreparer);
const publishers = new Publishers();
@@ -111,6 +116,32 @@ export default async function createPlugin({
}
}
const azureConfig = config.getOptionalConfig('scaffolder.azure');
if (azureConfig) {
try {
const baseUrl = azureConfig.getString('baseUrl');
const azureToken = azureConfig.getConfig('api').getString('token');
const authHandler = getPersonalAccessTokenHandler(azureToken);
const webApi = new WebApi(baseUrl, authHandler);
const azureClient = await webApi.getGitApi();
const azurePublisher = new AzurePublisher(azureClient, azureToken);
publishers.register('azure/api', azurePublisher);
} catch (e) {
const providerName = 'azure';
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
);
}
logger.warn(
`Skipping ${providerName} scaffolding provider, ${e.message}`,
);
}
}
const dockerClient = new Docker();
return await createRouter({
preparers,
+1
View File
@@ -28,6 +28,7 @@
"@octokit/rest": "^18.0.0",
"@types/dockerode": "^2.5.32",
"@types/express": "^4.17.6",
"azure-devops-node-api": "^10.1.1",
"command-exists-promise": "^2.0.2",
"compression": "^1.7.4",
"cors": "^2.8.5",
@@ -0,0 +1,25 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const mockGitApi = {
createRepository: jest.fn(),
};
export class GitApi {
constructor() {
return mockGitApi;
}
}
@@ -0,0 +1,138 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const mocks = {
Clone: { clone: jest.fn() },
CheckoutOptions: jest.fn(() => {}),
};
jest.doMock('nodegit', () => mocks);
import { AzurePreparer } from './azure';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
describe('AzurePreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
it('calls the clone command with the correct arguments for a repository', async () => {
const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
expect.any(String),
{},
);
});
it('calls the clone command with the correct arguments if an access token is provided for a repository', async () => {
const preparer = new AzurePreparer(
ConfigReader.fromConfigs([
{
context: '',
data: {
scaffolder: {
azure: {
api: {
token: 'fake-token',
},
},
},
},
},
]),
);
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
expect.any(String),
{
fetchOpts: {
callbacks: {
credentials: expect.anything(),
},
},
},
);
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
delete mockEntity.spec.path;
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
expect.any(String),
{},
);
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = new AzurePreparer(ConfigReader.fromConfigs([]));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity);
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
});
@@ -0,0 +1,76 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import path from 'path';
import os from 'os';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
import GitUriParser from 'git-url-parse';
import { Clone, Cred } from 'nodegit';
import { Config } from '@backstage/config';
export class AzurePreparer implements PreparerBase {
private readonly privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('scaffolder.azure.api.token') ?? '';
}
async prepare(template: TemplateEntityV1alpha1): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
if (protocol !== 'azure/api') {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'azure/api'`,
);
}
const templateId = template.metadata.name;
const url = new URL(location); // Need to extract filepath from search parameter
const parsedGitLocation = GitUriParser(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), templateId),
);
const templateDirectory = path.join(
`${path
.dirname(url.searchParams.get('path') || '')
.replace(/^\/+/g, '')}`, // Strip leading slash
template.spec.path ?? '.',
);
const options = this.privateToken
? {
fetchOpts: {
callbacks: {
credentials: () =>
// Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
Cred.userpassPlaintextNew('notempty', this.privateToken),
},
},
}
: {};
await Clone.clone(repositoryCheckoutUrl, tempDir, options);
return path.resolve(tempDir, templateDirectory);
}
}
@@ -18,3 +18,4 @@ export * from './types';
export * from './file';
export * from './github';
export * from './gitlab';
export * from './azure';
@@ -0,0 +1,179 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('nodegit');
jest.mock('azure-devops-node-api/GitApi');
jest.mock('azure-devops-node-api/interfaces/GitInterfaces');
import { AzurePublisher } from './azure';
import { GitApi } from 'azure-devops-node-api/GitApi';
import * as NodeGit from 'nodegit';
const { mockGitApi } = require('azure-devops-node-api/GitApi') as {
mockGitApi: {
createRepository: jest.MockedFunction<GitApi['createRepository']>;
};
};
const {
Repository,
mockRepo,
mockIndex,
Signature,
Remote,
mockRemote,
Cred,
} = require('nodegit') as {
Repository: jest.Mocked<{ init: any }>;
Signature: jest.Mocked<{ now: any }>;
Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
Remote: jest.Mocked<{ create: any }>;
mockIndex: jest.Mocked<NodeGit.Index>;
mockRepo: jest.Mocked<NodeGit.Repository>;
mockRemote: jest.Mocked<NodeGit.Remote>;
};
describe('Azure Publisher', () => {
const publisher = new AzurePublisher(new GitApi('', []), 'fake-token');
beforeEach(() => {
jest.clearAllMocks();
});
describe('publish: createRemoteInAzure', () => {
it('should use azure-devops-node-api to create a repo in the given project', async () => {
mockGitApi.createRepository.mockResolvedValue({
remoteUrl: 'mockclone',
} as { remoteUrl: string });
await publisher.publish({
values: {
storePath: 'project/repo',
owner: 'bob',
},
directory: '/tmp/test',
});
expect(mockGitApi.createRepository).toHaveBeenCalledWith(
{
name: 'repo',
},
'project',
);
});
});
describe('publish: createGitDirectory', () => {
const values = {
isOrg: true,
storePath: 'blam/test',
owner: 'lols',
};
const mockDir = '/tmp/test/dir';
mockGitApi.createRepository.mockResolvedValue({
remoteUrl: 'mockclone',
} as { remoteUrl: string });
it('should call init on the repo with the directory', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
});
it('should call refresh index on the index and write the new files', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(mockRepo.refreshIndex).toHaveBeenCalled();
});
it('should call add all files and write', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(mockIndex.addAll).toHaveBeenCalled();
expect(mockIndex.write).toHaveBeenCalled();
expect(mockIndex.writeTree).toHaveBeenCalled();
});
it('should create a commit with on head with the right name and commiter', async () => {
const mockSignature = { mockSignature: 'bloblly' };
Signature.now.mockReturnValue(mockSignature);
await publisher.publish({
values,
directory: mockDir,
});
expect(Signature.now).toHaveBeenCalledTimes(2);
expect(Signature.now).toHaveBeenCalledWith(
'Scaffolder',
'scaffolder@backstage.io',
);
expect(mockRepo.createCommit).toHaveBeenCalledWith(
'HEAD',
mockSignature,
mockSignature,
'initial commit',
'mockoid',
[],
);
});
it('creates a remote with the repo and remote', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(Remote.create).toHaveBeenCalledWith(
mockRepo,
'origin',
'mockclone',
);
});
it('shoud push to the remote repo', async () => {
await publisher.publish({
values,
directory: mockDir,
});
const [remotes, { callbacks }] = mockRemote.push.mock
.calls[0] as NodeGit.PushOptions[];
expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
callbacks?.credentials?.();
expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
'notempty',
'fake-token',
);
});
});
});
@@ -0,0 +1,82 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { PublisherBase } from './types';
import { GitApi } from 'azure-devops-node-api/GitApi';
import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces';
import { JsonValue } from '@backstage/config';
import { RequiredTemplateValues } from '../templater';
import { Repository, Remote, Signature, Cred } from 'nodegit';
export class AzurePublisher implements PublisherBase {
private readonly client: GitApi;
private readonly token: string;
constructor(client: GitApi, token: string) {
this.client = client;
this.token = token;
}
async publish({
values,
directory,
}: {
values: RequiredTemplateValues & Record<string, JsonValue>;
directory: string;
}): Promise<{ remoteUrl: string }> {
const remoteUrl = await this.createRemote(values);
await this.pushToRemote(directory, remoteUrl);
return { remoteUrl };
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
) {
const [project, name] = values.storePath.split('/');
const createOptions: GitRepositoryCreateOptions = { name };
const repo = await this.client.createRepository(createOptions, project);
return repo.remoteUrl || '';
}
private async pushToRemote(directory: string, remote: string): Promise<void> {
const repo = await Repository.init(directory, 0);
const index = await repo.refreshIndex();
await index.addAll();
await index.write();
const oid = await index.writeTree();
await repo.createCommit(
'HEAD',
Signature.now('Scaffolder', 'scaffolder@backstage.io'),
Signature.now('Scaffolder', 'scaffolder@backstage.io'),
'initial commit',
oid,
[],
);
const remoteRepo = await Remote.create(repo, 'origin', remote);
await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
callbacks: {
// Username can anything but the empty string according to: https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
credentials: () => Cred.userpassPlaintextNew('notempty', this.token),
},
});
}
}
@@ -16,4 +16,5 @@
export * from './publishers';
export * from './github';
export * from './gitlab';
export * from './azure';
export * from './types';
@@ -13,4 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type RemoteProtocol = 'file' | 'github' | 'gitlab' | 'gitlab/api';
export type RemoteProtocol =
| 'file'
| 'github'
| 'gitlab'
| 'gitlab/api'
| 'azure/api';
+29 -1
View File
@@ -6845,6 +6845,15 @@ axobject-query@^2.0.2:
resolved "https://registry.npmjs.org/axobject-query/-/axobject-query-2.1.2.tgz#2bdffc0371e643e5f03ba99065d5179b9ca79799"
integrity sha512-ICt34ZmrVt8UQnvPl6TVyDTkmhXmAyAT4Jh5ugfGUX4MOrZ+U/ZY6/sdylRw3qGNr9Ub5AJsaHeDMzNLehRdOQ==
azure-devops-node-api@^10.1.1:
version "10.1.1"
resolved "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-10.1.1.tgz#9016d8935316fff260f5f8fafd81d0caff90a19e"
integrity sha512-P4Hyrh/+Nzc2KXQk73z72/GsenSWIH5o8uiyELqykJYs9TWTVCxVwghoR7lPeiY6QVoXkq2S2KtvAgi5fyjl9w==
dependencies:
tunnel "0.0.6"
typed-rest-client "^1.7.3"
underscore "1.8.3"
babel-code-frame@^6.22.0:
version "6.26.0"
resolved "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b"
@@ -18698,7 +18707,7 @@ qs@6.7.0:
resolved "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==
qs@^6.5.1, qs@^6.6.0, qs@^6.9.4:
qs@^6.5.1, qs@^6.6.0, qs@^6.9.1, qs@^6.9.4:
version "6.9.4"
resolved "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687"
integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==
@@ -22460,6 +22469,11 @@ tunnel-agent@^0.6.0:
dependencies:
safe-buffer "^5.0.1"
tunnel@0.0.6:
version "0.0.6"
resolved "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c"
integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==
tweetnacl@^0.14.3, tweetnacl@~0.14.0:
version "0.14.5"
resolved "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
@@ -22527,6 +22541,15 @@ type@^2.0.0:
resolved "https://registry.npmjs.org/type/-/type-2.0.0.tgz#5f16ff6ef2eb44f260494dae271033b29c09a9c3"
integrity sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==
typed-rest-client@^1.7.3:
version "1.7.3"
resolved "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.7.3.tgz#1beb263b86b14d34596f6127c6172dd5fd652e7b"
integrity sha512-CwTpx/TkRHGZoHkJhBcp4X8K3/WtlzSHVQR0OIFnt10j4tgy4ypgq/SrrgVpA1s6tAL49Q6J3R5C0Cgfh2ddqA==
dependencies:
qs "^6.9.1"
tunnel "0.0.6"
underscore "1.8.3"
typed-styles@^0.0.7:
version "0.0.7"
resolved "https://registry.npmjs.org/typed-styles/-/typed-styles-0.0.7.tgz#93392a008794c4595119ff62dde6809dbc40a3d9"
@@ -22594,6 +22617,11 @@ undefsafe@^2.0.2:
dependencies:
debug "^2.2.0"
underscore@1.8.3:
version "1.8.3"
resolved "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022"
integrity sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=
underscore@^1.9.1:
version "1.11.0"
resolved "https://registry.npmjs.org/underscore/-/underscore-1.11.0.tgz#dd7c23a195db34267186044649870ff1bab5929e"