paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Merge pull request #2426 from muffix/fix-2372-add-gitlab-to-scaffolder

Browse Source 
Add GitLab integration for scaffolder
This commit is contained in:
Ben Lambert
2020-09-17 13:54:09 +02:00
committed by GitHub
parent cf32c9d02d 55542797a9
commit ef4108dcaa
30 changed files with 945 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app-config.yaml
+6
View File
@@ -116,6 +116,12 @@ scaffolder:
$secret:
env: GITHUB_ACCESS_TOKEN
visibility: public # or 'internal' or 'private'
gitlab:
api:
baseUrl: https://gitlab.com
token:
$secret:
env: GITLAB_ACCESS_TOKEN
auth:
providers:
docs/features/software-templates/index.md
+2 -2
View File
@@ -8,8 +8,8 @@ you create Components inside Backstage
The Software Templates part of Backstage is a tool that can help you create
Components inside Backstage. By default, it has the ability to load skeletons of
code, template in some variables, and then publish the template to some location
like GitHub.
code, template in some variables, and then publish the template to some
locations like GitHub or GitLab.
<video width="100%" height="100%" controls>
<source src="https://backstage.io/blog/assets/2020-08-05/feature.mp4" type="video/mp4">
docs/features/software-templates/installation.md
+41 -5
View File
@@ -87,13 +87,17 @@ import {
createRouter,
FilePreparer,
GithubPreparer,
GitlabPreparer,
Preparers,
Publishers,
GithubPublisher,
GitlabPublisher,
CreateReactAppTemplater,
Templaters,
RepoVisilityOptions,
} from '@backstage/plugin-scaffolder-backend';
import { Octokit } from '@octokit/rest';
import { Gitlab } from '@gitbeaker/node';
import type { PluginEnvironment } from '../types';
import Docker from 'dockerode';
@@ -104,17 +108,20 @@ export default async function createPlugin({
const cookiecutterTemplater = new CookieCutter();
const craTemplater = new CreateReactAppTemplater();
const templaters = new Templaters();
// Register default templaters
templaters.register('cookiecutter', cookiecutterTemplater);
templaters.register('cra', craTemplater);
const filePreparer = new FilePreparer();
const githubPreparer = new GithubPreparer();
const gitlabPreparer = new GitlabPreparer(config);
const preparers = new Preparers();
// Register default preparers
preparers.register('file', filePreparer);
preparers.register('github', githubPreparer);
preparers.register('gitlab', gitlabPreparer);
preparers.register('gitlab/api', gitlabPreparer);
const publishers = new Publishers();
const githubToken = config.getString('scaffolder.github.token');
const repoVisibility = config.getString(
@@ -122,17 +129,32 @@ export default async function createPlugin({
) as RepoVisilityOptions;
const githubClient = new Octokit({ auth: githubToken });
const publisher = new GithubPublisher({
const githubPublisher = new GithubPublisher({
client: githubClient,
token: githubToken,
repoVisibility,
});
publishers.register('file', githubPublisher);
publishers.register('github', githubPublisher);
const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
if (gitLabConfig) {
const gitLabToken = gitLabConfig.getString('token');
const gitLabClient = new Gitlab({
host: gitLabConfig.getOptionalString('baseUrl'),
token: gitLabToken,
});
const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
publishers.register('gitlab', gitLabPublisher);
publishers.register('gitlab/api', gitLabPublisher);
}
const dockerClient = new Docker();
return await createRouter({
preparers,
templaters,
publisher,
publishers,
logger,
dockerClient,
});
@@ -179,7 +201,7 @@ catalog:
target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml
```
### Runtime Dependencies
### Runtime Dependencies / Configuration
For the scaffolder backend plugin to function, it needs a GitHub access token,
and access to a running Docker daemon. You can create a GitHub access token
@@ -189,10 +211,18 @@ docs on creating private GitHub access tokens is available
Note that the need for private GitHub access tokens will be replaced with GitHub
Apps integration further down the line.
#### Github
The Github access token is retrieved from environment variables via the config.
The config file needs to specify what environment variable the token is
retrieved from. Your config should have the following objects.
#### Gitlab
For Gitlab, we currently support the configuration of the GitLab publisher and
allows to configure the private access token and the base URL of a GitLab
instance:
```yaml
scaffolder:
github:
@@ -200,6 +230,12 @@ scaffolder:
$secret:
env: GITHUB_ACCESS_TOKEN
visibility: public # or 'internal' or 'private'
gitlab:
api:
baseUrl: https://gitlab.com
token:
$secret:
env: SCAFFOLDER_GITLAB_PRIVATE_TOKEN
```
You can configure who can see the new repositories that the scaffolder creates
packages/backend/package.json
+1
View File
@@ -31,6 +31,7 @@
"@backstage/plugin-scaffolder-backend": "^0.1.1-alpha.21",
"@backstage/plugin-sentry-backend": "^0.1.1-alpha.21",
"@backstage/plugin-techdocs-backend": "^0.1.1-alpha.21",
"@gitbeaker/node": "^23.5.0",
"@octokit/rest": "^18.0.0",
"dockerode": "^3.2.0",
"example-app": "^0.1.1-alpha.21",
packages/backend/src/plugins/scaffolder.ts
+26 -2
View File
@@ -19,13 +19,17 @@ import {
createRouter,
FilePreparer,
GithubPreparer,
GitlabPreparer,
Preparers,
Publishers,
GithubPublisher,
GitlabPublisher,
CreateReactAppTemplater,
Templaters,
RepoVisilityOptions,
} from '@backstage/plugin-scaffolder-backend';
import { Octokit } from '@octokit/rest';
import { Gitlab } from '@gitbeaker/node';
import type { PluginEnvironment } from '../types';
import Docker from 'dockerode';
@@ -41,10 +45,15 @@ export default async function createPlugin({
const filePreparer = new FilePreparer();
const githubPreparer = new GithubPreparer();
const gitlabPreparer = new GitlabPreparer(config);
const preparers = new Preparers();
preparers.register('file', filePreparer);
preparers.register('github', githubPreparer);
preparers.register('gitlab', gitlabPreparer);
preparers.register('gitlab/api', gitlabPreparer);
const publishers = new Publishers();
const githubToken = config.getString('scaffolder.github.token');
const repoVisibility = config.getString(
@@ -52,17 +61,32 @@ export default async function createPlugin({
) as RepoVisilityOptions;
const githubClient = new Octokit({ auth: githubToken });
const publisher = new GithubPublisher({
const githubPublisher = new GithubPublisher({
client: githubClient,
token: githubToken,
repoVisibility,
});
publishers.register('file', githubPublisher);
publishers.register('github', githubPublisher);
const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
if (gitLabConfig) {
const gitLabToken = gitLabConfig.getString('token');
const gitLabClient = new Gitlab({
host: gitLabConfig.getOptionalString('baseUrl'),
token: gitLabToken,
});
const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
publishers.register('gitlab', gitLabPublisher);
publishers.register('gitlab/api', gitLabPublisher);
}
const dockerClient = new Docker();
return await createRouter({
preparers,
templaters,
publisher,
publishers,
logger,
dockerClient,
});
packages/create-app/templates/default-app/packages/backend/package.json.hbs
+1
View File
@@ -28,6 +28,7 @@
"@backstage/plugin-scaffolder-backend": "^{{version}}",
"@backstage/plugin-techdocs-backend": "^{{version}}",
"@octokit/rest": "^18.0.0",
"@gitbeaker/node": "^23.5.0",
"dockerode": "^3.2.0",
"express": "^4.17.1",
"knex": "^0.21.1",
packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts
+28 -4
View File
@@ -3,19 +3,23 @@ import {
createRouter,
FilePreparer,
GithubPreparer,
GitlabPreparer,
Preparers,
Publishers,
GithubPublisher,
GitlabPublisher,
CreateReactAppTemplater,
Templaters,
RepoVisilityOptions,
} from '@backstage/plugin-scaffolder-backend';
import { Octokit } from '@octokit/rest';
import { Gitlab } from '@gitbeaker/node';
import type { PluginEnvironment } from '../types';
import Docker from 'dockerode';
export default async function createPlugin({
logger,
config,
logger,
config,
}: PluginEnvironment) {
const cookiecutterTemplater = new CookieCutter();
const craTemplater = new CreateReactAppTemplater();
@@ -25,10 +29,15 @@ export default async function createPlugin({
const filePreparer = new FilePreparer();
const githubPreparer = new GithubPreparer();
const gitlabPreparer = new GitlabPreparer(config);
const preparers = new Preparers();
preparers.register('file', filePreparer);
preparers.register('github', githubPreparer);
preparers.register('gitlab', gitlabPreparer);
preparers.register('gitlab/api', gitlabPreparer);
const publishers = new Publishers();
const githubToken = config.getString('scaffolder.github.token');
const repoVisibility = config.getString(
@@ -36,17 +45,32 @@ export default async function createPlugin({
) as RepoVisilityOptions;
const githubClient = new Octokit({ auth: githubToken });
const publisher = new GithubPublisher({
const githubPublisher = new GithubPublisher({
client: githubClient,
token: githubToken,
repoVisibility,
});
publishers.register('file', githubPublisher);
publishers.register('github', githubPublisher);
const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab.api');
if (gitLabConfig) {
const gitLabToken = gitLabConfig.getString('token');
const gitLabClient = new Gitlab({
host: gitLabConfig.getOptionalString('baseUrl'),
token: gitLabToken,
});
const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
publishers.register('gitlab', gitLabPublisher);
publishers.register('gitlab/api', gitLabPublisher);
}
const dockerClient = new Docker();
return await createRouter({
preparers,
templaters,
publisher,
publishers,
logger,
dockerClient,
});
plugins/scaffolder-backend/package.json
+3 -1
View File
@@ -23,6 +23,8 @@
"@backstage/backend-common": "^0.1.1-alpha.21",
"@backstage/catalog-model": "^0.1.1-alpha.21",
"@backstage/config": "^0.1.1-alpha.21",
"@gitbeaker/core": "^23.5.0",
"@gitbeaker/node": "^23.5.0",
"@octokit/rest": "^18.0.0",
"@types/dockerode": "^2.5.32",
"@types/express": "^4.17.6",
@@ -33,7 +35,7 @@
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"fs-extra": "^9.0.0",
"git-url-parse": "^11.1.2",
"git-url-parse": "^11.2.0",
"globby": "^11.0.0",
"helmet": "^4.0.0",
"jsonschema": "^1.2.6",
plugins/scaffolder-backend/src/scaffolder/__mocks__/@gitbeaker/node/index.ts
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const mockGitlabClient = {
Namespaces: {
show: jest.fn(),
},
Projects: {
create: jest.fn(),
},
Users: {
current: jest.fn(),
},
};
export class Gitlab {
constructor() {
return mockGitlabClient;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.test.ts → plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts
View File
plugins/scaffolder-backend/src/scaffolder/stages/prepare/helpers.ts → plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts
View File
plugins/scaffolder-backend/src/scaffolder/stages/index.ts
+1
View File
@@ -16,3 +16,4 @@
export * from './prepare';
export * from './publish';
export * from './templater';
export * from './helpers';
plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts
+1 -1
View File
@@ -17,7 +17,7 @@ import fs from 'fs-extra';
import path from 'path';
import os from 'os';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from './helpers';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts
+1 -1
View File
@@ -17,7 +17,7 @@ import fs from 'fs-extra';
import path from 'path';
import os from 'os';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from './helpers';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
import GitUriParser from 'git-url-parse';
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts
+143
View File
@@ -0,0 +1,143 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const mocks = {
Clone: { clone: jest.fn() },
CheckoutOptions: jest.fn(() => {}),
};
jest.doMock('nodegit', () => mocks);
import { GitlabPreparer } from './gitlab';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
const mockEntityWithProtocol = (protocol: string): TemplateEntityV1alpha1 => ({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]: `${protocol}:https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/template.yaml`,
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
});
describe('GitLabPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
beforeEach(() => {
jest.clearAllMocks();
});
['gitlab', 'gitlab/api'].forEach(protocol => {
it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
mockEntity = mockEntityWithProtocol(protocol);
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://gitlab.com/benjdlambert/backstage-graphql-template',
expect.any(String),
{},
);
});
it(`calls the clone command with the correct arguments if an access token is provided for a repository using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(
ConfigReader.fromConfigs([
{
context: '',
data: {
catalog: {
processors: {
gitlabApi: {
privateToken: 'fake-token',
},
},
},
},
},
]),
);
mockEntity = mockEntityWithProtocol(protocol);
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://gitlab.com/benjdlambert/backstage-graphql-template',
expect.any(String),
{
fetchOpts: {
callbacks: {
credentials: expect.anything(),
},
},
},
);
});
it(`calls the clone command with the correct arguments for a repository when no path is provided using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
mockEntity = mockEntityWithProtocol(protocol);
delete mockEntity.spec.path;
await preparer.prepare(mockEntity);
expect(mocks.Clone.clone).toHaveBeenNthCalledWith(
1,
'https://gitlab.com/benjdlambert/backstage-graphql-template',
expect.any(String),
{},
);
});
it(`return the temp directory with the path to the folder if it is specified using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(ConfigReader.fromConfigs([]));
mockEntity = mockEntityWithProtocol(protocol);
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity);
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts
+73
View File
@@ -0,0 +1,73 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import path from 'path';
import os from 'os';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase } from './types';
import GitUriParser from 'git-url-parse';
import { Clone, Cred } from 'nodegit';
import { Config } from '@backstage/config';
export class GitlabPreparer implements PreparerBase {
private readonly privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
'';
}
async prepare(template: TemplateEntityV1alpha1): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
if (['gitlab', 'gitlab/api'].indexOf(protocol) < 0) {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'gitlab' or 'gitlab/api'`,
);
}
const templateId = template.metadata.name;
const parsedGitLocation = GitUriParser(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const options = this.privateToken
? {
fetchOpts: {
callbacks: {
credentials: () =>
Cred.userpassPlaintextNew('oauth2', this.privateToken),
},
},
}
: {};
await Clone.clone(repositoryCheckoutUrl, tempDir, options);
return path.resolve(tempDir, templateDirectory);
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/index.ts
+1 -1
View File
@@ -15,6 +15,6 @@
*/
export * from './preparers';
export * from './types';
export * from './helpers';
export * from './file';
export * from './github';
export * from './gitlab';
plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts
+3 -2
View File
@@ -14,9 +14,10 @@
* limitations under the License.
*/
import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types';
import { PreparerBase, PreparerBuilder } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from './helpers';
import { parseLocationAnnotation } from '../helpers';
import { RemoteProtocol } from '../types';
export class Preparers implements PreparerBuilder {
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts
+1 -2
View File
@@ -15,6 +15,7 @@
*/
import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
import { RemoteProtocol } from '../types';
export type PreparerBase = {
/**
@@ -32,5 +33,3 @@ export type PreparerBuilder = {
register(protocol: RemoteProtocol, preparer: PreparerBase): void;
get(template: TemplateEntityV1alpha1): PreparerBase;
};
export type RemoteProtocol = 'file' | 'github';
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts
+205
View File
@@ -0,0 +1,205 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('nodegit');
jest.mock('@gitbeaker/node');
import { GitlabPublisher } from './gitlab';
import { Gitlab as GitlabAPI } from '@gitbeaker/core';
import { Gitlab } from '@gitbeaker/node';
import * as NodeGit from 'nodegit';
const { mockGitlabClient } = require('@gitbeaker/node') as {
mockGitlabClient: {
Namespaces: jest.Mocked<GitlabAPI['Namespaces']>;
Projects: jest.Mocked<GitlabAPI['Projects']>;
Users: jest.Mocked<GitlabAPI['Users']>;
};
};
const {
Repository,
mockRepo,
mockIndex,
Signature,
Remote,
mockRemote,
Cred,
} = require('nodegit') as {
Repository: jest.Mocked<{ init: any }>;
Signature: jest.Mocked<{ now: any }>;
Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
Remote: jest.Mocked<{ create: any }>;
mockIndex: jest.Mocked<NodeGit.Index>;
mockRepo: jest.Mocked<NodeGit.Repository>;
mockRemote: jest.Mocked<NodeGit.Remote>;
};
describe('GitLab Publisher', () => {
const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token');
beforeEach(() => {
jest.clearAllMocks();
});
describe('publish: createRemoteInGitLab', () => {
it('should use gitbeaker to create a repo in a namespace if the namespace property is set', async () => {
mockGitlabClient.Namespaces.show.mockResolvedValue({
id: 42,
} as { id: number });
await publisher.publish({
values: {
isOrg: true,
storePath: 'blam/test',
owner: 'bob',
},
directory: '/tmp/test',
});
expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
namespace_id: 42,
name: 'test',
});
});
it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => {
mockGitlabClient.Namespaces.show.mockResolvedValue({});
mockGitlabClient.Users.current.mockResolvedValue({
id: 21,
} as { id: number });
mockGitlabClient.Projects.create.mockResolvedValue({
http_url_to_repo: 'mockclone',
} as { http_url_to_repo: string });
await publisher.publish({
values: {
storePath: 'blam/test',
owner: 'bob',
},
directory: '/tmp/test',
});
expect(mockGitlabClient.Users.current).toHaveBeenCalled();
expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
namespace_id: 21,
name: 'test',
});
});
});
describe('publish: createGitDirectory', () => {
const values = {
isOrg: true,
storePath: 'blam/test',
owner: 'lols',
};
const mockDir = '/tmp/test/dir';
mockGitlabClient.Projects.create.mockResolvedValue({
http_url_to_repo: 'mockclone',
} as { http_url_to_repo: string });
it('should call init on the repo with the directory', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
});
it('should call refresh index on the index and write the new files', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(mockRepo.refreshIndex).toHaveBeenCalled();
});
it('should call add all files and write', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(mockIndex.addAll).toHaveBeenCalled();
expect(mockIndex.write).toHaveBeenCalled();
expect(mockIndex.writeTree).toHaveBeenCalled();
});
it('should create a commit with on head with the right name and commiter', async () => {
const mockSignature = { mockSignature: 'bloblly' };
Signature.now.mockReturnValue(mockSignature);
await publisher.publish({
values,
directory: mockDir,
});
expect(Signature.now).toHaveBeenCalledTimes(2);
expect(Signature.now).toHaveBeenCalledWith(
'Scaffolder',
'scaffolder@backstage.io',
);
expect(mockRepo.createCommit).toHaveBeenCalledWith(
'HEAD',
mockSignature,
mockSignature,
'initial commit',
'mockoid',
[],
);
});
it('creates a remote with the repo and remote', async () => {
await publisher.publish({
values,
directory: mockDir,
});
expect(Remote.create).toHaveBeenCalledWith(
mockRepo,
'origin',
'mockclone',
);
});
it('shoud push to the remote repo', async () => {
await publisher.publish({
values,
directory: mockDir,
});
const [remotes, { callbacks }] = mockRemote.push.mock
.calls[0] as NodeGit.PushOptions[];
expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
callbacks?.credentials?.();
expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
'oauth2',
'fake-token',
);
});
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts
+90
View File
@@ -0,0 +1,90 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { PublisherBase } from './types';
import { Gitlab } from '@gitbeaker/core';
import { JsonValue } from '@backstage/config';
import { RequiredTemplateValues } from '../templater';
import { Repository, Remote, Signature, Cred } from 'nodegit';
export class GitlabPublisher implements PublisherBase {
private readonly client: Gitlab;
private readonly token: string;
constructor(client: Gitlab, token: string) {
this.client = client;
this.token = token;
}
async publish({
values,
directory,
}: {
values: RequiredTemplateValues & Record<string, JsonValue>;
directory: string;
}): Promise<{ remoteUrl: string }> {
const remoteUrl = await this.createRemote(values);
await this.pushToRemote(directory, remoteUrl);
return { remoteUrl };
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
) {
const [owner, name] = values.storePath.split('/');
let targetNamespace = ((await this.client.Namespaces.show(owner)) as {
id: number;
}).id;
if (!targetNamespace) {
targetNamespace = ((await this.client.Users.current()) as { id: number })
.id;
}
const project = (await this.client.Projects.create({
namespace_id: targetNamespace,
name: name,
})) as { http_url_to_repo: string };
return project?.http_url_to_repo;
}
private async pushToRemote(directory: string, remote: string): Promise<void> {
const repo = await Repository.init(directory, 0);
const index = await repo.refreshIndex();
await index.addAll();
await index.write();
const oid = await index.writeTree();
await repo.createCommit(
'HEAD',
Signature.now('Scaffolder', 'scaffolder@backstage.io'),
Signature.now('Scaffolder', 'scaffolder@backstage.io'),
'initial commit',
oid,
[],
);
const remoteRepo = await Remote.create(repo, 'origin', remote);
await remoteRepo.push(['refs/heads/master:refs/heads/master'], {
callbacks: {
credentials: () => Cred.userpassPlaintextNew('oauth2', this.token),
},
});
}
}
plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts
+2
View File
@@ -13,5 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './publishers';
export * from './github';
export * from './gitlab';
export * from './types';
plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Publishers } from './publishers';
import {
LOCATION_ANNOTATION,
TemplateEntityV1alpha1,
} from '@backstage/catalog-model';
import { GithubPublisher } from './github';
import { Octokit } from '@octokit/rest';
jest.mock('@octokit/rest');
describe('Publishers', () => {
const mockTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
it('should throw an error when the publisher for the source location is not registered', () => {
const publishers = new Publishers();
expect(() => publishers.get(mockTemplate)).toThrow(
expect.objectContaining({
message: 'No publisher registered for type: "github"',
}),
);
});
it('should return the correct preparer when the source matches', () => {
const publishers = new Publishers();
const publisher = new GithubPublisher({
client: new Octokit(),
token: 'fake',
repoVisibility: 'public',
});
publishers.register('github', publisher);
expect(publishers.get(mockTemplate)).toBe(publisher);
});
it('should throw an error if the metadata tag does not exist in the entity', () => {
const brokenTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: '.',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
const publishers = new Publishers();
expect(() => publishers.get(brokenTemplate)).toThrow(
expect.objectContaining({
message: expect.stringContaining('No location annotation provided'),
}),
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { PublisherBase, PublisherBuilder } from './types';
import { RemoteProtocol } from '../types';
export class Publishers implements PublisherBuilder {
private publisherMap = new Map<RemoteProtocol, PublisherBase>();
register(protocol: RemoteProtocol, publisher: PublisherBase) {
this.publisherMap.set(protocol, publisher);
}
get(template: TemplateEntityV1alpha1): PublisherBase {
const { protocol } = parseLocationAnnotation(template);
const publisher = this.publisherMap.get(protocol);
if (!publisher) {
throw new Error(`No publisher registered for type: "${protocol}"`);
}
return publisher;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts
+6
View File
@@ -16,6 +16,7 @@
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { RequiredTemplateValues } from '../templater';
import { JsonValue } from '@backstage/config';
import { RemoteProtocol } from '../types';
/**
* Publisher is in charge of taking a folder created by
@@ -34,3 +35,8 @@ export type PublisherBase = {
directory: string;
}): Promise<{ remoteUrl: string }>;
};
export type PublisherBuilder = {
register(protocol: RemoteProtocol, publisher: PublisherBase): void;
get(template: TemplateEntityV1alpha1): PublisherBase;
};
plugins/scaffolder-backend/src/scaffolder/stages/types.ts
+16
View File
@@ -0,0 +1,16 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type RemoteProtocol = 'file' | 'github' | 'gitlab' | 'gitlab/api';
plugins/scaffolder-backend/src/service/router.test.ts
+2 -3
View File
@@ -18,21 +18,20 @@ import { getVoidLogger } from '@backstage/backend-common';
import express from 'express';
import request from 'supertest';
import { createRouter } from './router';
import { Templaters, Preparers, PublisherBase } from '../scaffolder';
import { Templaters, Preparers, Publishers } from '../scaffolder';
import Docker from 'dockerode';
jest.mock('dockerode');
describe('createRouter', () => {
let app: express.Express;
const publisher: jest.Mocked<PublisherBase> = { publish: jest.fn() };
beforeAll(async () => {
const router = await createRouter({
logger: getVoidLogger(),
preparers: new Preparers(),
templaters: new Templaters(),
publisher: publisher,
publishers: new Publishers(),
dockerClient: new Docker(),
});
app = express().use(router);
plugins/scaffolder-backend/src/service/router.ts
+4 -3
View File
@@ -26,14 +26,14 @@ import {
RequiredTemplateValues,
StageContext,
TemplaterBuilder,
PublisherBase,
PublisherBuilder,
} from '../scaffolder';
import { validate, ValidatorResult } from 'jsonschema';
export interface RouterOptions {
preparers: PreparerBuilder;
templaters: TemplaterBuilder;
publisher: PublisherBase;
publishers: PublisherBuilder;
logger: Logger;
dockerClient: Docker;
@@ -48,7 +48,7 @@ export async function createRouter(
const {
preparers,
templaters,
publisher,
publishers,
logger: parentLogger,
dockerClient,
} = options;
@@ -125,6 +125,7 @@ export async function createRouter(
{
name: 'Publish template',
handler: async (ctx: StageContext<{ resultDir: string }>) => {
const publisher = publishers.get(ctx.entity);
ctx.logger.info('Will now store the template');
const { remoteUrl } = await publisher.publish({
entity: ctx.entity,
plugins/techdocs-backend/package.json
+1 -1
View File
@@ -30,7 +30,7 @@
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"fs-extra": "^9.0.1",
"git-url-parse": "^11.1.3",
"git-url-parse": "^11.2.0",
"knex": "^0.21.1",
"node-fetch": "^2.6.0",
"nodegit": "^0.27.0",
yarn.lock
+82 -1
View File
@@ -1400,6 +1400,34 @@
dependencies:
yaml-ast-parser "0.0.43"
"@gitbeaker/core@^23.5.0":
version "23.5.0"
resolved "https://registry.npmjs.org/@gitbeaker/core/-/core-23.5.0.tgz#e0be44eecd0d7bf5418c161997c36d45b0c1ba81"
integrity sha512-5geLk7SxAttABBJZxfuSp72lSYWRyRId583vGCpKB6ISkoDhP+vJxdE6ypmtOJPV4CaSRZhluAGAZd968pi9ng==
dependencies:
"@gitbeaker/requester-utils" "^23.5.0"
form-data "^3.0.0"
li "^1.3.0"
xcase "^2.0.1"
"@gitbeaker/node@^23.5.0":
version "23.5.0"
resolved "https://registry.npmjs.org/@gitbeaker/node/-/node-23.5.0.tgz#0243be78aad148e7d6b5d79b7acc340748b5fa94"
integrity sha512-GEyhcrF1Lm8YmsmwntfzuhXnq00TrG14wNZP2Hg+DGgexG25eBbbcyFXuFZlBFSaGMQlsc8aPeuEyfyGmgpwnw==
dependencies:
"@gitbeaker/core" "^23.5.0"
"@gitbeaker/requester-utils" "^23.5.0"
got "^11.1.4"
xcase "^2.0.1"
"@gitbeaker/requester-utils@^23.5.0":
version "23.5.0"
resolved "https://registry.npmjs.org/@gitbeaker/requester-utils/-/requester-utils-23.5.0.tgz#e6f5d0216cb978be95e6bf40adf606f23d426f14"
integrity sha512-MdmInOO4unkApvtbv6PnIpDYXosgZgClSOqbxF5S4aJRCZVTJ6oPjMoNP8luhyT9xQeknpKxn9Iv8psEh7IC1Q==
dependencies:
query-string "^6.12.1"
xcase "^2.0.1"
"@graphql-tools/delegate@6.0.15":
version "6.0.15"
resolved "https://registry.npmjs.org/@graphql-tools/delegate/-/delegate-6.0.15.tgz#9e060bfc31fe7735bd5b2b401e98dea3fa5d3b25"
@@ -10695,13 +10723,20 @@ git-up@^4.0.0:
is-ssh "^1.3.0"
parse-url "^5.0.0"
git-url-parse@^11.1.2, git-url-parse@^11.1.3:
git-url-parse@^11.1.2:
version "11.1.3"
resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.1.3.tgz#03625b6fc09905e9ad1da7bb2b84be1bf9123143"
integrity sha512-GPsfwticcu52WQ+eHp0IYkAyaOASgYdtsQDIt4rUp6GbiNt1P9ddrh3O0kQB0eD4UJZszVqNT3+9Zwcg40fywA==
dependencies:
git-up "^4.0.0"
git-url-parse@^11.2.0:
version "11.2.0"
resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.2.0.tgz#2955fd51befd6d96ea1389bbe2ef57e8e6042b04"
integrity sha512-KPoHZg8v+plarZvto4ruIzzJLFQoRx+sUs5DQSr07By9IBKguVd+e6jwrFR6/TP6xrCJlNV1tPqLO1aREc7O2g==
dependencies:
git-up "^4.0.0"
gitconfiglocal@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b"
@@ -10944,6 +10979,23 @@ got@^10.7.0:
to-readable-stream "^2.0.0"
type-fest "^0.10.0"
got@^11.1.4:
version "11.6.2"
resolved "https://registry.npmjs.org/got/-/got-11.6.2.tgz#79d7bb8c11df212b97f25565407a1f4ae73210ec"
integrity sha512-/21qgUePCeus29Jk7MEti8cgQUNXFSWfIevNIk4H7u1wmXNDrGPKPY6YsPY+o9CIT/a2DjCjRz0x1nM9FtS2/A==
dependencies:
"@sindresorhus/is" "^3.1.1"
"@szmarczak/http-timer" "^4.0.5"
"@types/cacheable-request" "^6.0.1"
"@types/responselike" "^1.0.0"
cacheable-lookup "^5.0.3"
cacheable-request "^7.0.1"
decompress-response "^6.0.0"
http2-wrapper "^1.0.0-beta.5.2"
lowercase-keys "^2.0.0"
p-cancelable "^2.0.0"
responselike "^2.0.0"
got@^11.5.2:
version "11.6.0"
resolved "https://registry.npmjs.org/got/-/got-11.6.0.tgz#4978c78f3cbc3a45ee95381f8bb6efd1db1f4638"
@@ -13589,6 +13641,11 @@ levn@~0.3.0:
prelude-ls "~1.1.2"
type-check "~0.3.2"
li@^1.3.0:
version "1.3.0"
resolved "https://registry.npmjs.org/li/-/li-1.3.0.tgz#22c59bcaefaa9a8ef359cf759784e4bf106aea1b"
integrity sha1-IsWbyu+qmo7zWc91l4TkvxBq6hs=
liftoff@3.1.0:
version "3.1.0"
resolved "https://registry.npmjs.org/liftoff/-/liftoff-3.1.0.tgz#c9ba6081f908670607ee79062d700df062c52ed3"
@@ -17150,6 +17207,15 @@ query-string@^4.1.0:
object-assign "^4.1.0"
strict-uri-encode "^1.0.0"
query-string@^6.12.1:
version "6.13.1"
resolved "https://registry.npmjs.org/query-string/-/query-string-6.13.1.tgz#d913ccfce3b4b3a713989fe6d39466d92e71ccad"
integrity sha512-RfoButmcK+yCta1+FuU8REvisx1oEzhMKwhLUNcepQTPGcNMp1sIqjnfCtfnvGSQZQEhaBHvccujtWoUV3TTbA==
dependencies:
decode-uri-component "^0.2.0"
split-on-first "^1.0.0"
strict-uri-encode "^2.0.0"
querystring-browser@^1.0.4:
version "1.0.4"
resolved "https://registry.npmjs.org/querystring-browser/-/querystring-browser-1.0.4.tgz#f2e35881840a819bc7b1bf597faf0979e6622dc6"
@@ -19278,6 +19344,11 @@ split-ca@^1.0.1:
resolved "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6"
integrity sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=
split-on-first@^1.0.0:
version "1.1.0"
resolved "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f"
integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==
split-string@^3.0.1, split-string@^3.0.2:
version "3.1.0"
resolved "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2"
@@ -19545,6 +19616,11 @@ strict-uri-encode@^1.0.0:
resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
integrity sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=
strict-uri-encode@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"
integrity sha1-ucczDHBChi9rFC3CdLvMWGbONUY=
string-argv@0.3.1:
version "0.3.1"
resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
@@ -21788,6 +21864,11 @@ x-is-string@^0.1.0:
resolved "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz#474b50865af3a49a9c4657f05acd145458f77d82"
integrity sha1-R0tQhlrzpJqcRlfwWs0UVFj3fYI=
xcase@^2.0.1:
version "2.0.1"
resolved "https://registry.npmjs.org/xcase/-/xcase-2.0.1.tgz#c7fa72caa0f440db78fd5673432038ac984450b9"
integrity sha1-x/pyyqD0QNt4/VZzQyA4rJhEULk=
xdg-basedir@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13"