From a33996edcc1cea29337c6d64473d4fdd471219f1 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Mon, 18 Jan 2021 16:41:01 +0100 Subject: [PATCH] Scaffolder: Refactor publish and prepare Co-authored-by: blam --- .../src/scaffolder/stages/helpers.test.ts | 11 +- .../src/scaffolder/stages/helpers.ts | 5 +- .../scaffolder/stages/prepare/azure.test.ts | 15 +- .../src/scaffolder/stages/prepare/azure.ts | 2 +- .../stages/prepare/bitbucket.test.ts | 17 +- .../scaffolder/stages/prepare/github.test.ts | 13 +- .../scaffolder/stages/prepare/gitlab.test.ts | 15 +- .../scaffolder/stages/prepare/preparers.ts | 3 +- .../scaffolder/stages/publish/azure.test.ts | 102 +++++++++-- .../src/scaffolder/stages/publish/azure.ts | 1 + .../stages/publish/bitbucket.test.ts | 41 ++++- .../scaffolder/stages/publish/bitbucket.ts | 78 ++++++-- .../scaffolder/stages/publish/github.test.ts | 84 ++++++--- .../scaffolder/stages/publish/gitlab.test.ts | 73 ++++++-- .../stages/publish/publishers.test.ts | 167 ++++++++---------- .../scaffolder/stages/publish/publishers.ts | 86 ++------- .../src/scaffolder/stages/types.ts | 3 +- 17 files changed, 425 insertions(+), 291 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts index 09bd6b7885..d02a0a5ee6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts @@ -30,10 +30,7 @@ describe('Helpers', () => { apiVersion: 'backstage.io/v1alpha1', kind: 'Template', metadata: { - annotations: { - // [LOCATION_ANNOTATION]: - // 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', - }, + annotations: {}, name: 'graphql-starter', title: 'GraphQL Service', description: @@ -275,11 +272,11 @@ describe('Helpers', () => { expect(detector('http://derp.org:80/wat')).toBe('gitlab'); expect(detector('https://foo.org/wat')).toBe('gitlab'); expect(detector('http://not.derp.net')).toBe(undefined); - expect(detector('http://derp.net')).toBe('azure/api'); - expect(detector('http://derp.net:8080/wat')).toBe('azure/api'); + expect(detector('http://derp.net')).toBe('azure'); + expect(detector('http://derp.net:8080/wat')).toBe('azure'); expect(detector('http://github.com')).toBe('github'); expect(detector('http://gitlab.com')).toBe('gitlab'); - expect(detector('http://dev.azure.com')).toBe('azure/api'); + expect(detector('http://dev.azure.com')).toBe('azure'); }); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts index 21b63609e8..c3ef01b359 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts @@ -74,7 +74,8 @@ export function makeDeprecatedLocationTypeDetector( // These are installed by default by the integrations hostMap.set('github.com', 'github'); hostMap.set('gitlab.com', 'gitlab'); - hostMap.set('dev.azure.com', 'azure/api'); + hostMap.set('dev.azure.com', 'azure'); + hostMap.set('bitbucket.org', 'bitbucket'); config.getOptionalConfigArray('integrations.github')?.forEach(sub => { hostMap.set(sub.getString('host'), 'github'); @@ -83,7 +84,7 @@ export function makeDeprecatedLocationTypeDetector( hostMap.set(sub.getString('host'), 'gitlab'); }); config.getOptionalConfigArray('integrations.azure')?.forEach(sub => { - hostMap.set(sub.getString('host'), 'azure/api'); + hostMap.set(sub.getString('host'), 'azure'); }); config.getOptionalConfigArray('integrations.bitbucket')?.forEach(sub => { hostMap.set(sub.getString('host'), 'bitbucket'); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts index 38c262eaaa..896d91f4ed 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts @@ -46,7 +46,7 @@ describe('AzurePreparer', () => { metadata: { annotations: { [LOCATION_ANNOTATION]: - 'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml', + 'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml', }, name: 'graphql-starter', title: 'GraphQL Service', @@ -95,7 +95,7 @@ describe('AzurePreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -116,7 +116,7 @@ describe('AzurePreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -128,7 +128,7 @@ describe('AzurePreparer', () => { it('calls the clone command with the correct arguments for a repository', async () => { const preparer = new AzurePreparer(new ConfigReader({}), { logger }); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: @@ -141,7 +141,7 @@ describe('AzurePreparer', () => { const preparer = new AzurePreparer(new ConfigReader({}), { logger }); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: @@ -154,7 +154,9 @@ describe('AzurePreparer', () => { const preparer = new AzurePreparer(new ConfigReader({}), { logger }); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + }); expect(response.split('\\').join('/')).toMatch( /\/template\/test\/1\/2\/3$/, @@ -167,6 +169,7 @@ describe('AzurePreparer', () => { const response = await preparer.prepare(mockEntity, { workingDirectory: '/workDir', + logger: getVoidLogger(), }); expect(response.split('\\').join('/')).toMatch( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index b2eab7bb85..35fa02e8ce 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -62,7 +62,7 @@ export class AzurePreparer implements PreparerBase { const workingDirectory = opts.workingDirectory ?? os.tmpdir(); const logger = opts.logger; - if (!['azure/api', 'url'].includes(protocol)) { + if (!['azure', 'url'].includes(protocol)) { throw new InputError( `Wrong location protocol: ${protocol}, should be 'url'`, ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts index 0be61a9f82..d55ca7a01d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts @@ -81,7 +81,7 @@ describe('BitbucketPreparer', () => { it('calls the clone command with the correct arguments for a repository', async () => { const preparer = new BitbucketPreparer(new ConfigReader({}), { logger }); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://bitbucket.org/backstage-project/backstage-repo', dir: expect.any(String), @@ -104,7 +104,7 @@ describe('BitbucketPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -116,7 +116,7 @@ describe('BitbucketPreparer', () => { it('calls the clone command with the correct arguments for a repository when no path is provided', async () => { const preparer = new BitbucketPreparer(new ConfigReader({}), { logger }); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://bitbucket.org/backstage-project/backstage-repo', dir: expect.any(String), @@ -126,7 +126,9 @@ describe('BitbucketPreparer', () => { it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new BitbucketPreparer(new ConfigReader({}), { logger }); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + }); expect(response.split('\\').join('/')).toMatch( /\/template\/test\/1\/2\/3$/, @@ -148,7 +150,7 @@ describe('BitbucketPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -173,7 +175,7 @@ describe('BitbucketPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -198,7 +200,7 @@ describe('BitbucketPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -212,6 +214,7 @@ describe('BitbucketPreparer', () => { mockEntity.spec.path = './template/test/1/2/3'; const response = await preparer.prepare(mockEntity, { workingDirectory: '/workDir', + logger: getVoidLogger(), }); expect(response.split('\\').join('/')).toMatch( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 6598a8734a..122a1abb4c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -90,7 +90,7 @@ describe('GitHubPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://github.com/benjdlambert/backstage-graphql-template', @@ -101,7 +101,7 @@ describe('GitHubPreparer', () => { const preparer = new GithubPreparer(new ConfigReader({}), { logger }); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://github.com/benjdlambert/backstage-graphql-template', @@ -112,7 +112,9 @@ describe('GitHubPreparer', () => { it('return the temp directory with the path to the folder if it is specified', async () => { const preparer = new GithubPreparer(new ConfigReader({}), { logger }); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + }); expect(response.split('\\').join('/')).toMatch( /\/template\/test\/1\/2\/3$/, ); @@ -123,6 +125,7 @@ describe('GitHubPreparer', () => { mockEntity.spec.path = './template/test/1/2/3'; const response = await preparer.prepare(mockEntity, { workingDirectory: '/workDir', + logger: getVoidLogger(), }); expect(response.split('\\').join('/')).toMatch( @@ -142,7 +145,7 @@ describe('GitHubPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -161,7 +164,7 @@ describe('GitHubPreparer', () => { { logger }, ); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index 8cb54ceb7a..c1e42920ae 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -78,12 +78,12 @@ describe('GitLabPreparer', () => { jest.clearAllMocks(); }); - ['gitlab', 'gitlab/api'].forEach(protocol => { + ['gitlab'].forEach(protocol => { it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => { const preparer = new GitlabPreparer(new ConfigReader({}), { logger }); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', @@ -107,7 +107,7 @@ describe('GitLabPreparer', () => { ); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -127,7 +127,7 @@ describe('GitLabPreparer', () => { ); mockEntity = mockEntityWithProtocol(protocol); - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger }); expect(Git.fromAuth).toHaveBeenCalledWith({ logger, @@ -141,7 +141,7 @@ describe('GitLabPreparer', () => { mockEntity = mockEntityWithProtocol(protocol); delete mockEntity.spec.path; - await preparer.prepare(mockEntity); + await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', @@ -153,7 +153,9 @@ describe('GitLabPreparer', () => { const preparer = new GitlabPreparer(new ConfigReader({}), { logger }); mockEntity = mockEntityWithProtocol(protocol); mockEntity.spec.path = './template/test/1/2/3'; - const response = await preparer.prepare(mockEntity); + const response = await preparer.prepare(mockEntity, { + logger: getVoidLogger(), + }); expect(response.split('\\').join('/')).toMatch( /\/template\/test\/1\/2\/3$/, ); @@ -164,6 +166,7 @@ describe('GitLabPreparer', () => { mockEntity.spec.path = './template/test/1/2/3'; const response = await preparer.prepare(mockEntity, { workingDirectory: '/workDir', + logger: getVoidLogger(), }); expect(response.split('\\').join('/')).toMatch( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts index ff23aa2990..251e8b0ae3 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts @@ -85,8 +85,7 @@ export class Preparers implements PreparerBuilder { preparers.register('file', filePreparer); preparers.register('gitlab', gitlabPreparer); - preparers.register('gitlab/api', gitlabPreparer); - preparers.register('azure/api', azurePreparer); + preparers.register('azure', azurePreparer); preparers.register('github', githubPreparer); preparers.register('bitbucket', bitbucketPreparer); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts index f1e0cd6175..9aaee4bcb8 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -15,28 +15,45 @@ */ jest.mock('./helpers'); +jest.mock('azure-devops-node-api', () => ({ + WebApi: jest.fn(), + getPersonalAccessTokenHandler: jest.fn(), +})); + import { AzurePublisher } from './azure'; -import { GitApi } from 'azure-devops-node-api/GitApi'; +import { WebApi } from 'azure-devops-node-api'; import * as helpers from './helpers'; import { getVoidLogger } from '@backstage/backend-common'; - -const { mockGitApi } = require('azure-devops-node-api/GitApi') as { - mockGitApi: { - createRepository: jest.MockedFunction; - }; -}; +import { ConfigReader } from '@backstage/config'; describe('Azure Publisher', () => { - const publisher = new AzurePublisher(new GitApi('', []), 'fake-token'); const logger = getVoidLogger(); - beforeEach(() => { - jest.clearAllMocks(); - }); - describe('publish: createRemoteInAzure', () => { it('should use azure-devops-node-api to create a repo in the given project', async () => { - mockGitApi.createRepository.mockResolvedValue({ + const mockGitClient = { + createRepository: jest.fn(), + }; + const mockGitApi = { + getGitApi: jest.fn().mockReturnValue(mockGitClient), + }; + + ((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi); + + const publisher = new AzurePublisher( + new ConfigReader({ + scaffolder: { + azure: { + api: { + baseUrl: 'https://dev.azure.com/myorg', + token: 'fake-azure-token', + }, + }, + }, + }), + { logger }, + ); + mockGitClient.createRepository.mockResolvedValue({ remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', } as { remoteUrl: string }); @@ -54,7 +71,7 @@ describe('Azure Publisher', () => { catalogInfoUrl: 'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml', }); - expect(mockGitApi.createRepository).toHaveBeenCalledWith( + expect(mockGitClient.createRepository).toHaveBeenCalledWith( { name: 'repo', }, @@ -63,7 +80,62 @@ describe('Azure Publisher', () => { expect(helpers.initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', - auth: { username: 'notempty', password: 'fake-token' }, + auth: { username: 'notempty', password: 'fake-azure-token' }, + logger, + }); + }); + + it('should use azure-devops-node-api with integrations config', async () => { + const mockGitClient = { + createRepository: jest.fn(), + }; + const mockGitApi = { + getGitApi: jest.fn().mockReturnValue(mockGitClient), + }; + + ((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi); + + const publisher = new AzurePublisher( + new ConfigReader({ + integrations: { + azure: [ + { + host: 'dev.azure.com', + token: 'fake-azure-token', + }, + ], + }, + }), + { logger }, + ); + mockGitClient.createRepository.mockResolvedValue({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + } as { remoteUrl: string }); + + const result = await publisher.publish({ + values: { + storePath: 'https://dev.azure.com/organization/project/_git/repo', + owner: 'bob', + }, + directory: '/tmp/test', + logger, + }); + + expect(result).toEqual({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + catalogInfoUrl: + 'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml', + }); + expect(mockGitClient.createRepository).toHaveBeenCalledWith( + { + name: 'repo', + }, + 'project', + ); + expect(helpers.initRepoAndPush).toHaveBeenCalledWith({ + dir: '/tmp/test', + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + auth: { username: 'notempty', password: 'fake-azure-token' }, logger, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 50de3331d8..4172f28f6b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -83,6 +83,7 @@ export class AzurePublisher implements PublisherBase { project: owner, name, }); + const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; await initRepoAndPush({ diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts index 3b084f1bf9..6cfcc7d69e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts @@ -22,6 +22,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; import { msw } from '@backstage/test-utils'; +import { ConfigReader } from '@backstage/config'; describe('Bitbucket Publisher', () => { const logger = getVoidLogger(); @@ -59,14 +60,25 @@ describe('Bitbucket Publisher', () => { ); const publisher = new BitbucketPublisher( - 'https://bitbucket.org', - 'fake-user', - 'fake-token', + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.org', + username: 'fake-user', + appPassword: 'fake-token', + }, + ], + }, + }), + { + logger: getVoidLogger(), + }, ); const result = await publisher.publish({ values: { - storePath: 'project/repo', + storePath: 'https://bitbucket.org/project/repo', owner: 'bob', }, directory: '/tmp/test', @@ -87,6 +99,7 @@ describe('Bitbucket Publisher', () => { }); }); }); + describe('publish: createRemoteInBitbucketServer', () => { it('should create repo in bitbucket server', async () => { server.use( @@ -117,14 +130,24 @@ describe('Bitbucket Publisher', () => { ); const publisher = new BitbucketPublisher( - 'https://bitbucket.mycompany.com', - 'fake-user', - 'fake-token', + new ConfigReader({ + integrations: { + bitbucket: [ + { + host: 'bitbucket.mycompany.com', + token: 'fake-token', + }, + ], + }, + }), + { + logger: getVoidLogger(), + }, ); const result = await publisher.publish({ values: { - storePath: 'project/repo', + storePath: 'https://bitbucket.mycompany.com/project/repo', owner: 'bob', }, directory: '/tmp/test', @@ -140,7 +163,7 @@ describe('Bitbucket Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo', - auth: { username: 'fake-user', password: 'fake-token' }, + auth: { username: 'x-token-auth', password: 'fake-token' }, logger: logger, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts index b07e77e49a..4cc7654136 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts @@ -25,10 +25,14 @@ import { import { Logger } from 'winston'; import gitUrlParse from 'git-url-parse'; +// TODO(blam): We should probably start to use a bitbucket client here that we can change +// the baseURL to point at on-prem or public bitbucket versions like we do for +// github and ghe export class BitbucketPublisher implements PublisherBase { private readonly host?: string; private readonly username?: string; private readonly token?: string; + private readonly appPassword?: string; private readonly integrations: BitbucketIntegrationConfig[]; constructor(config: Config, { logger }: { logger: Logger }) { @@ -64,6 +68,15 @@ export class BitbucketPublisher implements PublisherBase { "DEPRECATION: Using the apiBaseUrl format under 'scaffolder.bitbucket.api.username' will not be respected in future releases. Please consider using integrations config instead", ); } + + this.appPassword = config.getOptionalString( + 'scaffolder.bitbucket.api.appPassword', + ); + if (this.appPassword) { + logger.warn( + "DEPRECATION: Using the appPassword format under 'scaffolder.bitbucket.api.appassword' will not be respected in future releases. Please consider using integrations config instead", + ); + } } async publish({ @@ -71,20 +84,20 @@ export class BitbucketPublisher implements PublisherBase { directory, logger, }: PublisherOptions): Promise { - const { resource: host, owner: project, name } = gitUrlParse( + const { resource: hostname, owner: project, name } = gitUrlParse( values.storePath, ); - const token = this.getToken(host); - if (!token) { - throw new Error('No token provided to create the remote repository'); + const token = this.getToken(hostname); + const appPassword = this.getAppPassword(hostname); + const username = this.getUsername(hostname); + + if (!username && !appPassword && !token) { + throw new Error('Cannot create repository without bitbucket credentials'); } - const username = this.getUsername(host); - if (!username) { - throw new Error('No username provided to create the remote repository'); - } - const apiUrl = this.getHost(host); - if (!apiUrl) { + const host = this.getHost(hostname); + + if (!host) { throw new Error('No host provided to create the remote repository'); } @@ -94,14 +107,17 @@ export class BitbucketPublisher implements PublisherBase { name, description, host, + username, + token, + appPassword, }); await initRepoAndPush({ dir: directory, remoteUrl: result.remoteUrl, auth: { - username: username, - password: token, + username: username ? username : 'x-token-auth', + password: appPassword ? appPassword : token ?? '', }, logger, }); @@ -109,12 +125,15 @@ export class BitbucketPublisher implements PublisherBase { } private async createRemote(opts: { + username?: string; + token?: string; + appPassword?: string; project: string; name: string; description: string; host: string; }): Promise { - if (opts.host === 'https://bitbucket.org') { + if (opts.host === 'bitbucket.org') { return this.createBitbucketCloudRepository(opts); } return this.createBitbucketServerRepository(opts); @@ -124,11 +143,22 @@ export class BitbucketPublisher implements PublisherBase { project: string; name: string; description: string; + username?: string; + appPassword?: string; }): Promise { - const { project, name, description } = opts; + const { project, name, description, username, appPassword } = opts; + if (!appPassword) { + throw new Error( + 'appPassword is required to create the remote repository', + ); + } + + if (!username) { + throw new Error('username is required to create the remote repository'); + } let response: Response; - const buffer = Buffer.from(`${this.username}:${this.token}`, 'utf8'); + const buffer = Buffer.from(`${username}:${appPassword}`, 'utf8'); const options: RequestInit = { method: 'POST', @@ -169,8 +199,13 @@ export class BitbucketPublisher implements PublisherBase { project: string; name: string; description: string; + token?: string; + host: string; }): Promise { - const { project, name, description } = opts; + const { project, name, description, token, host } = opts; + if (!token) { + throw new Error('No token provided to create the remote repository'); + } let response: Response; const options: RequestInit = { @@ -180,13 +215,13 @@ export class BitbucketPublisher implements PublisherBase { description: description, }), headers: { - Authorization: `Bearer ${this.token}`, + Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, }; try { response = await fetch( - `${this.host}/rest/api/1.0/projects/${project}/repos`, + `https://${host}/rest/api/1.0/projects/${project}/repos`, options, ); } catch (e) { @@ -216,6 +251,13 @@ export class BitbucketPublisher implements PublisherBase { ); } + private getAppPassword(host: string): string | undefined { + return ( + this.appPassword || + this.integrations.find(c => c.host === host)?.appPassword + ); + } + private getHost(host: string): string | undefined { return this.host || this.integrations.find(c => c.host === host)?.host; } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts index 22f1cd3172..effed4763d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -26,6 +26,7 @@ import { import { GithubPublisher } from './github'; import { initRepoAndPush } from './helpers'; import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; const { mockGithubClient } = require('@octokit/rest') as { mockGithubClient: { @@ -42,11 +43,21 @@ describe('GitHub Publisher', () => { }); describe('with public repo visibility', () => { - const publisher = new GithubPublisher({ - client: new Octokit(), - token: 'abc', - repoVisibility: 'public', - }); + const publisher = new GithubPublisher( + new ConfigReader({ + integrations: { + github: [ + { + token: 'fake-token', + host: 'github.com', + }, + ], + }, + }), + { + logger, + }, + ); describe('publish: createRemoteInGithub', () => { it('should use octokit to create a repo in an organisation if the organisation property is set', async () => { @@ -63,7 +74,7 @@ describe('GitHub Publisher', () => { const result = await publisher.publish({ values: { - storePath: 'blam/test', + storePath: 'https://github.com/blam/test', owner: 'bob', access: 'blam/team', }, @@ -94,7 +105,7 @@ describe('GitHub Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'abc', password: 'x-oauth-basic' }, + auth: { username: 'fake-token', password: 'x-oauth-basic' }, logger, }); }); @@ -113,7 +124,7 @@ describe('GitHub Publisher', () => { const result = await publisher.publish({ values: { - storePath: 'blam/test', + storePath: 'https://github.com/blam/test', owner: 'bob', access: 'blam', }, @@ -137,7 +148,7 @@ describe('GitHub Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'abc', password: 'x-oauth-basic' }, + auth: { username: 'fake-token', password: 'x-oauth-basic' }, logger, }); }); @@ -157,7 +168,7 @@ describe('GitHub Publisher', () => { const result = await publisher.publish({ values: { - storePath: 'blam/test', + storePath: 'https://github.com/blam/test', owner: 'bob', access: 'bob', description: 'description', @@ -187,18 +198,26 @@ describe('GitHub Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'abc', password: 'x-oauth-basic' }, + auth: { username: 'fake-token', password: 'x-oauth-basic' }, logger, }); }); }); describe('with internal repo visibility', () => { - const publisher = new GithubPublisher({ - client: new Octokit(), - token: 'abc', - repoVisibility: 'internal', - }); + const publisher = new GithubPublisher( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'fake-token' }], + }, + scaffolder: { + github: { + visibility: 'internal', + }, + }, + }), + { logger }, + ); it('creates a private repository in the organization with visibility set to internal', async () => { mockGithubClient.repos.createInOrg.mockResolvedValue({ @@ -215,7 +234,7 @@ describe('GitHub Publisher', () => { const result = await publisher.publish({ values: { isOrg: true, - storePath: 'blam/test', + storePath: 'https://github.com/blam/test', owner: 'bob', }, directory: '/tmp/test', @@ -236,18 +255,33 @@ describe('GitHub Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'abc', password: 'x-oauth-basic' }, + auth: { username: 'fake-token', password: 'x-oauth-basic' }, logger, }); }); }); describe('private visibility in a user account', () => { - const publisher = new GithubPublisher({ - client: new Octokit(), - token: 'abc', - repoVisibility: 'private', - }); + const publisher = new GithubPublisher( + new ConfigReader({ + integrations: { + github: [ + { + token: 'fake-token', + host: 'github.com', + }, + ], + }, + scaffolder: { + github: { + visibility: 'private', + }, + }, + }), + { + logger, + }, + ); it('creates a private repository', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ @@ -263,7 +297,7 @@ describe('GitHub Publisher', () => { const result = await publisher.publish({ values: { - storePath: 'blam/test', + storePath: 'https://github.com/blam/test', owner: 'bob', }, directory: '/tmp/test', @@ -284,7 +318,7 @@ describe('GitHub Publisher', () => { expect(initRepoAndPush).toHaveBeenCalledWith({ dir: '/tmp/test', remoteUrl: 'https://github.com/backstage/backstage.git', - auth: { username: 'abc', password: 'x-oauth-basic' }, + auth: { username: 'fake-token', password: 'x-oauth-basic' }, logger, }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts index 4894d99291..67887f579f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts @@ -14,33 +14,56 @@ * limitations under the License. */ -jest.mock('@gitbeaker/node'); +jest.mock('@gitbeaker/node', () => ({ + Gitlab: jest.fn(), +})); + jest.mock('./helpers'); import { GitlabPublisher } from './gitlab'; -import { Gitlab as GitlabAPI } from '@gitbeaker/core'; import { Gitlab } from '@gitbeaker/node'; import { initRepoAndPush } from './helpers'; import { getVoidLogger } from '@backstage/backend-common'; - -const { mockGitlabClient } = require('@gitbeaker/node') as { - mockGitlabClient: { - Namespaces: jest.Mocked; - Projects: jest.Mocked; - Users: jest.Mocked; - }; -}; +import { ConfigReader } from '@backstage/config'; describe('GitLab Publisher', () => { const logger = getVoidLogger(); - const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token'); + const mockGitlabClient = { + Namespaces: { + show: jest.fn(), + }, + Projects: { + create: jest.fn(), + }, + Users: { + current: jest.fn(), + }, + }; beforeEach(() => { jest.clearAllMocks(); + + ((Gitlab as unknown) as jest.Mock).mockImplementation( + () => mockGitlabClient, + ); }); describe('publish: createRemoteInGitLab', () => { it('should use gitbeaker to create a repo in a namespace if the namespace property is set', async () => { + const publisher = new GitlabPublisher( + new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + token: 'fake-token', + }, + ], + }, + }), + { logger }, + ); + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 42, } as { id: number }); @@ -51,14 +74,17 @@ describe('GitLab Publisher', () => { const result = await publisher.publish({ values: { isOrg: true, - storePath: 'bloum/blam/test', + storePath: 'https://gitlab.com/blam/test', owner: 'bob', }, directory: '/tmp/test', logger, }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'mockclone', + catalogInfoUrl: 'mockclone', + }); expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ namespace_id: 42, name: 'test', @@ -72,6 +98,20 @@ describe('GitLab Publisher', () => { }); it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => { + const publisher = new GitlabPublisher( + new ConfigReader({ + integrations: { + gitlab: [ + { + host: 'gitlab.com', + token: 'fake-token', + }, + ], + }, + }), + { logger }, + ); + mockGitlabClient.Namespaces.show.mockResolvedValue({}); mockGitlabClient.Users.current.mockResolvedValue({ id: 21, @@ -82,14 +122,17 @@ describe('GitLab Publisher', () => { const result = await publisher.publish({ values: { - storePath: 'bloum/blam/test', + storePath: 'https://gitlab.com/blam/test', owner: 'bob', }, directory: '/tmp/test', logger, }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'mockclone', + catalogInfoUrl: 'mockclone', + }); expect(mockGitlabClient.Users.current).toHaveBeenCalled(); expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ namespace_id: 21, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts index b8181b0134..fdbc283f67 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts @@ -14,120 +14,97 @@ * limitations under the License. */ import { Publishers } from './publishers'; -import { - LOCATION_ANNOTATION, - TemplateEntityV1alpha1, -} from '@backstage/catalog-model'; import { GithubPublisher } from './github'; -import { Octokit } from '@octokit/rest'; +import { getVoidLogger } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { AzurePublisher } from './azure'; +import { GitlabPublisher } from './gitlab'; +import { BitbucketPublisher } from './bitbucket'; jest.mock('@octokit/rest'); describe('Publishers', () => { - const mockTemplate: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - [LOCATION_ANNOTATION]: - 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: './template', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - }, - }; - it('should throw an error when the publisher for the source location is not registered', () => { const publishers = new Publishers(); - expect(() => publishers.get(mockTemplate)).toThrow( + expect(() => + publishers.get('https://github.com/org/repo', { + logger: getVoidLogger(), + }), + ).toThrow( expect.objectContaining({ - message: 'No publisher registered for type: "github"', + message: + 'No matching publisher detected for "https://github.com/org/repo". Please make sure this host is registered in the integration config', }), ); }); - it('should return the correct preparer when the source matches', () => { - const publishers = new Publishers(); - const publisher = new GithubPublisher({ - client: new Octokit(), - token: 'fake', - repoVisibility: 'public', + it('should return the correct preparer when the source matches for github', async () => { + const publishers = await Publishers.fromConfig(new ConfigReader({}), { + logger: getVoidLogger(), }); - publishers.register('github', publisher); - expect(publishers.get(mockTemplate)).toBe(publisher); + expect( + publishers.get('https://github.com/org/repo', { + logger: getVoidLogger(), + }), + ).toBeInstanceOf(GithubPublisher); }); - it('should throw an error if the metadata tag does not exist in the entity', () => { - const brokenTemplate: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: {}, - name: 'react-ssr-template', - title: 'React SSR Template', - description: - 'Next.js application skeleton for creating isomorphic web applications.', - uid: '7357f4c5-aa58-4a1e-9670-18931eef771f', - etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw', - generation: 1, - }, - spec: { - type: 'website', - templater: 'cookiecutter', - path: '.', - schema: { - $schema: 'http://json-schema.org/draft-07/schema#', - required: ['storePath', 'owner'], - properties: { - owner: { - type: 'string', - title: 'Owner', - description: 'Who is going to own this component', - }, - storePath: { - type: 'string', - title: 'Store path', - description: 'GitHub store path in org/repo format', - }, - }, - }, - }, - }; + it('should return the correct preparer when the source matches for azure', async () => { + const publishers = await Publishers.fromConfig(new ConfigReader({}), { + logger: getVoidLogger(), + }); - const publishers = new Publishers(); - - expect(() => publishers.get(brokenTemplate)).toThrow( - expect.objectContaining({ - message: expect.stringContaining('No location annotation provided'), + expect( + publishers.get('https://dev.azure.com/org/project/_git/repo', { + logger: getVoidLogger(), }), + ).toBeInstanceOf(AzurePublisher); + }); + + it('should return the correct preparer when the source matches for bitbucket', async () => { + const publishers = await Publishers.fromConfig(new ConfigReader({}), { + logger: getVoidLogger(), + }); + + expect( + publishers.get('https://bitbucket.org/owner/repo', { + logger: getVoidLogger(), + }), + ).toBeInstanceOf(BitbucketPublisher); + }); + + it('should return the correct preparer when the source matches for gitlab', async () => { + const publishers = await Publishers.fromConfig(new ConfigReader({}), { + logger: getVoidLogger(), + }); + + expect( + publishers.get('https://gitlab.com/owner/repo', { + logger: getVoidLogger(), + }), + ).toBeInstanceOf(GitlabPublisher); + }); + + it('should respect registrations for custom URLs for providers using the integrations config', async () => { + const publishers = await Publishers.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'my.special.github.enterprise.thing', token: 'lolghe' }, + ], + }, + }), + { + logger: getVoidLogger(), + }, ); + + expect( + publishers.get('https://my.special.github.enterprise.thing/org/repo', { + logger: getVoidLogger(), + }), + ).toBeInstanceOf(GithubPublisher); }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts index 78cc85a6c4..1fafc33c68 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts @@ -52,7 +52,7 @@ export class Publishers implements PublisherBuilder { const publisher = this.publisherMap.get(protocol as RemoteProtocol); if (!publisher) { throw new Error( - `Failed to detect publisher type. Unable to determine integration type for location "${location}". ` + + `Failed to detect publisher type. Unable to determine integration type for location "${protocol}". ` + "Please add appropriate configuration to the 'integrations' configuration section", ); } @@ -69,85 +69,19 @@ export class Publishers implements PublisherBuilder { const typeDetector = makeDeprecatedLocationTypeDetector(config); const publishers = new Publishers(typeDetector); - const githubConfig = config.getOptionalConfig('scaffolder.github'); - if (githubConfig) { - try { - const githubPublisher = new GithubPublisher(config, { logger }); - publishers.register('file', githubPublisher); - publishers.register('github', githubPublisher); - } catch (e) { - const providerName = 'github'; - if (process.env.NODE_ENV !== 'development') { - throw new Error( - `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, - ); - } + const githubPublisher = new GithubPublisher(config, { logger }); + publishers.register('file', githubPublisher); + publishers.register('github', githubPublisher); - logger.warn( - `Skipping ${providerName} scaffolding provider, ${e.message}`, - ); - } - } + const gitLabPublisher = new GitlabPublisher(config, { logger }); + publishers.register('gitlab', gitLabPublisher); - const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab'); - if (gitLabConfig) { - try { - const gitLabPublisher = new GitlabPublisher(config, { logger }); - publishers.register('gitlab', gitLabPublisher); - publishers.register('gitlab/api', gitLabPublisher); - } catch (e) { - const providerName = 'gitlab'; - if (process.env.NODE_ENV !== 'development') { - throw new Error( - `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, - ); - } + const azurePublisher = new AzurePublisher(config, { logger }); + publishers.register('azure', azurePublisher); - logger.warn( - `Skipping ${providerName} scaffolding provider, ${e.message}`, - ); - } - } + const bitbucketPublisher = new BitbucketPublisher(config, { logger }); + publishers.register('bitbucket', bitbucketPublisher); - const azureConfig = config.getOptionalConfig('scaffolder.azure'); - if (azureConfig) { - try { - const azurePublisher = new AzurePublisher(config, { logger }); - publishers.register('azure/api', azurePublisher); - } catch (e) { - const providerName = 'azure'; - if (process.env.NODE_ENV !== 'development') { - throw new Error( - `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, - ); - } - - logger.warn( - `Skipping ${providerName} scaffolding provider, ${e.message}`, - ); - } - } - - const bitbucketConfig = config.getOptionalConfig( - 'scaffolder.bitbucket.api', - ); - if (bitbucketConfig) { - try { - const bitbucketPublisher = new BitbucketPublisher(config, { logger }); - publishers.register('bitbucket', bitbucketPublisher); - } catch (e) { - const providerName = 'bitbucket'; - if (process.env.NODE_ENV !== 'development') { - throw new Error( - `Failed to initialize ${providerName} scaffolding provider, ${e.message}`, - ); - } - - logger.warn( - `Skipping ${providerName} scaffolding provider, ${e.message}`, - ); - } - } return publishers; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/types.ts index 56111337fe..7f12eb71db 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/types.ts @@ -17,6 +17,5 @@ export type RemoteProtocol = | 'file' | 'github' | 'gitlab' - | 'gitlab/api' - | 'azure/api' + | 'azure' | 'bitbucket';