paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 45 Packages Projects Releases Wiki Activity

Merge pull request #3900 from backstage/blam/move-scaffolder-to-integrations

Browse Source 
Deprecate Scaffolder config, prefer integrations config instead
This commit is contained in:
Ben Lambert
2021-01-21 08:50:32 +01:00
committed by GitHub
parent e9bcb7b446 04c801978e
commit dd0872a645
36 changed files with 899 additions and 954 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts
+1 -1
View File
@@ -120,7 +120,7 @@ export class JobProcessor implements Processor {
// Log to the current stage the error that occurred and fail the stage.
stage.status = 'FAILED';
logger.error(`Stage failed with error: ${error.message}`);
logger.debug(error.stack);
// Throw the error so the job can be failed too.
throw error;
} finally {
plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts
+2 -34
View File
@@ -13,15 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
makeDeprecatedLocationTypeDetector,
parseLocationAnnotation,
} from './helpers';
import { parseLocationAnnotation } from './helpers';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
describe('Helpers', () => {
describe('parseLocationAnnotation', () => {
@@ -30,10 +26,7 @@ describe('Helpers', () => {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
// [LOCATION_ANNOTATION]:
// 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
},
annotations: {},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
@@ -257,29 +250,4 @@ describe('Helpers', () => {
});
});
});
describe('makeDeprecatedLocationTypeDetector', () => {
it('detects deprecated location types', () => {
const detector = makeDeprecatedLocationTypeDetector(
new ConfigReader({
integrations: {
github: [{ host: 'derp.com' }, { host: 'foo.com' }],
gitlab: [{ host: 'derp.org' }, { host: 'foo.org' }],
azure: [{ host: 'derp.net' }, { host: 'foo.net' }],
},
}),
);
expect(detector('http://lol:wut@derp.com/wat')).toBe('github');
expect(detector('https://foo.com/wat')).toBe('github');
expect(detector('http://derp.org:80/wat')).toBe('gitlab');
expect(detector('https://foo.org/wat')).toBe('gitlab');
expect(detector('http://not.derp.net')).toBe(undefined);
expect(detector('http://derp.net')).toBe('azure/api');
expect(detector('http://derp.net:8080/wat')).toBe('azure/api');
expect(detector('http://github.com')).toBe('github');
expect(detector('http://gitlab.com')).toBe('gitlab');
expect(detector('http://dev.azure.com')).toBe('azure/api');
});
});
});
plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts
+2 -42
View File
@@ -17,12 +17,10 @@ import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/backend-common';
import { RemoteProtocol } from './types';
export type ParsedLocationAnnotation = {
protocol: RemoteProtocol;
protocol: 'file' | 'url';
location: string;
};
@@ -40,7 +38,7 @@ export const parseLocationAnnotation = (
// split on the first colon for the protocol and the rest after the first split
// is the location.
const [protocol, location] = annotation.split(/:(.+)/) as [
RemoteProtocol?,
('file' | 'url')?,
string?,
];
@@ -55,41 +53,3 @@ export const parseLocationAnnotation = (
location,
};
};
export type DeprecatedLocationTypeDetector = (
url: string,
) => string | undefined;
// The reason for the existence of this is to help in migration to using mostly locations
// of type 'url'. This allows us to detect the deprecated location type based on the host,
// which we in turn can use to select out preparer or publisher.
//
// TODO(Rugvip): This should be removed in the future once we fully migrate to using
// integrations configuration for the scaffolder.
export function makeDeprecatedLocationTypeDetector(
config: Config,
): DeprecatedLocationTypeDetector {
const hostMap = new Map();
// These are installed by default by the integrations
hostMap.set('github.com', 'github');
hostMap.set('gitlab.com', 'gitlab');
hostMap.set('dev.azure.com', 'azure/api');
config.getOptionalConfigArray('integrations.github')?.forEach(sub => {
hostMap.set(sub.getString('host'), 'github');
});
config.getOptionalConfigArray('integrations.gitlab')?.forEach(sub => {
hostMap.set(sub.getString('host'), 'gitlab');
});
config.getOptionalConfigArray('integrations.azure')?.forEach(sub => {
hostMap.set(sub.getString('host'), 'azure/api');
});
config.getOptionalConfigArray('integrations.bitbucket')?.forEach(sub => {
hostMap.set(sub.getString('host'), 'bitbucket');
});
return (url: string): string | undefined => {
const parsed = new URL(url);
return hostMap.get(parsed.hostname);
};
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts
+24 -23
View File
@@ -26,13 +26,14 @@ import {
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
describe('AzurePreparer', () => {
const mockGitClient = {
clone: jest.fn(),
};
const logger = getVoidLogger();
jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
let mockEntity: TemplateEntityV1alpha1;
@@ -44,7 +45,7 @@ describe('AzurePreparer', () => {
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
@@ -78,30 +79,33 @@ describe('AzurePreparer', () => {
};
});
it('initializes git client with the correct arguments if an access token is provided for a repository', async () => {
const preparer = new AzurePreparer(
new ConfigReader({
scaffolder: {
azure: {
api: {
token: 'fake-token',
},
},
},
}),
);
const logger = getVoidLogger();
const preparer = AzurePreparer.fromConfig({
host: 'dev.azure.com',
token: 'fake-azure-token',
});
// TODO(blam): Here's a test that will fail when the deprecation is complete
it('calls the clone command with deprecated token', async () => {
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
username: 'notempty',
password: 'fake-token',
logger,
password: 'fake-azure-token',
username: 'notempty',
});
});
it('calls the clone command with the correct arguments for a repository', async () => {
const preparer = new AzurePreparer(new ConfigReader({}));
it('calls the clone command with token from integrations config', async () => {
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
password: 'fake-azure-token',
username: 'notempty',
});
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
@@ -112,7 +116,6 @@ describe('AzurePreparer', () => {
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
const preparer = new AzurePreparer(new ConfigReader({}));
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
@@ -125,7 +128,6 @@ describe('AzurePreparer', () => {
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = new AzurePreparer(new ConfigReader({}));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
@@ -138,12 +140,11 @@ describe('AzurePreparer', () => {
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = new AzurePreparer(new ConfigReader({}));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts
+11 -17
View File
@@ -18,32 +18,26 @@ import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError, Git } from '@backstage/backend-common';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
import { Config } from '@backstage/config';
import { AzureIntegrationConfig } from '@backstage/integration';
export class AzurePreparer implements PreparerBase {
private readonly privateToken: string;
constructor(config: Config) {
this.privateToken =
config.getOptionalString('scaffolder.azure.api.token') ?? '';
static fromConfig(config: AzureIntegrationConfig) {
return new AzurePreparer({ token: config.token });
}
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
const { logger } = opts;
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
if (!['azure/api', 'url'].includes(protocol)) {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'url'`,
);
}
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
@@ -59,9 +53,9 @@ export class AzurePreparer implements PreparerBase {
// Username can be anything but the empty string according to:
// https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#use-a-pat
const git = this.privateToken
const git = this.config.token
? Git.fromAuth({
password: this.privateToken,
password: this.config.token,
username: 'notempty',
logger,
})
plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts
+34 -23
View File
@@ -26,10 +26,10 @@ import {
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
describe('BitbucketPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const logger = getVoidLogger();
const mockGitClient = {
clone: jest.fn(),
};
@@ -78,8 +78,13 @@ describe('BitbucketPreparer', () => {
};
});
const preparer = BitbucketPreparer.fromConfig({
host: 'bitbucket.org',
username: 'fake-user',
appPassword: 'fake-password',
});
it('calls the clone command with the correct arguments for a repository', async () => {
const preparer = new BitbucketPreparer(new ConfigReader({}));
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
@@ -88,28 +93,21 @@ describe('BitbucketPreparer', () => {
});
it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => {
const preparer = new BitbucketPreparer(
new ConfigReader({
integrations: {
bitbucket: [
{
host: 'bitbucket.org',
username: 'fake-user',
appPassword: 'fake-password',
},
],
},
}),
);
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
dir: expect.any(String),
const preparer = BitbucketPreparer.fromConfig({
host: 'bitbucket.org',
username: 'fake-user',
appPassword: 'fake-password',
});
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'fake-user',
password: 'fake-password',
});
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
const preparer = new BitbucketPreparer(new ConfigReader({}));
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
@@ -119,7 +117,6 @@ describe('BitbucketPreparer', () => {
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = new BitbucketPreparer(new ConfigReader({}));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
@@ -130,12 +127,26 @@ describe('BitbucketPreparer', () => {
);
});
it('calls the clone command with with token for auth method', async () => {
const preparer = BitbucketPreparer.fromConfig({
host: 'bitbucket.org',
token: 'fake-token',
});
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'x-token-auth',
password: 'fake-token',
});
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = new BitbucketPreparer(new ConfigReader({}));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts
+39 -22
View File
@@ -18,35 +18,35 @@ import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError, Git } from '@backstage/backend-common';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import { BitbucketIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { Config } from '@backstage/config';
export class BitbucketPreparer implements PreparerBase {
private readonly privateToken: string;
private readonly username: string;
constructor(config: Config) {
this.username =
config.getOptionalString('scaffolder.bitbucket.api.username') ?? '';
this.privateToken =
config.getOptionalString('scaffolder.bitbucket.api.token') ?? '';
static fromConfig(config: BitbucketIntegrationConfig) {
return new BitbucketPreparer({
username: config.username,
token: config.token,
appPassword: config.appPassword,
});
}
constructor(
private readonly config: {
username?: string;
token?: string;
appPassword?: string;
},
) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
const { logger } = opts;
if (!['bitbucket', 'url'].includes(protocol)) {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'url'`,
);
}
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const repo = parseGitUrl(location);
@@ -63,10 +63,10 @@ export class BitbucketPreparer implements PreparerBase {
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const git = this.privateToken
const auth = this.getAuth();
const git = auth
? Git.fromAuth({
username: this.username,
password: this.privateToken,
...auth,
logger,
})
: Git.fromAuth({ logger });
@@ -78,4 +78,21 @@ export class BitbucketPreparer implements PreparerBase {
return checkoutLocation;
}
private getAuth(): { username: string; password: string } | undefined {
const { username, token, appPassword } = this.config;
if (username && appPassword) {
return { username: username, password: appPassword };
}
if (token) {
return {
username: 'x-token-auth',
password: token! || appPassword!,
};
}
return undefined;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts
+20 -12
View File
@@ -32,6 +32,7 @@ describe('GitHubPreparer', () => {
const mockGitClient = {
clone: jest.fn(),
};
const logger = getVoidLogger();
jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
@@ -76,9 +77,13 @@ describe('GitHubPreparer', () => {
},
};
});
it('calls the clone command with the correct arguments for a repository', async () => {
const preparer = new GithubPreparer();
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
@@ -86,8 +91,8 @@ describe('GitHubPreparer', () => {
dir: expect.any(String),
});
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
const preparer = new GithubPreparer();
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
@@ -99,23 +104,29 @@ describe('GitHubPreparer', () => {
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = new GithubPreparer();
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = new GithubPreparer();
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
@@ -123,15 +134,12 @@ describe('GitHubPreparer', () => {
);
});
it('calls the clone command with the token when provided', async () => {
const preparer = new GithubPreparer({ token: 'abc' });
const logger = getVoidLogger();
it('calls the clone command with token', async () => {
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'abc',
username: 'fake-token',
password: 'x-oauth-basic',
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts
+11 -15
View File
@@ -18,30 +18,26 @@ import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError, Git } from '@backstage/backend-common';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
import { GitHubIntegrationConfig } from '@backstage/integration';
export class GithubPreparer implements PreparerBase {
token?: string;
constructor(params: { token?: string } = {}) {
this.token = params.token;
static fromConfig(config: GitHubIntegrationConfig) {
return new GithubPreparer({ token: config.token });
}
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
const { logger } = opts;
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
if (!['github', 'url'].includes(protocol)) {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'url'`,
);
}
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
@@ -57,9 +53,9 @@ export class GithubPreparer implements PreparerBase {
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const git = this.token
const git = this.config.token
? Git.fromAuth({
username: this.token,
username: this.config.token,
password: 'x-oauth-basic',
logger,
})
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts
+61 -97
View File
@@ -24,15 +24,15 @@ import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { getVoidLogger, Git } from '@backstage/backend-common';
const mockEntityWithProtocol = (protocol: string): TemplateEntityV1alpha1 => ({
const mockTemplate = (): TemplateEntityV1alpha1 => ({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]: `${protocol}:https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/template.yaml`,
[LOCATION_ANNOTATION]:
'url:https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
@@ -70,108 +70,72 @@ describe('GitLabPreparer', () => {
const mockGitClient = {
clone: jest.fn(),
};
const logger = getVoidLogger();
jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
beforeEach(() => {
jest.clearAllMocks();
});
const preparer = GitlabPreparer.fromConfig({
host: 'gitlab.com',
token: 'fake-token',
});
it(`calls the clone command with the correct arguments for a repository`, async () => {
mockEntity = mockTemplate();
['gitlab', 'gitlab/api'].forEach(protocol => {
it(`calls the clone command with the correct arguments for a repository using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(new ConfigReader({}));
mockEntity = mockEntityWithProtocol(protocol);
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
});
});
it(`calls the clone command with the correct arguments if an access token is provided in integrations for a repository using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(
new ConfigReader({
integrations: {
gitlab: [
{
host: 'gitlab.com',
token: 'fake-token',
},
],
},
}),
);
mockEntity = mockEntityWithProtocol(protocol);
const logger = getVoidLogger();
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'oauth2',
password: 'fake-token',
});
});
it(`calls the clone command with the correct arguments if an access token is provided in scaffolder for a repository using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(
new ConfigReader({
scaffolder: {
gitlab: { api: { token: 'fake-token' } },
},
}),
);
mockEntity = mockEntityWithProtocol(protocol);
const logger = getVoidLogger();
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'oauth2',
password: 'fake-token',
});
});
it(`calls the clone command with the correct arguments for a repository when no path is provided using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(new ConfigReader({}));
mockEntity = mockEntityWithProtocol(protocol);
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
});
});
it(`return the temp directory with the path to the folder if it is specified using the ${protocol} protocol`, async () => {
const preparer = new GitlabPreparer(new ConfigReader({}));
mockEntity = mockEntityWithProtocol(protocol);
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = new GitlabPreparer(new ConfigReader({}));
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
workingDirectory: '/workDir',
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
});
});
it(`calls the clone command with the correct arguments if an access token is provided in integrations for a repository`, async () => {
mockEntity = mockTemplate();
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'oauth2',
password: 'fake-token',
});
});
it(`calls the clone command with the correct arguments for a repository when no path is provided`, async () => {
mockEntity = mockTemplate();
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
});
});
it(`return the temp directory with the path to the folder if it is specified`, async () => {
mockEntity = mockTemplate();
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts
+11 -34
View File
@@ -13,13 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { InputError, Git } from '@backstage/backend-common';
import { Git } from '@backstage/backend-common';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
GitLabIntegrationConfig,
readGitLabIntegrationConfigs,
} from '@backstage/integration';
import { GitLabIntegrationConfig } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
@@ -28,31 +24,20 @@ import { parseLocationAnnotation } from '../helpers';
import { PreparerBase, PreparerOptions } from './types';
export class GitlabPreparer implements PreparerBase {
private readonly integrations: GitLabIntegrationConfig[];
private readonly scaffolderToken: string | undefined;
constructor(config: Config) {
this.integrations = readGitLabIntegrationConfigs(
config.getOptionalConfigArray('integrations.gitlab') ?? [],
);
this.scaffolderToken = config.getOptionalString(
'scaffolder.gitlab.api.token',
);
static fromConfig(config: GitLabIntegrationConfig) {
return new GitlabPreparer({ token: config.token });
}
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const { logger } = opts;
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
const { location } = parseLocationAnnotation(template);
const logger = opts.logger;
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
if (!['gitlab', 'gitlab/api', 'url'].includes(protocol)) {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'url'`,
);
}
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
@@ -66,10 +51,9 @@ export class GitlabPreparer implements PreparerBase {
template.spec.path ?? '.',
);
const token = this.getToken(parsedGitLocation.resource);
const git = token
const git = this.config.token
? Git.fromAuth({
password: token,
password: this.config.token,
username: 'oauth2',
logger,
})
@@ -82,11 +66,4 @@ export class GitlabPreparer implements PreparerBase {
return path.resolve(tempDir, templateDirectory);
}
private getToken(host: string): string | undefined {
return (
this.scaffolderToken ||
this.integrations.find(c => c.host === host)?.token
);
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.test.ts
+21 -94
View File
@@ -14,109 +14,36 @@
* limitations under the License.
*/
import { Preparers } from '.';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { FilePreparer } from './file';
import { GithubPreparer } from './github';
describe('Preparers', () => {
const mockTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'file:/Users/bingo/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
templater: 'cookiecutter',
path: '.',
type: 'website',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
it('should throw an error when the preparer for the source location is not registered', () => {
it('should return the correct preparer based on the hostname', async () => {
const preparer = await GithubPreparer.fromConfig({
host: 'github.com',
apiBaseUrl: 'lols',
token: 'something else yo',
});
const preparers = new Preparers();
preparers.register('github.com', preparer);
expect(() => preparers.get(mockTemplate)).toThrow(
expect.objectContaining({
message: 'No preparer registered for type: "file"',
}),
);
});
it('should return the correct preparer when the source matches', () => {
const preparers = new Preparers();
const preparer = new FilePreparer();
preparers.register('file', preparer);
expect(preparers.get(mockTemplate)).toBe(preparer);
expect(
preparers.get('https://github.com/please/find/me/something/from/github'),
).toBe(preparer);
});
it('should throw an error if the metadata tag does not exist in the entity', () => {
const brokenTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: '.',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
it('should throw an error if there is nothing that will match the url provided', async () => {
const preparer = await GithubPreparer.fromConfig({
host: 'github.com',
apiBaseUrl: 'lols',
token: 'something else yo',
});
const preparers = new Preparers();
preparers.register('github.com', preparer);
expect(() => preparers.get(brokenTemplate)).toThrow(
expect.objectContaining({
message: expect.stringContaining('No location annotation provided'),
}),
expect(() => preparers.get('https://404.com')).toThrow(
`Unable to find a preparer for URL: https://404.com. Please make sure to register this host under an integration in app-config`,
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/preparers.ts
+38 -68
View File
@@ -15,95 +15,65 @@
*/
import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { PreparerBase, PreparerBuilder } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import {
DeprecatedLocationTypeDetector,
makeDeprecatedLocationTypeDetector,
parseLocationAnnotation,
} from '../helpers';
import { RemoteProtocol } from '../types';
import { FilePreparer } from './file';
import { Logger } from 'winston';
import { GitlabPreparer } from './gitlab';
import { AzurePreparer } from './azure';
import { GithubPreparer } from './github';
import { BitbucketPreparer } from './bitbucket';
import { ScmIntegrations } from '@backstage/integration';
export class Preparers implements PreparerBuilder {
private preparerMap = new Map<RemoteProtocol, PreparerBase>();
private preparerMap = new Map<string, PreparerBase>();
constructor(private readonly typeDetector?: DeprecatedLocationTypeDetector) {}
register(protocol: RemoteProtocol, preparer: PreparerBase) {
this.preparerMap.set(protocol, preparer);
register(host: string, preparer: PreparerBase) {
this.preparerMap.set(host, preparer);
}
get(template: TemplateEntityV1alpha1): PreparerBase {
const { protocol, location } = parseLocationAnnotation(template);
const preparer = this.preparerMap.get(protocol);
get(url: string): PreparerBase {
const preparer = this.preparerMap.get(new URL(url).host);
if (!preparer) {
if ((protocol as string) === 'url') {
const type = this.typeDetector?.(location);
const detected = type && this.preparerMap.get(type as RemoteProtocol);
if (detected) {
return detected;
}
if (type) {
throw new Error(
`No preparer configuration available for type '${type}' with url "${location}". ` +
"Make sure you've added appropriate configuration in the 'scaffolder' configuration section",
);
} else {
throw new Error(
`Failed to detect preparer type. Unable to determine integration type for location "${location}". ` +
"Please add appropriate configuration to the 'integrations' configuration section",
);
}
}
throw new Error(`No preparer registered for type: "${protocol}"`);
throw new Error(
`Unable to find a preparer for URL: ${url}. Please make sure to register this host under an integration in app-config`,
);
}
return preparer;
}
static async fromConfig(
config: Config,
{ logger }: { logger: Logger },
// eslint-disable-next-line
_: { logger: Logger },
): Promise<PreparerBuilder> {
const typeDetector = makeDeprecatedLocationTypeDetector(config);
const preparers = new Preparers();
const scm = ScmIntegrations.fromConfig(config);
for (const integration of scm.azure.list()) {
preparers.register(
integration.config.host,
AzurePreparer.fromConfig(integration.config),
);
}
const preparers = new Preparers(typeDetector);
for (const integration of scm.github.list()) {
preparers.register(
integration.config.host,
GithubPreparer.fromConfig(integration.config),
);
}
const filePreparer = new FilePreparer();
const gitlabPreparer = new GitlabPreparer(config);
const azurePreparer = new AzurePreparer(config);
const bitbucketPreparer = new BitbucketPreparer(config);
for (const integration of scm.gitlab.list()) {
preparers.register(
integration.config.host,
GitlabPreparer.fromConfig(integration.config),
);
}
preparers.register('file', filePreparer);
preparers.register('gitlab', gitlabPreparer);
preparers.register('gitlab/api', gitlabPreparer);
preparers.register('azure/api', azurePreparer);
preparers.register('bitbucket', bitbucketPreparer);
const githubConfig = config.getOptionalConfig('scaffolder.github');
if (githubConfig) {
try {
const githubToken = githubConfig.getString('token');
const githubPreparer = new GithubPreparer({ token: githubToken });
preparers.register('github', githubPreparer);
} catch (e) {
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize github scaffolding provider, ${e.message}`,
);
}
logger.warn(`Skipping github scaffolding provider, ${e.message}`);
}
for (const integration of scm.bitbucket.list()) {
preparers.register(
integration.config.host,
BitbucketPreparer.fromConfig(integration.config),
);
}
return preparers;
plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts
+6 -7
View File
@@ -15,14 +15,13 @@
*/
import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
import { RemoteProtocol } from '../types';
export type PreparerOptions = {
logger: Logger;
workingDirectory?: string;
logger: Logger;
};
export type PreparerBase = {
export interface PreparerBase {
/**
* Given an Entity definition from the Service Catalog, go and prepare a directory
* with contents from the remote location in temporary storage and return the path
@@ -30,11 +29,11 @@ export type PreparerBase = {
*/
prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
opts?: PreparerOptions,
): Promise<string>;
};
}
export type PreparerBuilder = {
register(protocol: RemoteProtocol, preparer: PreparerBase): void;
get(template: TemplateEntityV1alpha1): PreparerBase;
register(host: string, preparer: PreparerBase): void;
get(url: string): PreparerBase;
};
plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts
+24 -16
View File
@@ -15,32 +15,40 @@
*/
jest.mock('./helpers');
jest.mock('azure-devops-node-api', () => ({
WebApi: jest.fn(),
getPersonalAccessTokenHandler: jest.fn(),
}));
import { AzurePublisher } from './azure';
import { GitApi } from 'azure-devops-node-api/GitApi';
import { WebApi } from 'azure-devops-node-api';
import * as helpers from './helpers';
import { getVoidLogger } from '@backstage/backend-common';
const { mockGitApi } = require('azure-devops-node-api/GitApi') as {
mockGitApi: {
createRepository: jest.MockedFunction<GitApi['createRepository']>;
};
};
describe('Azure Publisher', () => {
const publisher = new AzurePublisher(new GitApi('', []), 'fake-token');
const logger = getVoidLogger();
beforeEach(() => {
jest.clearAllMocks();
});
describe('publish: createRemoteInAzure', () => {
it('should use azure-devops-node-api to create a repo in the given project', async () => {
mockGitApi.createRepository.mockResolvedValue({
const mockGitClient = {
createRepository: jest.fn(),
};
const mockGitApi = {
getGitApi: jest.fn().mockReturnValue(mockGitClient),
};
((WebApi as unknown) as jest.Mock).mockImplementation(() => mockGitApi);
const publisher = await AzurePublisher.fromConfig({
host: 'dev.azure.com',
token: 'fake-azure-token',
});
mockGitClient.createRepository.mockResolvedValue({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
} as { remoteUrl: string });
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'project/repo',
owner: 'bob',
@@ -54,7 +62,7 @@ describe('Azure Publisher', () => {
catalogInfoUrl:
'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml',
});
expect(mockGitApi.createRepository).toHaveBeenCalledWith(
expect(mockGitClient.createRepository).toHaveBeenCalledWith(
{
name: 'repo',
},
@@ -63,7 +71,7 @@ describe('Azure Publisher', () => {
expect(helpers.initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
auth: { username: 'notempty', password: 'fake-token' },
auth: { username: 'notempty', password: 'fake-azure-token' },
logger,
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts
+28 -17
View File
@@ -15,27 +15,38 @@
*/
import { PublisherBase, PublisherOptions, PublisherResult } from './types';
import { GitApi } from 'azure-devops-node-api/GitApi';
import { IGitApi } from 'azure-devops-node-api/GitApi';
import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces';
import { JsonValue } from '@backstage/config';
import { RequiredTemplateValues } from '../templater';
import { initRepoAndPush } from './helpers';
import { AzureIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
export class AzurePublisher implements PublisherBase {
private readonly client: GitApi;
private readonly token: string;
constructor(client: GitApi, token: string) {
this.client = client;
this.token = token;
static async fromConfig(config: AzureIntegrationConfig) {
if (!config.token) {
return undefined;
}
const authHandler = getPersonalAccessTokenHandler(config.token);
const webApi = new WebApi(config.host, authHandler);
const azureClient = await webApi.getGitApi();
return new AzurePublisher({ token: config.token, client: azureClient });
}
constructor(private readonly config: { token: string; client: IGitApi }) {}
async publish({
values,
directory,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const remoteUrl = await this.createRemote(values);
const { owner, name } = parseGitUrl(values.storePath);
const remoteUrl = await this.createRemote({
project: owner,
name,
});
const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`;
await initRepoAndPush({
@@ -43,7 +54,7 @@ export class AzurePublisher implements PublisherBase {
remoteUrl,
auth: {
username: 'notempty',
password: this.token,
password: this.config.token,
},
logger,
});
@@ -51,13 +62,13 @@ export class AzurePublisher implements PublisherBase {
return { remoteUrl, catalogInfoUrl };
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
) {
const [project, name] = values.storePath.split('/');
private async createRemote(opts: { name: string; project: string }) {
const { name, project } = opts;
const createOptions: GitRepositoryCreateOptions = { name };
const repo = await this.client.createRepository(createOptions, project);
const repo = await this.config.client.createRepository(
createOptions,
project,
);
return repo.remoteUrl || '';
}
plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts
+13 -13
View File
@@ -58,15 +58,15 @@ describe('Bitbucket Publisher', () => {
),
);
const publisher = new BitbucketPublisher(
'https://bitbucket.org',
'fake-user',
'fake-token',
);
const publisher = await BitbucketPublisher.fromConfig({
host: 'bitbucket.org',
username: 'fake-user',
appPassword: 'fake-token',
});
const result = await publisher.publish({
values: {
storePath: 'project/repo',
storePath: 'https://bitbucket.org/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
@@ -87,6 +87,7 @@ describe('Bitbucket Publisher', () => {
});
});
});
describe('publish: createRemoteInBitbucketServer', () => {
it('should create repo in bitbucket server', async () => {
server.use(
@@ -116,15 +117,14 @@ describe('Bitbucket Publisher', () => {
),
);
const publisher = new BitbucketPublisher(
'https://bitbucket.mycompany.com',
'fake-user',
'fake-token',
);
const publisher = await BitbucketPublisher.fromConfig({
host: 'bitbucket.mycompany.com',
token: 'fake-token',
});
const result = await publisher.publish({
values: {
storePath: 'project/repo',
storePath: 'https://bitbucket.mycompany.com/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
@@ -140,7 +140,7 @@ describe('Bitbucket Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo',
auth: { username: 'fake-user', password: 'fake-token' },
auth: { username: 'x-token-auth', password: 'fake-token' },
logger: logger,
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts
+62 -32
View File
@@ -16,62 +16,90 @@
import { PublisherBase, PublisherOptions, PublisherResult } from './types';
import { initRepoAndPush } from './helpers';
import { RequiredTemplateValues } from '../templater';
import { JsonValue } from '../../../../../../packages/config/src';
import fetch from 'cross-fetch';
import { BitbucketIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
// TODO(blam): We should probably start to use a bitbucket client here that we can change
// the baseURL to point at on-prem or public bitbucket versions like we do for
// github and ghe. There's to much logic and not enough types here for us to say that this way is better than using
// a supported bitbucket client if one exists.
export class BitbucketPublisher implements PublisherBase {
private readonly host: string;
private readonly username: string;
private readonly token: string;
constructor(host: string, username: string, token: string) {
this.host = host;
this.username = username;
this.token = token;
static async fromConfig(config: BitbucketIntegrationConfig) {
return new BitbucketPublisher({
host: config.host,
token: config.token,
appPassword: config.appPassword,
username: config.username,
});
}
constructor(
private readonly config: {
host: string;
token?: string;
appPassword?: string;
username?: string;
},
) {}
async publish({
values,
directory,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const result = await this.createRemote(values);
const { owner: project, name } = parseGitUrl(values.storePath);
const description = values.description as string;
const result = await this.createRemote({
project,
name,
description,
});
await initRepoAndPush({
dir: directory,
remoteUrl: result.remoteUrl,
auth: {
username: this.username,
password: this.token,
username: this.config.username ? this.config.username : 'x-token-auth',
password: this.config.appPassword
? this.config.appPassword
: this.config.token ?? '',
},
logger,
});
return result;
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
): Promise<PublisherResult> {
if (this.host === 'https://bitbucket.org') {
return this.createBitbucketCloudRepository(values);
private async createRemote(opts: {
project: string;
name: string;
description: string;
}): Promise<PublisherResult> {
if (this.config.host === 'bitbucket.org') {
return this.createBitbucketCloudRepository(opts);
}
return this.createBitbucketServerRepository(values);
return this.createBitbucketServerRepository(opts);
}
private async createBitbucketCloudRepository(
values: RequiredTemplateValues & Record<string, JsonValue>,
): Promise<PublisherResult> {
const [project, name] = values.storePath.split('/');
private async createBitbucketCloudRepository(opts: {
project: string;
name: string;
description: string;
}): Promise<PublisherResult> {
const { project, name, description } = opts;
let response: Response;
const buffer = Buffer.from(`${this.username}:${this.token}`, 'utf8');
const buffer = Buffer.from(
`${this.config.username}:${this.config.appPassword}`,
'utf8',
);
const options: RequestInit = {
method: 'POST',
body: JSON.stringify({
scm: 'git',
description: values.description,
description: description,
}),
headers: {
Authorization: `Basic ${buffer.toString('base64')}`,
@@ -102,26 +130,28 @@ export class BitbucketPublisher implements PublisherBase {
throw new Error(`Not a valid response code ${await response.text()}`);
}
private async createBitbucketServerRepository(
values: RequiredTemplateValues & Record<string, JsonValue>,
): Promise<PublisherResult> {
const [project, name] = values.storePath.split('/');
private async createBitbucketServerRepository(opts: {
project: string;
name: string;
description: string;
}): Promise<PublisherResult> {
const { project, name, description } = opts;
let response: Response;
const options: RequestInit = {
method: 'POST',
body: JSON.stringify({
name: name,
description: values.description,
description: description,
}),
headers: {
Authorization: `Bearer ${this.token}`,
Authorization: `Bearer ${this.config.token}`,
'Content-Type': 'application/json',
},
};
try {
response = await fetch(
`${this.host}/rest/api/1.0/projects/${project}/repos`,
`https://${this.config.host}/rest/api/1.0/projects/${project}/repos`,
options,
);
} catch (e) {
plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts
+55 -33
View File
@@ -37,14 +37,16 @@ describe('GitHub Publisher', () => {
});
describe('with public repo visibility', () => {
const publisher = new GithubPublisher({
client: new Octokit(),
token: 'abc',
repoVisibility: 'public',
});
describe('publish: createRemoteInGithub', () => {
it('should use octokit to create a repo in an organisation if the organisation property is set', async () => {
const publisher = await GithubPublisher.fromConfig(
{
token: 'fake-token',
host: 'github.com',
},
{ repoVisibility: 'public' },
);
mockGithubClient.repos.createInOrg.mockResolvedValue({
data: {
clone_url: 'https://github.com/backstage/backstage.git',
@@ -56,9 +58,9 @@ describe('GitHub Publisher', () => {
},
} as RestEndpointMethodTypes['users']['getByUsername']['response']);
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'blam/test',
storePath: 'https://github.com/blam/test',
owner: 'bob',
access: 'blam/team',
},
@@ -89,12 +91,20 @@ describe('GitHub Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'abc', password: 'x-oauth-basic' },
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
});
});
it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => {
const publisher = await GithubPublisher.fromConfig(
{
token: 'fake-token',
host: 'github.com',
},
{ repoVisibility: 'public' },
);
mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
data: {
clone_url: 'https://github.com/backstage/backstage.git',
@@ -106,9 +116,9 @@ describe('GitHub Publisher', () => {
},
} as RestEndpointMethodTypes['users']['getByUsername']['response']);
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'blam/test',
storePath: 'https://github.com/blam/test',
owner: 'bob',
access: 'blam',
},
@@ -132,13 +142,21 @@ describe('GitHub Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'abc', password: 'x-oauth-basic' },
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
});
});
});
it('should invite other user in the authed user', async () => {
const publisher = await GithubPublisher.fromConfig(
{
token: 'fake-token',
host: 'github.com',
},
{ repoVisibility: 'public' },
);
mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
data: {
clone_url: 'https://github.com/backstage/backstage.git',
@@ -150,9 +168,9 @@ describe('GitHub Publisher', () => {
},
} as RestEndpointMethodTypes['users']['getByUsername']['response']);
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'blam/test',
storePath: 'https://github.com/blam/test',
owner: 'bob',
access: 'bob',
description: 'description',
@@ -182,20 +200,22 @@ describe('GitHub Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'abc', password: 'x-oauth-basic' },
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
});
});
});
describe('with internal repo visibility', () => {
const publisher = new GithubPublisher({
client: new Octokit(),
token: 'abc',
repoVisibility: 'internal',
});
it('creates a private repository in the organization with visibility set to internal', async () => {
const publisher = await GithubPublisher.fromConfig(
{
token: 'fake-token',
host: 'github.com',
},
{ repoVisibility: 'internal' },
);
mockGithubClient.repos.createInOrg.mockResolvedValue({
data: {
clone_url: 'https://github.com/backstage/backstage.git',
@@ -207,10 +227,10 @@ describe('GitHub Publisher', () => {
},
} as RestEndpointMethodTypes['users']['getByUsername']['response']);
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
isOrg: true,
storePath: 'blam/test',
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
@@ -231,20 +251,22 @@ describe('GitHub Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'abc', password: 'x-oauth-basic' },
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
});
});
});
describe('private visibility in a user account', () => {
const publisher = new GithubPublisher({
client: new Octokit(),
token: 'abc',
repoVisibility: 'private',
});
it('creates a private repository', async () => {
const publisher = await GithubPublisher.fromConfig(
{
token: 'fake-token',
host: 'github.com',
},
{ repoVisibility: 'private' },
);
mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
data: {
clone_url: 'https://github.com/backstage/backstage.git',
@@ -256,9 +278,9 @@ describe('GitHub Publisher', () => {
},
} as RestEndpointMethodTypes['users']['getByUsername']['response']);
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'blam/test',
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
@@ -279,7 +301,7 @@ describe('GitHub Publisher', () => {
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'abc', password: 'x-oauth-basic' },
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts
+55 -37
View File
@@ -15,46 +15,61 @@
*/
import { PublisherBase, PublisherOptions, PublisherResult } from './types';
import { Octokit } from '@octokit/rest';
import { initRepoAndPush } from './helpers';
import { JsonValue } from '@backstage/config';
import { RequiredTemplateValues } from '../templater';
import { GitHubIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { Octokit } from '@octokit/rest';
export type RepoVisibilityOptions = 'private' | 'internal' | 'public';
interface GithubPublisherParams {
client: Octokit;
token: string;
repoVisibility: RepoVisibilityOptions;
}
export class GithubPublisher implements PublisherBase {
private client: Octokit;
private token: string;
private repoVisibility: RepoVisibilityOptions;
static async fromConfig(
config: GitHubIntegrationConfig,
{ repoVisibility }: { repoVisibility: RepoVisibilityOptions },
) {
if (!config.token) {
return undefined;
}
constructor({
client,
token,
repoVisibility = 'public',
}: GithubPublisherParams) {
this.client = client;
this.token = token;
this.repoVisibility = repoVisibility;
const githubClient = new Octokit({
auth: config.token,
baseUrl: config.apiBaseUrl,
});
return new GithubPublisher({
token: config.token,
client: githubClient,
repoVisibility,
});
}
constructor(
private readonly config: {
token: string;
client: Octokit;
repoVisibility: RepoVisibilityOptions;
},
) {}
async publish({
values,
directory,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const remoteUrl = await this.createRemote(values);
const { owner, name } = parseGitUrl(values.storePath);
const description = values.description as string;
const access = values.access as string;
const remoteUrl = await this.createRemote({
description,
access,
name,
owner,
});
await initRepoAndPush({
dir: directory,
remoteUrl,
auth: {
username: this.token,
username: this.config.token,
password: 'x-oauth-basic',
},
logger,
@@ -68,35 +83,38 @@ export class GithubPublisher implements PublisherBase {
return { remoteUrl, catalogInfoUrl };
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
) {
const [owner, name] = values.storePath.split('/');
const description = values.description as string;
private async createRemote(opts: {
access: string;
name: string;
owner: string;
description: string;
}) {
const { access, description, owner, name } = opts;
const user = await this.client.users.getByUsername({ username: owner });
const user = await this.config.client.users.getByUsername({
username: owner,
});
const repoCreationPromise =
user.data.type === 'Organization'
? this.client.repos.createInOrg({
? this.config.client.repos.createInOrg({
name,
org: owner,
private: this.repoVisibility !== 'public',
visibility: this.repoVisibility,
private: this.config.repoVisibility !== 'public',
visibility: this.config.repoVisibility,
description,
})
: this.client.repos.createForAuthenticatedUser({
: this.config.client.repos.createForAuthenticatedUser({
name,
private: this.repoVisibility === 'private',
private: this.config.repoVisibility === 'private',
description,
});
const { data } = await repoCreationPromise;
const access = values.access as string;
if (access?.startsWith(`${owner}/`)) {
const [, team] = access.split('/');
await this.client.teams.addOrUpdateRepoPermissionsInOrg({
await this.config.client.teams.addOrUpdateRepoPermissionsInOrg({
org: owner,
team_slug: team,
owner,
@@ -105,7 +123,7 @@ export class GithubPublisher implements PublisherBase {
});
// no need to add access if it's the person who own's the personal account
} else if (access && access !== owner) {
await this.client.repos.addCollaborator({
await this.config.client.repos.addCollaborator({
owner,
repo: name,
username: access,
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts
+46 -17
View File
@@ -14,33 +14,47 @@
* limitations under the License.
*/
jest.mock('@gitbeaker/node');
jest.mock('@gitbeaker/node', () => ({
Gitlab: jest.fn(),
}));
jest.mock('./helpers');
import { GitlabPublisher } from './gitlab';
import { Gitlab as GitlabAPI } from '@gitbeaker/core';
import { Gitlab } from '@gitbeaker/node';
import { initRepoAndPush } from './helpers';
import { getVoidLogger } from '@backstage/backend-common';
const { mockGitlabClient } = require('@gitbeaker/node') as {
mockGitlabClient: {
Namespaces: jest.Mocked<GitlabAPI['Namespaces']>;
Projects: jest.Mocked<GitlabAPI['Projects']>;
Users: jest.Mocked<GitlabAPI['Users']>;
};
};
describe('GitLab Publisher', () => {
const logger = getVoidLogger();
const publisher = new GitlabPublisher(new Gitlab({}), 'fake-token');
const mockGitlabClient = {
Namespaces: {
show: jest.fn(),
},
Projects: {
create: jest.fn(),
},
Users: {
current: jest.fn(),
},
};
beforeEach(() => {
jest.clearAllMocks();
((Gitlab as unknown) as jest.Mock).mockImplementation(
() => mockGitlabClient,
);
});
describe('publish: createRemoteInGitLab', () => {
it('should use gitbeaker to create a repo in a namespace if the namespace property is set', async () => {
const publisher = await GitlabPublisher.fromConfig({
host: 'gitlab.com',
token: 'fake-token',
baseUrl: 'https://gitlab.hosted.com',
});
mockGitlabClient.Namespaces.show.mockResolvedValue({
id: 42,
} as { id: number });
@@ -48,17 +62,24 @@ describe('GitLab Publisher', () => {
http_url_to_repo: 'mockclone',
} as { http_url_to_repo: string });
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
isOrg: true,
storePath: 'bloum/blam/test',
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
logger,
});
expect(result).toEqual({ remoteUrl: 'mockclone' });
expect(Gitlab).toHaveBeenCalledWith({
token: 'fake-token',
host: 'https://gitlab.hosted.com',
});
expect(result).toEqual({
remoteUrl: 'mockclone',
catalogInfoUrl: 'mockclone',
});
expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
namespace_id: 42,
name: 'test',
@@ -72,6 +93,11 @@ describe('GitLab Publisher', () => {
});
it('should use gitbeaker to create a repo in the authed user if the namespace property is not set', async () => {
const publisher = await GitlabPublisher.fromConfig({
host: 'gitlab.com',
token: 'fake-token',
});
mockGitlabClient.Namespaces.show.mockResolvedValue({});
mockGitlabClient.Users.current.mockResolvedValue({
id: 21,
@@ -80,16 +106,19 @@ describe('GitLab Publisher', () => {
http_url_to_repo: 'mockclone',
} as { http_url_to_repo: string });
const result = await publisher.publish({
const result = await publisher!.publish({
values: {
storePath: 'bloum/blam/test',
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
logger,
});
expect(result).toEqual({ remoteUrl: 'mockclone' });
expect(result).toEqual({
remoteUrl: 'mockclone',
catalogInfoUrl: 'mockclone',
});
expect(mockGitlabClient.Users.current).toHaveBeenCalled();
expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({
namespace_id: 21,
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts
+39 -22
View File
@@ -15,57 +15,74 @@
*/
import { PublisherBase, PublisherOptions, PublisherResult } from './types';
import { Gitlab } from '@gitbeaker/core';
import { JsonValue } from '@backstage/config';
import { Gitlab } from '@gitbeaker/node';
import { Gitlab as GitlabClient } from '@gitbeaker/core';
import { initRepoAndPush } from './helpers';
import { RequiredTemplateValues } from '../templater';
import parseGitUrl from 'git-url-parse';
import { GitLabIntegrationConfig } from '@backstage/integration';
export class GitlabPublisher implements PublisherBase {
private readonly client: Gitlab;
private readonly token: string;
static async fromConfig(config: GitLabIntegrationConfig) {
if (!config.token) {
return undefined;
}
constructor(client: Gitlab, token: string) {
this.client = client;
this.token = token;
const client = new Gitlab({ host: config.baseUrl, token: config.token });
return new GitlabPublisher({ token: config.token, client });
}
constructor(
private readonly config: { token: string; client: GitlabClient },
) {}
async publish({
values,
directory,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const remoteUrl = await this.createRemote(values);
const { owner, name } = parseGitUrl(values.storePath);
const remoteUrl = await this.createRemote({
owner,
name,
});
await initRepoAndPush({
dir: directory,
remoteUrl,
auth: {
username: 'oauth2',
password: this.token,
password: this.config.token,
},
logger,
});
return { remoteUrl };
const catalogInfoUrl = remoteUrl.replace(
/\.git$/,
'/-/blob/master/catalog-info.yaml',
);
return { remoteUrl, catalogInfoUrl };
}
private async createRemote(
values: RequiredTemplateValues & Record<string, JsonValue>,
) {
const pathElements = values.storePath.split('/');
const name = pathElements[pathElements.length - 1];
pathElements.pop();
const owner = pathElements.join('/');
private async createRemote(opts: { name: string; owner: string }) {
const { owner, name } = opts;
let targetNamespace = ((await this.client.Namespaces.show(owner)) as {
// TODO(blam): this needs cleaning up to be nicer. The amount of brackets is too damn high!
// Shouldn't have to cast things now
let targetNamespace = ((await this.config.client.Namespaces.show(
owner,
)) as {
id: number;
}).id;
if (!targetNamespace) {
targetNamespace = ((await this.client.Users.current()) as { id: number })
.id;
targetNamespace = ((await this.config.client.Users.current()) as {
id: number;
}).id;
}
const project = (await this.client.Projects.create({
const project = (await this.config.client.Projects.create({
namespace_id: targetNamespace,
name: name,
})) as { http_url_to_repo: string };
plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.test.ts
+90 -98
View File
@@ -14,120 +14,112 @@
* limitations under the License.
*/
import { Publishers } from './publishers';
import {
LOCATION_ANNOTATION,
TemplateEntityV1alpha1,
} from '@backstage/catalog-model';
import { GithubPublisher } from './github';
import { Octokit } from '@octokit/rest';
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { AzurePublisher } from './azure';
import { GitlabPublisher } from './gitlab';
import { BitbucketPublisher } from './bitbucket';
jest.mock('@octokit/rest');
jest.mock('azure-devops-node-api');
describe('Publishers', () => {
const mockTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
const logger = getVoidLogger();
it('should throw an error when the publisher for the source location is not registered', () => {
const publishers = new Publishers();
expect(() => publishers.get(mockTemplate)).toThrow(
expect(() => publishers.get('https://github.com/org/repo')).toThrow(
expect.objectContaining({
message: 'No publisher registered for type: "github"',
message:
'Unable to find a publisher for URL: https://github.com/org/repo. Please make sure to register this host under an integration in app-config',
}),
);
});
it('should return the correct preparer when the source matches', () => {
const publishers = new Publishers();
const publisher = new GithubPublisher({
client: new Octokit(),
token: 'fake',
repoVisibility: 'public',
});
publishers.register('github', publisher);
expect(publishers.get(mockTemplate)).toBe(publisher);
});
it('should throw an error if the metadata tag does not exist in the entity', () => {
const brokenTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: '.',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
it('should return the correct preparer when the source matches for github', async () => {
const publishers = await Publishers.fromConfig(
new ConfigReader({
integrations: {
github: [{ host: 'github.com', token: 'blob' }],
},
},
};
const publishers = new Publishers();
expect(() => publishers.get(brokenTemplate)).toThrow(
expect.objectContaining({
message: expect.stringContaining('No location annotation provided'),
}),
{
logger,
},
);
expect(publishers.get('https://github.com/org/repo')).toBeInstanceOf(
GithubPublisher,
);
});
it('should return the correct preparer when the source matches for azure', async () => {
const publishers = await Publishers.fromConfig(
new ConfigReader({
integrations: {
azure: [{ host: 'dev.azure.com', token: 'blob' }],
},
}),
{
logger,
},
);
expect(
publishers.get('https://dev.azure.com/org/project/_git/repo'),
).toBeInstanceOf(AzurePublisher);
});
it('should return the correct preparer when the source matches for bitbucket', async () => {
const publishers = await Publishers.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [{ host: 'bitbucket.com', token: 'blob' }],
},
}),
{
logger,
},
);
expect(publishers.get('https://bitbucket.org/owner/repo')).toBeInstanceOf(
BitbucketPublisher,
);
});
it('should return the correct preparer when the source matches for gitlab', async () => {
const publishers = await Publishers.fromConfig(
new ConfigReader({
integrations: {
gitlab: [{ host: 'gitlab.com', token: 'blob' }],
},
}),
{
logger,
},
);
expect(publishers.get('https://gitlab.com/owner/repo')).toBeInstanceOf(
GitlabPublisher,
);
});
it('should respect registrations for custom URLs for providers using the integrations config', async () => {
const publishers = await Publishers.fromConfig(
new ConfigReader({
integrations: {
github: [
{ host: 'my.special.github.enterprise.thing', token: 'lolghe' },
],
},
}),
{
logger,
},
);
expect(
publishers.get('https://my.special.github.enterprise.thing/org/repo'),
).toBeInstanceOf(GithubPublisher);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/publish/publishers.ts
+82 -136
View File
@@ -14,184 +14,130 @@
* limitations under the License.
*/
import { Logger } from 'winston';
import { Octokit } from '@octokit/rest';
import { Gitlab } from '@gitbeaker/node';
import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
import { Config } from '@backstage/config';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import {
DeprecatedLocationTypeDetector,
makeDeprecatedLocationTypeDetector,
parseLocationAnnotation,
} from '../helpers';
import { PublisherBase, PublisherBuilder } from './types';
import { RemoteProtocol } from '../types';
import { GithubPublisher, RepoVisibilityOptions } from './github';
import { GitlabPublisher } from './gitlab';
import { AzurePublisher } from './azure';
import { BitbucketPublisher } from './bitbucket';
import { Logger } from 'winston';
import { ScmIntegrations } from '@backstage/integration';
export class Publishers implements PublisherBuilder {
private publisherMap = new Map<RemoteProtocol, PublisherBase>();
private publisherMap = new Map<string, PublisherBase | undefined>();
constructor(private readonly typeDetector?: DeprecatedLocationTypeDetector) {}
register(protocol: RemoteProtocol, publisher: PublisherBase) {
this.publisherMap.set(protocol, publisher);
register(host: string, preparer: PublisherBase | undefined) {
this.publisherMap.set(host, preparer);
}
get(template: TemplateEntityV1alpha1): PublisherBase {
const { protocol, location } = parseLocationAnnotation(template);
const publisher = this.publisherMap.get(protocol);
if (!publisher) {
if ((protocol as string) === 'url') {
const type = this.typeDetector?.(location);
const detected = type && this.publisherMap.get(type as RemoteProtocol);
if (detected) {
return detected;
}
if (type) {
throw new Error(
`No publisher configuration available for type '${type}' with url "${location}". ` +
"Make sure you've added appropriate configuration in the 'scaffolder' configuration section",
);
} else {
throw new Error(
`Failed to detect publisher type. Unable to determine integration type for location "${location}". ` +
"Please add appropriate configuration to the 'integrations' configuration section",
);
}
}
throw new Error(`No publisher registered for type: "${protocol}"`);
get(url: string): PublisherBase {
const preparer = this.publisherMap.get(new URL(url).host);
if (!preparer) {
throw new Error(
`Unable to find a publisher for URL: ${url}. Please make sure to register this host under an integration in app-config`,
);
}
return publisher;
return preparer;
}
static async fromConfig(
config: Config,
{ logger }: { logger: Logger },
): Promise<PublisherBuilder> {
const typeDetector = makeDeprecatedLocationTypeDetector(config);
const publishers = new Publishers(typeDetector);
const publishers = new Publishers();
const githubConfig = config.getOptionalConfig('scaffolder.github');
if (githubConfig) {
try {
const repoVisibility = githubConfig.getString(
'visibility',
) as RepoVisibilityOptions;
const scm = ScmIntegrations.fromConfig(config);
const githubToken = githubConfig.getString('token');
const githubHost =
githubConfig.getOptionalString('host') ?? 'https://api.github.com';
const githubClient = new Octokit({
auth: githubToken,
baseUrl: githubHost,
});
const githubPublisher = new GithubPublisher({
client: githubClient,
token: githubToken,
repoVisibility,
});
const deprecationWarning = (name: string) => {
logger.warn(
`'Specifying credentials for ${name} in the Scaffolder configuration is deprecated. This will cause errors in a future release. Please migrate to using integrations config and specifying tokens under hostnames'`,
);
};
publishers.register('file', githubPublisher);
publishers.register('github', githubPublisher);
} catch (e) {
const providerName = 'github';
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
);
}
for (const integration of scm.azure.list()) {
const publisher = await AzurePublisher.fromConfig(integration.config);
if (publisher) {
publishers.register(integration.config.host, publisher);
} else {
deprecationWarning('Azure');
logger.warn(
`Skipping ${providerName} scaffolding provider, ${e.message}`,
publishers.register(
integration.config.host,
await AzurePublisher.fromConfig({
token: config.getOptionalString('scaffolder.azure.token'),
host: integration.config.host,
}),
);
}
}
const gitLabConfig = config.getOptionalConfig('scaffolder.gitlab');
if (gitLabConfig) {
try {
const gitLabToken = gitLabConfig.getConfig('api').getString('token');
const gitLabClient = new Gitlab({
host: gitLabConfig.getConfig('api').getOptionalString('baseUrl'),
token: gitLabToken,
});
const gitLabPublisher = new GitlabPublisher(gitLabClient, gitLabToken);
publishers.register('gitlab', gitLabPublisher);
publishers.register('gitlab/api', gitLabPublisher);
} catch (e) {
const providerName = 'gitlab';
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
);
}
for (const integration of scm.github.list()) {
const repoVisibility = (config.getOptionalString(
'scaffolder.github.visibility',
) ?? 'public') as RepoVisibilityOptions;
logger.warn(
`Skipping ${providerName} scaffolding provider, ${e.message}`,
const publisher = await GithubPublisher.fromConfig(integration.config, {
repoVisibility,
});
if (publisher) {
publishers.register(integration.config.host, publisher);
} else {
deprecationWarning('GitHub');
publishers.register(
integration.config.host,
await GithubPublisher.fromConfig(
{
token: config.getOptionalString('scaffolder.github.token') ?? '',
host: integration.config.host,
},
{ repoVisibility },
),
);
}
}
const azureConfig = config.getOptionalConfig('scaffolder.azure');
if (azureConfig) {
try {
const baseUrl = azureConfig.getString('baseUrl');
const azureToken = azureConfig.getConfig('api').getString('token');
for (const integration of scm.gitlab.list()) {
const publisher = await GitlabPublisher.fromConfig(integration.config);
const authHandler = getPersonalAccessTokenHandler(azureToken);
const webApi = new WebApi(baseUrl, authHandler);
const azureClient = await webApi.getGitApi();
if (publisher) {
publishers.register(integration.config.host, publisher);
} else {
deprecationWarning('Gitlab');
const azurePublisher = new AzurePublisher(azureClient, azureToken);
publishers.register('azure/api', azurePublisher);
} catch (e) {
const providerName = 'azure';
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
);
}
logger.warn(
`Skipping ${providerName} scaffolding provider, ${e.message}`,
publishers.register(
integration.config.host,
await GitlabPublisher.fromConfig({
token: config.getOptionalString('scaffolder.gitlab.token') ?? '',
host: integration.config.host,
}),
);
}
}
const bitbucketConfig = config.getOptionalConfig(
'scaffolder.bitbucket.api',
);
if (bitbucketConfig) {
try {
const baseUrl = bitbucketConfig.getString('host');
const bitbucketUsername = bitbucketConfig.getString('username');
const bitbucketToken = bitbucketConfig.getString('token');
for (const integration of scm.bitbucket.list()) {
const publisher = await BitbucketPublisher.fromConfig(integration.config);
const bitbucketPublisher = new BitbucketPublisher(
baseUrl,
bitbucketUsername,
bitbucketToken,
);
publishers.register('bitbucket', bitbucketPublisher);
} catch (e) {
const providerName = 'bitbucket';
if (process.env.NODE_ENV !== 'development') {
throw new Error(
`Failed to initialize ${providerName} scaffolding provider, ${e.message}`,
);
}
if (publisher) {
publishers.register(integration.config.host, publisher);
} else {
deprecationWarning('Bitbucket');
logger.warn(
`Skipping ${providerName} scaffolding provider, ${e.message}`,
publishers.register(
integration.config.host,
await BitbucketPublisher.fromConfig({
token: config.getOptionalString('scaffolder.bitbucket.token') ?? '',
username:
config.getOptionalString('scaffolder.bitbucket.username') ?? '',
appPassword:
config.getOptionalString('scaffolder.bitbucket.appPassword') ??
'',
host: integration.config.host,
}),
);
}
}
return publishers;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts
+3 -5
View File
@@ -13,10 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { RequiredTemplateValues } from '../templater';
import { JsonValue } from '@backstage/config';
import { RemoteProtocol } from '../types';
import { Logger } from 'winston';
/**
@@ -35,8 +33,8 @@ export type PublisherBase = {
export type PublisherOptions = {
values: RequiredTemplateValues & Record<string, JsonValue>;
logger: Logger;
directory: string;
logger: Logger;
};
export type PublisherResult = {
@@ -45,6 +43,6 @@ export type PublisherResult = {
};
export type PublisherBuilder = {
register(protocol: RemoteProtocol, publisher: PublisherBase): void;
get(template: TemplateEntityV1alpha1): PublisherBase;
register(host: string, publisher: PublisherBase): void;
get(storePath: string): PublisherBase;
};
plugins/scaffolder-backend/src/scaffolder/stages/types.ts
-22
View File
@@ -1,22 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type RemoteProtocol =
| 'file'
| 'github'
| 'gitlab'
| 'gitlab/api'
| 'azure/api'
| 'bitbucket';
plugins/scaffolder-backend/src/service/router.test.ts
+2 -2
View File
@@ -47,7 +47,7 @@ describe('createRouter - working directory', () => {
const mockPreparer = {
prepare: mockPrepare,
};
mockPreparers.register('azure/api', mockPreparer);
mockPreparers.register('dev.azure.com', mockPreparer);
});
beforeEach(() => {
@@ -65,7 +65,7 @@ describe('createRouter - working directory', () => {
kind: 'Template',
metadata: {
annotations: {
'backstage.io/managed-by-location': 'azure/api:dev.azure.com',
'backstage.io/managed-by-location': 'url:https://dev.azure.com',
},
},
spec: {
plugins/scaffolder-backend/src/service/router.ts
+14 -4
View File
@@ -27,6 +27,8 @@ import {
StageContext,
TemplaterBuilder,
PublisherBuilder,
parseLocationAnnotation,
FilePreparer,
} from '../scaffolder';
import { CatalogEntityClient } from '../lib/catalog';
import { validate, ValidatorResult } from 'jsonschema';
@@ -129,7 +131,15 @@ export async function createRouter(
{
name: 'Prepare the skeleton',
handler: async ctx => {
const preparer = preparers.get(ctx.entity);
const { protocol, location: pullPath } = parseLocationAnnotation(
ctx.entity,
);
const preparer =
protocol === 'file'
? new FilePreparer()
: preparers.get(pullPath);
const skeletonDir = await preparer.prepare(ctx.entity, {
logger: ctx.logger,
workingDirectory,
@@ -154,7 +164,7 @@ export async function createRouter(
{
name: 'Publish template',
handler: async (ctx: StageContext<{ resultDir: string }>) => {
const publisher = publishers.get(ctx.entity);
const publisher = publishers.get(ctx.values.storePath);
ctx.logger.info('Will now store the template');
const result = await publisher.publish({
values: ctx.values,
@@ -167,9 +177,9 @@ export async function createRouter(
],
});
res.status(201).json({ id: job.id });
jobProcessor.run(job);
res.status(201).json({ id: job.id });
});
const app = express();
plugins/scaffolder/package.json
+1
View File
@@ -40,6 +40,7 @@
"@rjsf/core": "^2.4.0",
"@rjsf/material-ui": "^2.4.0",
"classnames": "^2.2.6",
"git-url-parse": "^11.4.3",
"moment": "^2.26.0",
"react": "^16.13.1",
"react-dom": "^16.13.1",
plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx
+31 -8
View File
@@ -39,6 +39,7 @@ import { rootRoute } from '../../routes';
import { JobStatusModal } from '../JobStatusModal';
import { MultistepJsonForm } from '../MultistepJsonForm';
import { useJobPolling } from '../hooks/useJobPolling';
import parseGitUrl from 'git-url-parse';
const useTemplate = (
templateName: string,
@@ -63,10 +64,10 @@ const OWNER_REPO_SCHEMA = {
description: 'Who is going to own this component',
},
storePath: {
format: 'GitHub user or org / Repo name',
type: 'string' as const,
title: 'Store path',
description: 'GitHub store path in org/repo format',
description:
'A full URL to the repository that should be created. e.g https://github.com/backstage/new-repo',
},
access: {
type: 'string' as const,
@@ -75,11 +76,6 @@ const OWNER_REPO_SCHEMA = {
},
},
};
const REPO_FORMAT = {
'GitHub user or org / Repo name': /[^\/]*\/[^\/]*/,
};
export const TemplatePage = () => {
const errorApi = useApi(errorApiRef);
const catalogApi = useApi(catalogApiRef);
@@ -182,7 +178,34 @@ export const TemplatePage = () => {
{
label: 'Choose owner and repo',
schema: OWNER_REPO_SCHEMA,
customFormats: REPO_FORMAT,
validate: (formData, errors) => {
const { storePath } = formData;
try {
const parsedUrl = parseGitUrl(storePath);
if (
!parsedUrl.resource ||
!parsedUrl.owner ||
!parsedUrl.name
) {
if (parsedUrl.resource === 'dev.azure.com') {
errors.storePath.addError(
"The store path should be formatted like https://dev.azure.com/{org}/{project}/_git/{repo} for Azure URL's",
);
} else {
errors.storePath.addError(
'The store path should be a complete Git URL to the new repository location. For example: https://github.com/{owner}/{repo}',
);
}
}
} catch (ex) {
errors.storePath.addError(
`Failed validation of the store pathn with message ${ex.message}`,
);
}
return errors;
},
},
]}
/>