Merge pull request #4270 from backstage/mob/stages-refactor

Refactor scaffolder to operate on known directories
This commit is contained in:
Johan Haals
2021-01-29 10:58:47 +01:00
committed by GitHub
35 changed files with 748 additions and 934 deletions
@@ -70,7 +70,7 @@ describe('JobProcessor', () => {
describe('create', () => {
it('creates should create a new job with a unique id', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -84,7 +84,7 @@ describe('JobProcessor', () => {
});
it('should setup the correct context for the job', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -97,7 +97,7 @@ describe('JobProcessor', () => {
});
it('should set the status as pending', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -120,7 +120,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -139,12 +139,12 @@ describe('JobProcessor', () => {
describe('get', () => {
it('return undefined for when the job does not exist', () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
expect(processor.get('123')).not.toBeDefined();
});
it('should return the exact same instance of the job when one is created', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -156,7 +156,7 @@ describe('JobProcessor', () => {
});
describe('process', () => {
it('throws an error when the status of the job is not in pending state', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -182,7 +182,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -208,7 +208,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -244,7 +244,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -283,7 +283,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -13,13 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import { Processor, Job, StageContext, StageInput } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import * as uuid from 'uuid';
import Docker from 'dockerode';
import path from 'path';
import { TemplaterValues, TemplaterBase } from '../stages/templater';
import { PreparerBuilder } from '../stages/prepare';
import { makeLogStream } from './logger';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
export type JobProcessorArguments = {
preparers: PreparerBuilder;
@@ -33,7 +39,45 @@ export type JobAndDirectoryTuple = {
};
export class JobProcessor implements Processor {
private jobs = new Map<string, Job>();
private readonly workingDirectory: string;
private readonly jobs: Map<string, Job>;
static async fromConfig({
config,
logger,
}: {
config: Config;
logger: Logger;
}) {
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
} else {
workingDirectory = os.tmpdir();
}
return new JobProcessor(workingDirectory);
}
constructor(workingDirectory: string) {
this.workingDirectory = workingDirectory;
this.jobs = new Map<string, Job>();
}
create({
entity,
@@ -52,6 +96,7 @@ export class JobProcessor implements Processor {
values,
logger,
logStream: stream,
workspacePath: path.join(this.workingDirectory, id),
};
const job: Job = {
@@ -80,6 +125,8 @@ export class JobProcessor implements Processor {
throw new Error("Job is not in a 'PENDING' state");
}
await fs.mkdir(job.context.workspacePath);
job.status = 'STARTED';
try {
@@ -134,6 +181,8 @@ export class JobProcessor implements Processor {
// If something went wrong, fail the job, and set the error property on the job.
job.error = { name: error.name, message: error.message };
job.status = 'FAILED';
} finally {
await fs.remove(job.context.workspacePath);
}
}
}
@@ -26,6 +26,7 @@ export type StageContext<T = {}> = {
entity: TemplateEntityV1alpha1;
logger: Logger;
logStream: Writable;
workspacePath: string;
} & T;
export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseLocationAnnotation } from './helpers';
import { parseLocationAnnotation, joinGitUrlPath } from './helpers';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
@@ -73,7 +73,7 @@ describe('Helpers', () => {
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
':https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
':https://github.com/o/r/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
@@ -250,4 +250,76 @@ describe('Helpers', () => {
});
});
});
describe('joinGitUrlPath', () => {
it.each([
[
'https://github.com/o/r/blob/master/template.yaml',
'template',
'https://github.com/o/r/blob/master/template',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml',
undefined,
'https://dev.azure.com/o/p/_git/template-repo?path=%2F',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml',
'a',
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Ftemplate.yaml',
'b',
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Fb',
],
[
'https://github.com/o/r/blob/master/template.yaml',
undefined,
'https://github.com/o/r/blob/master',
],
[
'https://github.com/o/r/blob/master/template.yaml',
'template',
'https://github.com/o/r/blob/master/template',
],
[
'https://github.com/o/r/blob/master/templates/graphql-starter/template.yaml',
'template',
'https://github.com/o/r/blob/master/templates/graphql-starter/template',
],
[
'https://gitlab.com/o/r/-/blob/master/template.yaml',
undefined,
'https://gitlab.com/o/r/-/blob/master',
],
[
'https://gitlab.com/o/r/-/blob/master/template.yaml',
'template',
'https://gitlab.com/o/r/-/blob/master/template',
],
[
'https://gitlab.com/o/r/-/blob/master/a/b/c/template.yaml',
'../../c',
'https://gitlab.com/o/r/-/blob/master/a/c',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
undefined,
'https://bitbucket.org/p/r/src/master/a/b',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
'c',
'https://bitbucket.org/p/r/src/master/a/b/c',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
'../c',
'https://bitbucket.org/p/r/src/master/a/c',
],
])('should join git url %s with path %s', (url, path, result) => {
expect(joinGitUrlPath(url, path)).toBe(result);
});
});
});
@@ -13,6 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { posix as posixPath } from 'path';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
@@ -53,3 +55,20 @@ export const parseLocationAnnotation = (
location,
};
};
export function joinGitUrlPath(repoUrl: string, path?: string): string {
const parsed = new URL(repoUrl);
if (parsed.hostname.endsWith('azure.com')) {
const templatePath = posixPath.normalize(
posixPath.join(
posixPath.dirname(parsed.searchParams.get('path') || '/'),
path || '.',
),
);
parsed.searchParams.set('path', templatePath);
return parsed.toString();
}
return new URL(path || '.', repoUrl).toString().replace(/\/$/, '');
}
@@ -14,19 +14,14 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { AzurePreparer } from './azure';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
jest.mock('fs-extra');
describe('AzurePreparer', () => {
const mockGitClient = {
clone: jest.fn(),
@@ -36,121 +31,87 @@ describe('AzurePreparer', () => {
jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
let mockEntity: TemplateEntityV1alpha1;
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = AzurePreparer.fromConfig({
host: 'dev.azure.com',
token: 'fake-azure-token',
});
// TODO(blam): Here's a test that will fail when the deprecation is complete
it('calls the clone command with deprecated token', async () => {
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
password: 'fake-azure-token',
username: 'notempty',
});
});
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const prepareOptions = {
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
workspacePath,
logger,
};
it('calls the clone command with token from integrations config', async () => {
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
password: 'fake-azure-token',
username: 'notempty',
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: expect.any(String),
dir: checkoutPath,
});
});
it('calls the clone command with the correct arguments for a repository with a specified branch', async () => {
await preparer.prepare({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: checkoutPath,
ref: 'master',
});
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
workspacePath,
logger,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: expect.any(String),
ref: 'master',
dir: checkoutPath,
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
});
it('return the temp directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
it('moves the template from path if it is specified', async () => {
const path = './subdir';
await preparer.prepare({
url: `https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=${encodeURIComponent(
path,
)}`,
logger,
workspacePath,
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, 'subdir'),
templatePath,
);
});
});
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,26 +27,13 @@ export class AzurePreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath ?? '',
);
// Username can be anything but the empty string according to:
@@ -63,11 +47,17 @@ export class AzurePreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
ref: parsedGitUrl.ref,
dir: checkoutPath,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -14,21 +14,15 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import { BitbucketPreparer } from './bitbucket';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
import { resolve } from 'path';
import os from 'os';
jest.mock('fs-extra');
describe('BitbucketPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const logger = getVoidLogger();
const mockGitClient = {
clone: jest.fn(),
@@ -38,44 +32,6 @@ describe('BitbucketPreparer', () => {
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'bitbucket:https://bitbucket.org/backstage-project/backstage-repo',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = BitbucketPreparer.fromConfig({
@@ -84,13 +40,25 @@ describe('BitbucketPreparer', () => {
appPassword: 'fake-password',
});
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const prepareOptions = {
url: 'https://bitbucket.org/backstage-project/backstage-repo',
logger,
workspacePath,
};
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => {
@@ -99,7 +67,7 @@ describe('BitbucketPreparer', () => {
username: 'fake-user',
appPassword: 'fake-password',
});
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
@@ -109,23 +77,23 @@ describe('BitbucketPreparer', () => {
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
});
it('return the temp directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
it('moves a template subdirectory to checkout if specified', async () => {
await preparer.prepare({
url: 'https://bitbucket.org/foo/bar/src/master/1/2/3',
logger,
workspacePath,
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, '1', '2', '3'),
templatePath,
);
});
@@ -135,7 +103,7 @@ describe('BitbucketPreparer', () => {
token: 'fake-token',
});
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
@@ -143,16 +111,4 @@ describe('BitbucketPreparer', () => {
password: 'fake-token',
});
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
});
});
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import { BitbucketIntegrationConfig } from '@backstage/integration';
@@ -40,44 +37,30 @@ export class BitbucketPreparer implements PreparerBase {
},
) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const auth = this.getAuth();
const git = auth
? Git.fromAuth({
...auth,
logger,
})
: Git.fromAuth({ logger });
const git = Git.fromAuth({ logger, ...this.getAuth() });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
private getAuth(): { username: string; password: string } | undefined {
@@ -14,57 +14,43 @@
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import YAML from 'yaml';
import { FilePreparer } from './file';
import path from 'path';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger } from '@backstage/backend-common';
import fs from 'fs-extra';
import { FilePreparer } from './file';
import os from 'os';
import { resolve } from 'path';
const setupTest = async (fixturePath: string) => {
const locationForTemplateYaml = path.resolve(
path.dirname(
require.resolve('@backstage/plugin-scaffolder-backend/package'),
),
'fixtures',
fixturePath,
);
const [parsedDocument] = YAML.parseAllDocuments(
await fs.readFile(locationForTemplateYaml, 'utf-8'),
);
const template: TemplateEntityV1alpha1 = parsedDocument.toJSON();
template.metadata.annotations = {
[LOCATION_ANNOTATION]: `file:${locationForTemplateYaml}`,
};
const filePreparer = new FilePreparer();
const resultDir = await filePreparer.prepare(template, {
logger: getVoidLogger(),
workingDirectory: os.tmpdir(),
});
return { filePreparer, template, resultDir };
};
jest.mock('fs-extra');
describe('File preparer', () => {
it('excludes the yaml file from the temp folder', async () => {
const { resultDir } = await setupTest('test-simple-template/template.yaml');
expect(fs.existsSync(`${resultDir}/template.yaml`)).toBe(false);
});
it('prepares templates from a file path', async () => {
const logger = getVoidLogger();
const preparer = new FilePreparer();
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
it('resolves relative path from the template', async () => {
const { resultDir } = await setupTest('test-simple-template/template.yaml');
expect(fs.existsSync(`${resultDir}/expected_file.ts`)).toBe(true);
});
await preparer.prepare({
url: 'file:///path/to/template',
logger,
workspacePath,
});
expect(fs.copy).toHaveBeenCalledWith(
resolve('/path', 'to', 'template'),
checkoutPath,
{
recursive: true,
},
);
expect(fs.ensureDir).toHaveBeenCalledWith(checkoutPath);
it('resolves relative path from the nested template', async () => {
const { resultDir } = await setupTest('test-nested-template/template.yaml');
expect(fs.existsSync(`${resultDir}/expected_file.ts`)).toBe(true);
await expect(
preparer.prepare({
url: 'file://not/full/path',
logger,
workspacePath,
}),
).rejects.toThrow(
"Wrong location protocol, should be 'file', file://not/full/path",
);
});
});
@@ -13,43 +13,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
export class FilePreparer implements PreparerBase {
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
if (protocol !== 'file') {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'file'`,
);
async prepare({ url, workspacePath }: PreparerOptions) {
if (!url.startsWith('file:///')) {
throw new InputError(`Wrong location protocol, should be 'file', ${url}`);
}
const templateId = template.metadata.name;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const checkoutDir = path.join(workspacePath, 'checkout');
await fs.ensureDir(checkoutDir);
const parentDirectory = path.resolve(
path.dirname(location),
template.spec.path ?? '.',
);
const templatePath = url.slice('file://'.length);
await fs.copy(parentDirectory, tempDir, {
filter: src => src !== location,
await fs.copy(templatePath, checkoutDir, {
recursive: true,
});
return tempDir;
}
}
@@ -14,21 +14,19 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { GithubPreparer } from './github';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
jest.mock('fs-extra');
describe('GitHubPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const mockGitClient = {
clone: jest.fn(),
};
@@ -38,44 +36,6 @@ describe('GitHubPreparer', () => {
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = GithubPreparer.fromConfig({
@@ -84,60 +44,49 @@ describe('GitHubPreparer', () => {
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master/templates/graphql-starter/template',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://github.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, 'templates', 'graphql-starter', 'template'),
templatePath,
);
expect(fs.rmdir).toHaveBeenCalledWith('/tmp/template/.git');
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://github.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
ref: expect.any(String),
dir: checkoutPath,
ref: 'master',
});
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with token', async () => {
await preparer.prepare(mockEntity, { logger });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master',
logger,
workspacePath,
});
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,30 +27,15 @@ export class GithubPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const git = this.config.token
? Git.fromAuth({
username: this.config.token,
@@ -63,11 +45,17 @@ export class GithubPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -13,60 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { GitlabPreparer } from './gitlab';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
const mockTemplate = (): TemplateEntityV1alpha1 => ({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'url:https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
});
jest.mock('fs-extra');
describe('GitLabPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const mockGitClient = {
clone: jest.fn(),
};
@@ -81,63 +40,41 @@ describe('GitLabPreparer', () => {
host: 'gitlab.com',
token: 'fake-token',
});
it(`calls the clone command with the correct arguments for a repository`, async () => {
mockEntity = mockTemplate();
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
it(`calls the clone command with the correct arguments for a repository`, async () => {
await preparer.prepare({
url:
'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
});
it(`calls the clone command with the correct arguments if an access token is provided in integrations for a repository`, async () => {
mockEntity = mockTemplate();
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'oauth2',
password: 'fake-token',
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it(`calls the clone command with the correct arguments for a repository when no path is provided`, async () => {
mockEntity = mockTemplate();
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git',
dir: expect.any(String),
ref: expect.any(String),
it(`clones the template from a sub directory if specified`, async () => {
await preparer.prepare({
url:
'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/1/2/3',
logger,
workspacePath,
});
});
it(`return the temp directory with the path to the folder if it is specified`, async () => {
mockEntity = mockTemplate();
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, '1', '2', '3'),
templatePath,
);
});
});
@@ -14,13 +14,10 @@
* limitations under the License.
*/
import { Git } from '@backstage/backend-common';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { GitLabIntegrationConfig } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
import path from 'path';
import { parseLocationAnnotation } from '../helpers';
import { PreparerBase, PreparerOptions } from './types';
export class GitlabPreparer implements PreparerBase {
@@ -30,28 +27,15 @@ export class GitlabPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const logger = opts.logger;
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
parsedGitLocation.git_suffix = true;
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
parsedGitUrl.git_suffix = true;
const git = this.config.token
? Git.fromAuth({
@@ -62,11 +46,17 @@ export class GitlabPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -13,24 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
export type PreparerOptions = {
workingDirectory?: string;
/**
* Full URL to the directory containg template data
*/
url: string;
/**
* The workspace path that will eventually be the the root of the new repo
*/
workspacePath: string;
logger: Logger;
};
export interface PreparerBase {
/**
* Given an Entity definition from the Service Catalog, go and prepare a directory
* with contents from the remote location in temporary storage and return the path
* @param template The template entity from the Service Catalog
* Prepare a directory with contents from the remote location
*/
prepare(
template: TemplateEntityV1alpha1,
opts?: PreparerOptions,
): Promise<string>;
prepare(opts: PreparerOptions): Promise<void>;
}
export type PreparerBuilder = {
@@ -53,7 +53,7 @@ describe('Azure Publisher', () => {
storePath: 'https://dev.azure.com/organisation/project/_git/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -74,7 +74,7 @@ describe('Azure Publisher', () => {
'project',
);
expect(helpers.initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
auth: { username: 'notempty', password: 'fake-azure-token' },
logger,
@@ -21,6 +21,7 @@ import { initRepoAndPush } from './helpers';
import { AzureIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
import path from 'path';
export class AzurePublisher implements PublisherBase {
static async fromConfig(config: AzureIntegrationConfig) {
@@ -34,7 +35,7 @@ export class AzurePublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name, organization, resource } = parseGitUrl(
@@ -56,7 +57,7 @@ export class AzurePublisher implements PublisherBase {
const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`;
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: 'notempty',
@@ -69,7 +69,7 @@ describe('Bitbucket Publisher', () => {
storePath: 'https://bitbucket.org/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger: logger,
});
@@ -80,7 +80,7 @@ describe('Bitbucket Publisher', () => {
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://bitbucket.org/project/repo',
auth: { username: 'fake-user', password: 'fake-token' },
logger: logger,
@@ -127,7 +127,7 @@ describe('Bitbucket Publisher', () => {
storePath: 'https://bitbucket.mycompany.com/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger: logger,
});
@@ -138,7 +138,7 @@ describe('Bitbucket Publisher', () => {
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo',
auth: { username: 'x-token-auth', password: 'fake-token' },
logger: logger,
@@ -19,6 +19,7 @@ import { initRepoAndPush } from './helpers';
import fetch from 'cross-fetch';
import { BitbucketIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import path from 'path';
// TODO(blam): We should probably start to use a bitbucket client here that we can change
// the baseURL to point at on-prem or public bitbucket versions like we do for
@@ -45,7 +46,7 @@ export class BitbucketPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner: project, name } = parseGitUrl(values.storePath);
@@ -58,7 +59,7 @@ export class BitbucketPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl: result.remoteUrl,
auth: {
username: this.config.username ? this.config.username : 'x-token-auth',
@@ -64,7 +64,7 @@ describe('GitHub Publisher', () => {
owner: 'bob',
access: 'blam/team',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -89,7 +89,7 @@ describe('GitHub Publisher', () => {
permission: 'admin',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -122,7 +122,7 @@ describe('GitHub Publisher', () => {
owner: 'bob',
access: 'blam',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -140,7 +140,7 @@ describe('GitHub Publisher', () => {
expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled();
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -175,7 +175,7 @@ describe('GitHub Publisher', () => {
access: 'bob',
description: 'description',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -198,7 +198,7 @@ describe('GitHub Publisher', () => {
permission: 'admin',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -233,7 +233,7 @@ describe('GitHub Publisher', () => {
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -249,7 +249,7 @@ describe('GitHub Publisher', () => {
visibility: 'internal',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -283,7 +283,7 @@ describe('GitHub Publisher', () => {
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -299,7 +299,7 @@ describe('GitHub Publisher', () => {
private: true,
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -19,8 +19,10 @@ import { initRepoAndPush } from './helpers';
import { GitHubIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { Octokit } from '@octokit/rest';
import path from 'path';
export type RepoVisibilityOptions = 'private' | 'internal' | 'public';
export class GithubPublisher implements PublisherBase {
static async fromConfig(
config: GitHubIntegrationConfig,
@@ -41,6 +43,7 @@ export class GithubPublisher implements PublisherBase {
repoVisibility,
});
}
constructor(
private readonly config: {
token: string;
@@ -51,7 +54,7 @@ export class GithubPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name } = parseGitUrl(values.storePath);
@@ -66,7 +69,7 @@ export class GithubPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: this.config.token,
@@ -79,7 +82,6 @@ export class GithubPublisher implements PublisherBase {
/\.git$/,
'/blob/master/catalog-info.yaml',
);
return { remoteUrl, catalogInfoUrl };
}
@@ -68,7 +68,7 @@ describe('GitLab Publisher', () => {
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -85,7 +85,7 @@ describe('GitLab Publisher', () => {
name: 'test',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'mockclone',
auth: { username: 'oauth2', password: 'fake-token' },
logger,
@@ -111,7 +111,7 @@ describe('GitLab Publisher', () => {
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -125,7 +125,7 @@ describe('GitLab Publisher', () => {
name: 'test',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'mockclone',
auth: { username: 'oauth2', password: 'fake-token' },
logger,
@@ -19,7 +19,7 @@ import { Gitlab } from '@gitbeaker/node';
import { Gitlab as GitlabClient } from '@gitbeaker/core';
import { initRepoAndPush } from './helpers';
import parseGitUrl from 'git-url-parse';
import path from 'path';
import { GitLabIntegrationConfig } from '@backstage/integration';
export class GitlabPublisher implements PublisherBase {
@@ -38,7 +38,7 @@ export class GitlabPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name } = parseGitUrl(values.storePath);
@@ -49,7 +49,7 @@ export class GitlabPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: 'oauth2',
@@ -32,7 +32,7 @@ export type PublisherBase = {
export type PublisherOptions = {
values: TemplaterValues;
directory: string;
workspacePath: string;
logger: Logger;
};
@@ -13,48 +13,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('./helpers', () => ({
runDockerContainer: jest.fn(),
runCommand: jest.fn(),
}));
jest.mock('command-exists-promise', () => jest.fn());
const runDockerContainer = jest.fn();
const runCommand = jest.fn();
const commandExists = jest.fn();
jest.mock('./helpers', () => ({ runDockerContainer, runCommand }));
jest.mock('command-exists-promise', () => commandExists);
jest.mock('fs-extra');
import { CookieCutter } from './cookiecutter';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { RunDockerContainerOptions, RunCommandOptions } from './helpers';
import { PassThrough } from 'stream';
import Docker from 'dockerode';
import parseGitUrl from 'git-url-parse';
const commandExists = require('command-exists-promise');
describe('CookieCutter Templater', () => {
const cookie = new CookieCutter();
const mockDocker = {} as Docker;
const {
runDockerContainer,
}: {
runDockerContainer: jest.Mock<RunDockerContainerOptions>;
} = require('./helpers');
jest
.spyOn(fs, 'readdir')
.mockImplementation(() => Promise.resolve(['newthing']));
beforeEach(async () => {
beforeEach(() => {
jest.clearAllMocks();
});
const mkTemp = async () => {
const tempDir = os.tmpdir();
return await fs.promises.mkdtemp(path.join(tempDir, 'temp'));
};
it('should write a cookiecutter.json file with the values from the entity', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -65,22 +46,30 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(cookieCutterJson).toEqual(
expect.objectContaining(JSON.parse(JSON.stringify(values))),
expect(fs.writeJson).toBeCalledWith(
'tempdir/template/cookiecutter.json',
expect.objectContaining(values),
);
});
it('should merge any value that is in the cookiecutter.json path already', async () => {
const tempdir = await mkTemp();
const existingJson = {
_copy_without_render: ['./github/workflows/*'],
};
await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
jest
.spyOn(fs, 'readJSON')
.mockImplementationOnce(() => Promise.resolve(existingJson));
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const values = {
owner: 'blobby',
@@ -91,11 +80,14 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
expect(cookieCutterJson).toEqual({
expect(fs.writeJSON).toBeCalledWith('tempdir/template/cookiecutter.json', {
...existingJson,
...values,
destination: {
@@ -105,9 +97,9 @@ describe('CookieCutter Templater', () => {
});
it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
const tempdir = await mkTemp();
await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
jest.spyOn(fs, 'readJSON').mockImplementationOnce(() => {
throw new Error('BAM');
});
const values = {
owner: 'blobby',
@@ -117,14 +109,17 @@ describe('CookieCutter Templater', () => {
},
};
const templater = new CookieCutter();
await expect(
cookie.run({ directory: tempdir, values, dockerClient: mockDocker }),
).rejects.toThrow(/Unexpected token ' in JSON at position 1/);
templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
}),
).rejects.toThrow('BAM');
});
it('should run the correct docker container with the correct bindings for the volumes', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -134,7 +129,14 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(runDockerContainer).toHaveBeenCalledWith({
imageName: 'spotify/backstage-cookiecutter',
@@ -146,39 +148,16 @@ describe('CookieCutter Templater', () => {
'/template',
'--verbose',
],
templateDir: tempdir,
resultDir: expect.stringContaining(`${tempdir}-result`),
templateDir: 'tempdir/template',
resultDir: 'tempdir/intermediate',
logStream: undefined,
dockerClient: mockDocker,
});
});
it('should return the result path to the end templated folder', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
component_id: 'newthing',
destination: {
git: parseGitUrl('https://github.com/org/repo'),
},
};
const { resultDir } = await cookie.run({
directory: tempdir,
values,
dockerClient: mockDocker,
});
expect(resultDir.startsWith(`${tempdir}-result`)).toBeTruthy();
});
it('should pass through the streamer to the run docker helper', async () => {
const stream = new PassThrough();
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -188,8 +167,11 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({
directory: tempdir,
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
@@ -205,29 +187,17 @@ describe('CookieCutter Templater', () => {
'/template',
'--verbose',
],
templateDir: tempdir,
resultDir: expect.stringContaining(`${tempdir}-result`),
templateDir: 'tempdir/template',
resultDir: 'tempdir/intermediate',
logStream: stream,
dockerClient: mockDocker,
});
});
describe('when cookiecutter is available', () => {
beforeAll(() => {
commandExists.mockImplementation(() => () => true);
});
it('use the binary', async () => {
const {
runCommand,
}: {
runCommand: jest.Mock<RunCommandOptions>;
} = require('./helpers');
const stream = new PassThrough();
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -237,8 +207,12 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({
directory: tempdir,
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
commandExists.mockImplementationOnce(() => () => true);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
@@ -249,8 +223,8 @@ describe('CookieCutter Templater', () => {
args: expect.arrayContaining([
'--no-input',
'-o',
tempdir,
expect.stringContaining(`${tempdir}-result`),
'tempdir/intermediate',
'tempdir/template',
'--verbose',
]),
logStream: stream,
@@ -259,18 +233,17 @@ describe('CookieCutter Templater', () => {
});
describe('when nothing was generated', () => {
beforeEach(() => {
jest.spyOn(fs, 'readdir').mockImplementation(() => Promise.resolve([]));
});
it('throws an error', async () => {
const stream = new PassThrough();
const tempdir = await mkTemp();
jest
.spyOn(fs, 'readdir')
.mockImplementationOnce(() => Promise.resolve([]));
return expect(
cookie.run({
directory: tempdir,
const templater = new CookieCutter();
await expect(
templater.run({
workspacePath: 'tempdir',
values: {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -281,7 +254,7 @@ describe('CookieCutter Templater', () => {
logStream: stream,
dockerClient: mockDocker,
}),
).rejects.toThrow(/Cookie Cutter did not generate anything/);
).rejects.toThrow(/No data generated by cookiecutter/);
});
});
});
@@ -19,8 +19,6 @@ import { runDockerContainer, runCommand } from './helpers';
import { TemplaterBase, TemplaterRunOptions } from '.';
import path from 'path';
import { TemplaterRunResult } from './types';
const commandExists = require('command-exists-promise');
export class CookieCutter implements TemplaterBase {
@@ -38,28 +36,32 @@ export class CookieCutter implements TemplaterBase {
}
}
public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult> {
public async run({
workspacePath,
dockerClient,
values,
logStream,
}: TemplaterRunOptions): Promise<void> {
const templateDir = path.join(workspacePath, 'template');
const intermediateDir = path.join(workspacePath, 'intermediate');
const resultDir = path.join(workspacePath, 'result');
// First lets grab the default cookiecutter.json file
const cookieCutterJson = await this.fetchTemplateCookieCutter(
options.directory,
);
const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir);
const cookieInfo = {
...cookieCutterJson,
...options.values,
...values,
};
await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
const templateDir = options.directory;
const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
await fs.writeJSON(`${templateDir}/cookiecutter.json`, cookieInfo);
const cookieCutterInstalled = await commandExists('cookiecutter');
if (cookieCutterInstalled) {
await runCommand({
command: 'cookiecutter',
args: ['--no-input', '-o', resultDir, templateDir, '--verbose'],
logStream: options.logStream,
args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'],
logStream,
});
} else {
await runDockerContainer({
@@ -73,22 +75,20 @@ export class CookieCutter implements TemplaterBase {
'--verbose',
],
templateDir,
resultDir,
logStream: options.logStream,
dockerClient: options.dockerClient,
resultDir: intermediateDir,
logStream,
dockerClient,
});
}
// if cookiecutter was successful, resultDir will contain
// if cookiecutter was successful, intermediateDir will contain
// exactly one directory.
const [generated] = await fs.readdir(resultDir);
const [generated] = await fs.readdir(intermediateDir);
if (generated === undefined) {
throw new Error('Cookie Cutter did not generate anything');
throw new Error('No data generated by cookiecutter');
}
return {
resultDir: path.resolve(resultDir, generated),
};
await fs.move(path.join(intermediateDir, generated), resultDir);
}
}
@@ -17,25 +17,30 @@ import fs from 'fs-extra';
import { runDockerContainer } from '../helpers';
import { TemplaterBase, TemplaterRunOptions } from '..';
import path from 'path';
import { TemplaterRunResult } from '../types';
import * as yaml from 'yaml';
import { resolvePackagePath } from '@backstage/backend-common';
// TODO(blam): Replace with the universal import from github-actions after a release
// As it will break the E2E without it
const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug';
export class CreateReactAppTemplater implements TemplaterBase {
public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult> {
public async run({
workspacePath,
values,
logStream,
dockerClient,
}: TemplaterRunOptions): Promise<void> {
const {
component_id: componentName,
use_typescript: withTypescript,
use_github_actions: withGithubActions,
description,
owner,
} = options.values;
} = values;
const intermediateDir = path.join(workspacePath, 'template');
await fs.ensureDir(intermediateDir);
const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
const resultDir = path.join(workspacePath, 'result');
await runDockerContainer({
imageName: 'node:lts-alpine',
@@ -44,35 +49,80 @@ export class CreateReactAppTemplater implements TemplaterBase {
componentName as string,
withTypescript ? ' --template typescript' : '',
],
templateDir: options.directory,
resultDir,
logStream: options.logStream,
dockerClient: options.dockerClient,
templateDir: intermediateDir,
resultDir: intermediateDir,
logStream: logStream,
dockerClient: dockerClient,
createOptions: {
Entrypoint: ['npx'],
WorkingDir: '/result',
},
});
const extraAnnotations: Record<string, string> = {};
const finalDir = path.resolve(
resultDir,
options.values.component_id as string,
);
// if cookiecutter was successful, intermediateDir will contain
// exactly one directory.
const [generated] = await fs.readdir(intermediateDir);
if (generated === undefined) {
throw new Error('No data generated by cookiecutter');
}
await fs.move(path.join(intermediateDir, generated), resultDir);
const extraAnnotations: Record<string, string> = {};
if (withGithubActions) {
await fs.promises.mkdir(`${finalDir}/.github`);
await fs.promises.mkdir(`${finalDir}/.github/workflows`);
await fs.promises.copyFile(
`${resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
)}/templates/.github/workflows/main.yml`,
`${finalDir}/.github/workflows/main.yml`,
await fs.mkdir(`${resultDir}/.github`);
await fs.mkdir(`${resultDir}/.github/workflows`);
const githubActionsYaml = `
name: CRA Build
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [12.x]
steps:
- name: checkout code
uses: actions/checkout@v1
- name: get yarn cache
id: yarn-cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: \${{ steps.yarn-cache.outputs.dir }}
key: \${{ runner.os }}-yarn-\${{ hashFiles('**/yarn.lock') }}
restore-keys: |
\${{ runner.os }}-yarn-
- name: use node.js \${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: \${{ matrix.node-version }}
- name: yarn install, build, and test
working-directory: .
run: |
yarn install
yarn build
yarn test
env:
CI: true
`;
await fs.writeFile(
`${resultDir}/.github/workflows/main.yml`,
githubActionsYaml,
);
extraAnnotations[
GITHUB_ACTIONS_ANNOTATION
] = `${options.values?.destination?.git?.owner}/${options.values?.destination?.git?.name}`;
] = `${values?.destination?.git?.owner}/${values?.destination?.git?.name}`;
}
const componentInfo = {
@@ -92,13 +142,9 @@ export class CreateReactAppTemplater implements TemplaterBase {
},
};
await fs.promises.writeFile(
`${finalDir}/catalog-info.yaml`,
await fs.writeFile(
`${resultDir}/catalog-info.yaml`,
yaml.stringify(componentInfo),
);
return {
resultDir: finalDir,
};
}
}
@@ -1,41 +0,0 @@
name: CRA Build
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [12.x]
steps:
- name: checkout code
uses: actions/checkout@v1
- name: get yarn cache
id: yarn-cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn-cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: use node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: yarn install, build, and test
working-directory: .
run: |
yarn install
yarn build
yarn test
env:
CI: true
@@ -14,52 +14,13 @@
* limitations under the License.
*/
import { Templaters } from '.';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { CookieCutter } from './cookiecutter';
describe('Templaters', () => {
const mockTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'file:/Users/bingo/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
templater: 'cookiecutter',
path: '.',
type: 'website',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
it('should throw an error when the templater is not registered', () => {
const templaters = new Templaters();
expect(() => templaters.get(mockTemplate)).toThrow(
expect(() => templaters.get('cookiecutter')).toThrow(
expect.objectContaining({
message: 'No templater registered for template: "cookiecutter"',
}),
@@ -71,55 +32,6 @@ describe('Templaters', () => {
templaters.register('cookiecutter', templater);
expect(templaters.get(mockTemplate)).toBe(templater);
});
it('should throw an error if the templater does not exist in the entity', () => {
const brokenTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
type: 'website',
path: '.',
templater: '',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
const templaters = new Templaters();
expect(() => templaters.get(brokenTemplate)).toThrow(
expect.objectContaining({
name: 'InputError',
message: expect.stringContaining(
'Template does not have a required templating key',
),
}),
);
expect(templaters.get('cookiecutter')).toBe(templater);
});
});
@@ -20,26 +20,20 @@ import {
TemplaterBuilder,
} from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { getTemplaterKey } from './helpers';
export class Templaters implements TemplaterBuilder {
private preparerMap = new Map<SupportedTemplatingKey, TemplaterBase>();
private templaterMap = new Map<SupportedTemplatingKey, TemplaterBase>();
register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) {
this.preparerMap.set(templaterKey, templater);
this.templaterMap.set(templaterKey, templater);
}
get(template: TemplateEntityV1alpha1): TemplaterBase {
const templaterKey = getTemplaterKey(template);
const preparer = this.preparerMap.get(templaterKey);
get(templaterId: string): TemplaterBase {
const templater = this.templaterMap.get(templaterId);
if (!preparer) {
throw new Error(
`No templater registered for template: "${templaterKey}"`,
);
if (!templater) {
throw new Error(`No templater registered for template: "${templaterId}"`);
}
return preparer;
return templater;
}
}
@@ -15,7 +15,6 @@
*/
import type { Writable } from 'stream';
import Docker from 'dockerode';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import gitUrlParse from 'git-url-parse';
/**
@@ -46,7 +45,7 @@ export type TemplaterRunResult = {
* client to run any templater on top of your directory.
*/
export type TemplaterRunOptions = {
directory: string;
workspacePath: string;
values: TemplaterValues;
logStream?: Writable;
dockerClient: Docker;
@@ -54,7 +53,7 @@ export type TemplaterRunOptions = {
export type TemplaterBase = {
// runs the templating with the values and returns the directory to push the VCS
run(opts: TemplaterRunOptions): Promise<TemplaterRunResult>;
run(opts: TemplaterRunOptions): Promise<void>;
};
export type TemplaterConfig = {
@@ -71,5 +70,5 @@ export type SupportedTemplatingKey = 'cookiecutter' | string;
*/
export type TemplaterBuilder = {
register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void;
get(template: TemplateEntityV1alpha1): TemplaterBase;
get(templater: string): TemplaterBase;
};
@@ -23,6 +23,8 @@ jest.doMock('fs-extra', () => ({
F_OK: 0,
W_OK: 1,
},
mkdir: jest.fn(),
remove: jest.fn(),
}));
import { getVoidLogger } from '@backstage/backend-common';
@@ -116,9 +118,10 @@ describe('createRouter - working directory', () => {
},
});
expect(mockPrepare).toBeCalledWith(expect.anything(), {
expect(mockPrepare).toBeCalledWith({
logger: expect.anything(),
workingDirectory: '/path',
workspacePath: expect.stringContaining('path'),
url: expect.anything(),
});
});
@@ -143,8 +146,10 @@ describe('createRouter - working directory', () => {
},
});
expect(mockPrepare).toBeCalledWith(expect.anything(), {
expect(mockPrepare).toBeCalledWith({
logger: expect.anything(),
workspacePath: expect.anything(),
url: expect.anything(),
});
});
});
@@ -15,19 +15,19 @@
*/
import { Config } from '@backstage/config';
import fs from 'fs-extra';
import Docker from 'dockerode';
import express from 'express';
import { resolve as resolvePath } from 'path';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import {
JobProcessor,
PreparerBuilder,
StageContext,
TemplaterBuilder,
TemplaterValues,
PublisherBuilder,
parseLocationAnnotation,
joinGitUrlPath,
FilePreparer,
} from '../scaffolder';
import { CatalogEntityClient } from '../lib/catalog';
@@ -62,27 +62,7 @@ export async function createRouter(
} = options;
const logger = parentLogger.child({ plugin: 'scaffolder' });
const jobProcessor = new JobProcessor();
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
}
const jobProcessor = await JobProcessor.fromConfig({ config, logger });
router
.get('/v1/job/:jobId', ({ params }, res) => {
@@ -128,51 +108,69 @@ export async function createRouter(
res.status(400).json({ errors: validationResult.errors });
return;
}
const job = jobProcessor.create({
entity: template,
values,
stages: [
{
name: 'Prepare the skeleton',
handler: async ctx => {
const { protocol, location: pullPath } = parseLocationAnnotation(
ctx.entity,
async handler(ctx) {
const {
protocol,
location: templateEntityLocation,
} = parseLocationAnnotation(ctx.entity);
if (protocol === 'file') {
const preparer = new FilePreparer();
const path = resolvePath(
templateEntityLocation,
template.spec.path || '.',
);
await preparer.prepare({
url: `file://${path}`,
logger: ctx.logger,
workspacePath: ctx.workspacePath,
});
return;
}
const preparer = preparers.get(templateEntityLocation);
const url = joinGitUrlPath(
templateEntityLocation,
template.spec.path,
);
const preparer =
protocol === 'file'
? new FilePreparer()
: preparers.get(pullPath);
const skeletonDir = await preparer.prepare(ctx.entity, {
await preparer.prepare({
url,
logger: ctx.logger,
workingDirectory,
workspacePath: ctx.workspacePath,
});
return { skeletonDir };
},
},
{
name: 'Run the templater',
handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
const templater = templaters.get(ctx.entity);
const { resultDir } = await templater.run({
directory: ctx.skeletonDir,
async handler(ctx) {
const templater = templaters.get(ctx.entity.spec.templater);
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
logStream: ctx.logStream,
values: ctx.values,
});
return { resultDir };
},
},
{
name: 'Publish template',
handler: async (ctx: StageContext<{ resultDir: string }>) => {
handler: async ctx => {
const publisher = publishers.get(ctx.values.storePath);
ctx.logger.info('Will now store the template');
const result = await publisher.publish({
values: ctx.values,
directory: ctx.resultDir,
workspacePath: ctx.workspacePath,
logger: ctx.logger,
});
return result;