paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

Merge pull request #4270 from backstage/mob/stages-refactor

Browse Source 
Refactor scaffolder to operate on known directories
This commit is contained in:
Johan Haals
2021-01-29 10:58:47 +01:00
committed by GitHub
parent aca1ada8dc 1236f22909
commit 2bc408228b
35 changed files with 748 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wicked-beds-buy.md
+61
View File
@@ -0,0 +1,61 @@
---
'@backstage/plugin-scaffolder-backend': minor
---
The scaffolder is updated to generate a unique workspace directory inside the temp folder. This directory is cleaned up by the job processor after each run.
The prepare/template/publish steps have been refactored to operate on known directories, `template/` and `result/`, inside the temporary workspace path.
Updated preparers to accept the template url instead of the entire template. This is done primarily to allow for backwards compatibility between v1 and v2 scaffolder templates.
Fixes broken GitHub actions templating in the Create React App template.
#### For those with **custom** preparers, templates, or publishers
The preparer interface has changed, the prepare method now only takes a single argument, and doesn't return anything. As part of this change the preparers were refactored to accept a URL pointing to the target directory, rather than computing that from the template entity.
The `workingDirectory` option was also removed, and replaced with a `workspacePath` option. The difference between the two is that `workingDirectory` was a place for the preparer to create temporary directories, while the `workspacePath` is the specific folder were the entire templating process for a single template job takes place. Instead of returning a path to the folder were the prepared contents were placed, the contents are put at the `<workspacePath>/template` path.
```diff
type PreparerOptions = {
- workingDirectory?: string;
+ /**
+ * Full URL to the directory containg template data
+ */
+ url: string;
+ /**
+ * The workspace path that will eventually be the the root of the new repo
+ */
+ workspacePath: string;
logger: Logger;
};
-prepare(template: TemplateEntityV1alpha1, opts?: PreparerOptions): Promise<string>
+prepare(opts: PreparerOptions): Promise<void>;
```
Instead of returning a path to the folder were the templaters contents were placed, the contents are put at the `<workspacePath>/result` path. All templaters now also expect the source template to be present in the `template` directory within the `workspacePath`.
```diff
export type TemplaterRunOptions = {
- directory: string;
+ workspacePath: string;
values: TemplaterValues;
logStream?: Writable;
dockerClient: Docker;
};
-public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult>
+public async run(options: TemplaterRunOptions): Promise<void>
```
Just like the preparer and templaters, the publishers have also switched to using `workspacePath`. The root of the new repo is expected to be located at `<workspacePath>/result`.
```diff
export type PublisherOptions = {
values: TemplaterValues;
- directory: string;
+ workspacePath: string;
logger: Logger;
};
```
plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts
+11 -11
View File
@@ -70,7 +70,7 @@ describe('JobProcessor', () => {
describe('create', () => {
it('creates should create a new job with a unique id', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -84,7 +84,7 @@ describe('JobProcessor', () => {
});
it('should setup the correct context for the job', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -97,7 +97,7 @@ describe('JobProcessor', () => {
});
it('should set the status as pending', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -120,7 +120,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
@@ -139,12 +139,12 @@ describe('JobProcessor', () => {
describe('get', () => {
it('return undefined for when the job does not exist', () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
expect(processor.get('123')).not.toBeDefined();
});
it('should return the exact same instance of the job when one is created', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -156,7 +156,7 @@ describe('JobProcessor', () => {
});
describe('process', () => {
it('throws an error when the status of the job is not in pending state', async () => {
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -182,7 +182,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -208,7 +208,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -244,7 +244,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
@@ -283,7 +283,7 @@ describe('JobProcessor', () => {
},
];
const processor = new JobProcessor();
const processor = new JobProcessor('/tmp');
const job = processor.create({
entity: mockEntity,
values: mockValues,
plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts
+50 -1
View File
@@ -13,13 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import { Processor, Job, StageContext, StageInput } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import * as uuid from 'uuid';
import Docker from 'dockerode';
import path from 'path';
import { TemplaterValues, TemplaterBase } from '../stages/templater';
import { PreparerBuilder } from '../stages/prepare';
import { makeLogStream } from './logger';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
export type JobProcessorArguments = {
preparers: PreparerBuilder;
@@ -33,7 +39,45 @@ export type JobAndDirectoryTuple = {
};
export class JobProcessor implements Processor {
private jobs = new Map<string, Job>();
private readonly workingDirectory: string;
private readonly jobs: Map<string, Job>;
static async fromConfig({
config,
logger,
}: {
config: Config;
logger: Logger;
}) {
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
} else {
workingDirectory = os.tmpdir();
}
return new JobProcessor(workingDirectory);
}
constructor(workingDirectory: string) {
this.workingDirectory = workingDirectory;
this.jobs = new Map<string, Job>();
}
create({
entity,
@@ -52,6 +96,7 @@ export class JobProcessor implements Processor {
values,
logger,
logStream: stream,
workspacePath: path.join(this.workingDirectory, id),
};
const job: Job = {
@@ -80,6 +125,8 @@ export class JobProcessor implements Processor {
throw new Error("Job is not in a 'PENDING' state");
}
await fs.mkdir(job.context.workspacePath);
job.status = 'STARTED';
try {
@@ -134,6 +181,8 @@ export class JobProcessor implements Processor {
// If something went wrong, fail the job, and set the error property on the job.
job.error = { name: error.name, message: error.message };
job.status = 'FAILED';
} finally {
await fs.remove(job.context.workspacePath);
}
}
}
plugins/scaffolder-backend/src/scaffolder/jobs/types.ts
+1
View File
@@ -26,6 +26,7 @@ export type StageContext<T = {}> = {
entity: TemplateEntityV1alpha1;
logger: Logger;
logStream: Writable;
workspacePath: string;
} & T;
export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
plugins/scaffolder-backend/src/scaffolder/stages/helpers.test.ts
+74 -2
View File
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseLocationAnnotation } from './helpers';
import { parseLocationAnnotation, joinGitUrlPath } from './helpers';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
@@ -73,7 +73,7 @@ describe('Helpers', () => {
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
':https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
':https://github.com/o/r/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
@@ -250,4 +250,76 @@ describe('Helpers', () => {
});
});
});
describe('joinGitUrlPath', () => {
it.each([
[
'https://github.com/o/r/blob/master/template.yaml',
'template',
'https://github.com/o/r/blob/master/template',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml',
undefined,
'https://dev.azure.com/o/p/_git/template-repo?path=%2F',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Ftemplate.yaml',
'a',
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa',
],
[
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Ftemplate.yaml',
'b',
'https://dev.azure.com/o/p/_git/template-repo?path=%2Fa%2Fb',
],
[
'https://github.com/o/r/blob/master/template.yaml',
undefined,
'https://github.com/o/r/blob/master',
],
[
'https://github.com/o/r/blob/master/template.yaml',
'template',
'https://github.com/o/r/blob/master/template',
],
[
'https://github.com/o/r/blob/master/templates/graphql-starter/template.yaml',
'template',
'https://github.com/o/r/blob/master/templates/graphql-starter/template',
],
[
'https://gitlab.com/o/r/-/blob/master/template.yaml',
undefined,
'https://gitlab.com/o/r/-/blob/master',
],
[
'https://gitlab.com/o/r/-/blob/master/template.yaml',
'template',
'https://gitlab.com/o/r/-/blob/master/template',
],
[
'https://gitlab.com/o/r/-/blob/master/a/b/c/template.yaml',
'../../c',
'https://gitlab.com/o/r/-/blob/master/a/c',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
undefined,
'https://bitbucket.org/p/r/src/master/a/b',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
'c',
'https://bitbucket.org/p/r/src/master/a/b/c',
],
[
'https://bitbucket.org/p/r/src/master/a/b/template.yaml',
'../c',
'https://bitbucket.org/p/r/src/master/a/c',
],
])('should join git url %s with path %s', (url, path, result) => {
expect(joinGitUrlPath(url, path)).toBe(result);
});
});
});
plugins/scaffolder-backend/src/scaffolder/stages/helpers.ts
+19
View File
@@ -13,6 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { posix as posixPath } from 'path';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
@@ -53,3 +55,20 @@ export const parseLocationAnnotation = (
location,
};
};
export function joinGitUrlPath(repoUrl: string, path?: string): string {
const parsed = new URL(repoUrl);
if (parsed.hostname.endsWith('azure.com')) {
const templatePath = posixPath.normalize(
posixPath.join(
posixPath.dirname(parsed.searchParams.get('path') || '/'),
path || '.',
),
);
parsed.searchParams.set('path', templatePath);
return parsed.toString();
}
return new URL(path || '.', repoUrl).toString().replace(/\/$/, '');
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts
+53 -92
View File
@@ -14,19 +14,14 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { AzurePreparer } from './azure';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
jest.mock('fs-extra');
describe('AzurePreparer', () => {
const mockGitClient = {
clone: jest.fn(),
@@ -36,121 +31,87 @@ describe('AzurePreparer', () => {
jest.spyOn(Git, 'fromAuth').mockReturnValue(mockGitClient as any);
let mockEntity: TemplateEntityV1alpha1;
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = AzurePreparer.fromConfig({
host: 'dev.azure.com',
token: 'fake-azure-token',
});
// TODO(blam): Here's a test that will fail when the deprecation is complete
it('calls the clone command with deprecated token', async () => {
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
password: 'fake-azure-token',
username: 'notempty',
});
});
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const prepareOptions = {
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
workspacePath,
logger,
};
it('calls the clone command with token from integrations config', async () => {
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
password: 'fake-azure-token',
username: 'notempty',
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: expect.any(String),
dir: checkoutPath,
});
});
it('calls the clone command with the correct arguments for a repository with a specified branch', async () => {
await preparer.prepare({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: checkoutPath,
ref: 'master',
});
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
workspacePath,
logger,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url:
'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo',
dir: expect.any(String),
ref: 'master',
dir: checkoutPath,
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
});
it('return the temp directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
it('moves the template from path if it is specified', async () => {
const path = './subdir';
await preparer.prepare({
url: `https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=${encodeURIComponent(
path,
)}`,
logger,
workspacePath,
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, 'subdir'),
templatePath,
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts
+17 -27
View File
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,26 +27,13 @@ export class AzurePreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath ?? '',
);
// Username can be anything but the empty string according to:
@@ -63,11 +47,17 @@ export class AzurePreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
ref: parsedGitUrl.ref,
dir: checkoutPath,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts
+31 -75
View File
@@ -14,21 +14,15 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import { BitbucketPreparer } from './bitbucket';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
import { resolve } from 'path';
import os from 'os';
jest.mock('fs-extra');
describe('BitbucketPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const logger = getVoidLogger();
const mockGitClient = {
clone: jest.fn(),
@@ -38,44 +32,6 @@ describe('BitbucketPreparer', () => {
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'bitbucket:https://bitbucket.org/backstage-project/backstage-repo',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = BitbucketPreparer.fromConfig({
@@ -84,13 +40,25 @@ describe('BitbucketPreparer', () => {
appPassword: 'fake-password',
});
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const prepareOptions = {
url: 'https://bitbucket.org/backstage-project/backstage-repo',
logger,
workspacePath,
};
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with the correct arguments if an app password is provided for a repository', async () => {
@@ -99,7 +67,7 @@ describe('BitbucketPreparer', () => {
username: 'fake-user',
appPassword: 'fake-password',
});
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
@@ -109,23 +77,23 @@ describe('BitbucketPreparer', () => {
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare(prepareOptions);
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://bitbucket.org/backstage-project/backstage-repo',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
});
it('return the temp directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
it('moves a template subdirectory to checkout if specified', async () => {
await preparer.prepare({
url: 'https://bitbucket.org/foo/bar/src/master/1/2/3',
logger,
workspacePath,
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, '1', '2', '3'),
templatePath,
);
});
@@ -135,7 +103,7 @@ describe('BitbucketPreparer', () => {
token: 'fake-token',
});
await preparer.prepare(mockEntity, { logger });
await preparer.prepare(prepareOptions);
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
@@ -143,16 +111,4 @@ describe('BitbucketPreparer', () => {
password: 'fake-token',
});
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts
+18 -35
View File
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import { BitbucketIntegrationConfig } from '@backstage/integration';
@@ -40,44 +37,30 @@ export class BitbucketPreparer implements PreparerBase {
},
) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const auth = this.getAuth();
const git = auth
? Git.fromAuth({
...auth,
logger,
})
: Git.fromAuth({ logger });
const git = Git.fromAuth({ logger, ...this.getAuth() });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
private getAuth(): { username: string; password: string } | undefined {
plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.test.ts
+32 -46
View File
@@ -14,57 +14,43 @@
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import YAML from 'yaml';
import { FilePreparer } from './file';
import path from 'path';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger } from '@backstage/backend-common';
import fs from 'fs-extra';
import { FilePreparer } from './file';
import os from 'os';
import { resolve } from 'path';
const setupTest = async (fixturePath: string) => {
const locationForTemplateYaml = path.resolve(
path.dirname(
require.resolve('@backstage/plugin-scaffolder-backend/package'),
),
'fixtures',
fixturePath,
);
const [parsedDocument] = YAML.parseAllDocuments(
await fs.readFile(locationForTemplateYaml, 'utf-8'),
);
const template: TemplateEntityV1alpha1 = parsedDocument.toJSON();
template.metadata.annotations = {
[LOCATION_ANNOTATION]: `file:${locationForTemplateYaml}`,
};
const filePreparer = new FilePreparer();
const resultDir = await filePreparer.prepare(template, {
logger: getVoidLogger(),
workingDirectory: os.tmpdir(),
});
return { filePreparer, template, resultDir };
};
jest.mock('fs-extra');
describe('File preparer', () => {
it('excludes the yaml file from the temp folder', async () => {
const { resultDir } = await setupTest('test-simple-template/template.yaml');
expect(fs.existsSync(`${resultDir}/template.yaml`)).toBe(false);
});
it('prepares templates from a file path', async () => {
const logger = getVoidLogger();
const preparer = new FilePreparer();
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
it('resolves relative path from the template', async () => {
const { resultDir } = await setupTest('test-simple-template/template.yaml');
expect(fs.existsSync(`${resultDir}/expected_file.ts`)).toBe(true);
});
await preparer.prepare({
url: 'file:///path/to/template',
logger,
workspacePath,
});
expect(fs.copy).toHaveBeenCalledWith(
resolve('/path', 'to', 'template'),
checkoutPath,
{
recursive: true,
},
);
expect(fs.ensureDir).toHaveBeenCalledWith(checkoutPath);
it('resolves relative path from the nested template', async () => {
const { resultDir } = await setupTest('test-nested-template/template.yaml');
expect(fs.existsSync(`${resultDir}/expected_file.ts`)).toBe(true);
await expect(
preparer.prepare({
url: 'file://not/full/path',
logger,
workspacePath,
}),
).rejects.toThrow(
"Wrong location protocol, should be 'file', file://not/full/path",
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts
+7 -26
View File
@@ -13,43 +13,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
export class FilePreparer implements PreparerBase {
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
if (protocol !== 'file') {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'file'`,
);
async prepare({ url, workspacePath }: PreparerOptions) {
if (!url.startsWith('file:///')) {
throw new InputError(`Wrong location protocol, should be 'file', ${url}`);
}
const templateId = template.metadata.name;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const checkoutDir = path.join(workspacePath, 'checkout');
await fs.ensureDir(checkoutDir);
const parentDirectory = path.resolve(
path.dirname(location),
template.spec.path ?? '.',
);
const templatePath = url.slice('file://'.length);
await fs.copy(parentDirectory, tempDir, {
filter: src => src !== location,
await fs.copy(templatePath, checkoutDir, {
recursive: true,
});
return tempDir;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts
+37 -88
View File
@@ -14,21 +14,19 @@
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { GithubPreparer } from './github';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
jest.mock('fs-extra');
describe('GitHubPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const mockGitClient = {
clone: jest.fn(),
};
@@ -38,44 +36,6 @@ describe('GitHubPreparer', () => {
beforeEach(() => {
jest.clearAllMocks();
mockEntity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
});
const preparer = GithubPreparer.fromConfig({
@@ -84,60 +44,49 @@ describe('GitHubPreparer', () => {
});
it('calls the clone command with the correct arguments for a repository', async () => {
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master/templates/graphql-starter/template',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://github.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, 'templates', 'graphql-starter', 'template'),
templatePath,
);
expect(fs.rmdir).toHaveBeenCalledWith('/tmp/template/.git');
});
it('calls the clone command with the correct arguments for a repository when no path is provided', async () => {
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://github.com/benjdlambert/backstage-graphql-template',
dir: expect.any(String),
ref: expect.any(String),
dir: checkoutPath,
ref: 'master',
});
});
it('return the temp directory with the path to the folder if it is specified', async () => {
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
const preparer = GithubPreparer.fromConfig({
host: 'github.com',
token: 'fake-token',
});
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
);
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it('calls the clone command with token', async () => {
await preparer.prepare(mockEntity, { logger });
await preparer.prepare({
url:
'https://github.com/benjdlambert/backstage-graphql-template/blob/master',
logger,
workspacePath,
});
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts
+17 -29
View File
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,30 +27,15 @@ export class GithubPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const git = this.config.token
? Git.fromAuth({
username: this.config.token,
@@ -63,11 +45,17 @@ export class GithubPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts
+28 -91
View File
@@ -13,60 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.doMock('fs-extra', () => ({
promises: {
mkdtemp: jest.fn(dir => `${dir}-static`),
},
}));
import fs from 'fs-extra';
import os from 'os';
import { resolve } from 'path';
import { GitlabPreparer } from './gitlab';
import {
TemplateEntityV1alpha1,
LOCATION_ANNOTATION,
} from '@backstage/catalog-model';
import { getVoidLogger, Git } from '@backstage/backend-common';
const mockTemplate = (): TemplateEntityV1alpha1 => ({
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
[LOCATION_ANNOTATION]:
'url:https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/template.yaml',
},
name: 'graphql-starter',
title: 'GraphQL Service',
description:
'A GraphQL starter template for backstage to get you up and running\nthe best practices with GraphQL\n',
uid: '9cf16bad-16e0-4213-b314-c4eec773c50b',
etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2',
generation: 1,
},
spec: {
type: 'website',
templater: 'cookiecutter',
path: './template',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
});
jest.mock('fs-extra');
describe('GitLabPreparer', () => {
let mockEntity: TemplateEntityV1alpha1;
const workspacePath = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp';
const checkoutPath = resolve(workspacePath, 'checkout');
const templatePath = resolve(workspacePath, 'template');
const mockGitClient = {
clone: jest.fn(),
};
@@ -81,63 +40,41 @@ describe('GitLabPreparer', () => {
host: 'gitlab.com',
token: 'fake-token',
});
it(`calls the clone command with the correct arguments for a repository`, async () => {
mockEntity = mockTemplate();
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
it(`calls the clone command with the correct arguments for a repository`, async () => {
await preparer.prepare({
url:
'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master',
logger,
workspacePath,
});
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git',
dir: expect.any(String),
dir: checkoutPath,
ref: expect.any(String),
});
});
it(`calls the clone command with the correct arguments if an access token is provided in integrations for a repository`, async () => {
mockEntity = mockTemplate();
await preparer.prepare(mockEntity, { logger });
expect(Git.fromAuth).toHaveBeenCalledWith({
logger,
username: 'oauth2',
password: 'fake-token',
});
expect(fs.move).toHaveBeenCalledWith(checkoutPath, templatePath);
expect(fs.rmdir).toHaveBeenCalledWith(resolve(templatePath, '.git'));
});
it(`calls the clone command with the correct arguments for a repository when no path is provided`, async () => {
mockEntity = mockTemplate();
delete mockEntity.spec.path;
await preparer.prepare(mockEntity, { logger: getVoidLogger() });
expect(mockGitClient.clone).toHaveBeenCalledWith({
url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git',
dir: expect.any(String),
ref: expect.any(String),
it(`clones the template from a sub directory if specified`, async () => {
await preparer.prepare({
url:
'https://gitlab.com/benjdlambert/backstage-graphql-template/-/blob/master/1/2/3',
logger,
workspacePath,
});
});
it(`return the temp directory with the path to the folder if it is specified`, async () => {
mockEntity = mockTemplate();
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/template\/test\/1\/2\/3$/,
);
});
it('return the working directory with the path to the folder if it is specified', async () => {
mockEntity.spec.path = './template/test/1/2/3';
const response = await preparer.prepare(mockEntity, {
workingDirectory: '/workDir',
logger: getVoidLogger(),
});
expect(response.split('\\').join('/')).toMatch(
/\/workDir\/graphql-starter-static\/template\/test\/1\/2\/3$/,
expect(fs.move).toHaveBeenCalledWith(
resolve(checkoutPath, '1', '2', '3'),
templatePath,
);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts
+18 -28
View File
@@ -14,13 +14,10 @@
* limitations under the License.
*/
import { Git } from '@backstage/backend-common';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { GitLabIntegrationConfig } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
import path from 'path';
import { parseLocationAnnotation } from '../helpers';
import { PreparerBase, PreparerOptions } from './types';
export class GitlabPreparer implements PreparerBase {
@@ -30,28 +27,15 @@ export class GitlabPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const logger = opts.logger;
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
parsedGitLocation.git_suffix = true;
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const ref = parsedGitLocation.ref;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
parsedGitUrl.git_suffix = true;
const git = this.config.token
? Git.fromAuth({
@@ -62,11 +46,17 @@ export class GitlabPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
ref: ref,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
ref: parsedGitUrl.ref,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
plugins/scaffolder-backend/src/scaffolder/stages/prepare/types.ts
+10 -9
View File
@@ -13,24 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
export type PreparerOptions = {
workingDirectory?: string;
/**
* Full URL to the directory containg template data
*/
url: string;
/**
* The workspace path that will eventually be the the root of the new repo
*/
workspacePath: string;
logger: Logger;
};
export interface PreparerBase {
/**
* Given an Entity definition from the Service Catalog, go and prepare a directory
* with contents from the remote location in temporary storage and return the path
* @param template The template entity from the Service Catalog
* Prepare a directory with contents from the remote location
*/
prepare(
template: TemplateEntityV1alpha1,
opts?: PreparerOptions,
): Promise<string>;
prepare(opts: PreparerOptions): Promise<void>;
}
export type PreparerBuilder = {
plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts
+2 -2
View File
@@ -53,7 +53,7 @@ describe('Azure Publisher', () => {
storePath: 'https://dev.azure.com/organisation/project/_git/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -74,7 +74,7 @@ describe('Azure Publisher', () => {
'project',
);
expect(helpers.initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
auth: { username: 'notempty', password: 'fake-azure-token' },
logger,
plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts
+3 -2
View File
@@ -21,6 +21,7 @@ import { initRepoAndPush } from './helpers';
import { AzureIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { getPersonalAccessTokenHandler, WebApi } from 'azure-devops-node-api';
import path from 'path';
export class AzurePublisher implements PublisherBase {
static async fromConfig(config: AzureIntegrationConfig) {
@@ -34,7 +35,7 @@ export class AzurePublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name, organization, resource } = parseGitUrl(
@@ -56,7 +57,7 @@ export class AzurePublisher implements PublisherBase {
const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`;
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: 'notempty',
plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.test.ts
+4 -4
View File
@@ -69,7 +69,7 @@ describe('Bitbucket Publisher', () => {
storePath: 'https://bitbucket.org/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger: logger,
});
@@ -80,7 +80,7 @@ describe('Bitbucket Publisher', () => {
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://bitbucket.org/project/repo',
auth: { username: 'fake-user', password: 'fake-token' },
logger: logger,
@@ -127,7 +127,7 @@ describe('Bitbucket Publisher', () => {
storePath: 'https://bitbucket.mycompany.com/project/repo',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger: logger,
});
@@ -138,7 +138,7 @@ describe('Bitbucket Publisher', () => {
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://bitbucket.mycompany.com/scm/project/repo',
auth: { username: 'x-token-auth', password: 'fake-token' },
logger: logger,
plugins/scaffolder-backend/src/scaffolder/stages/publish/bitbucket.ts
+3 -2
View File
@@ -19,6 +19,7 @@ import { initRepoAndPush } from './helpers';
import fetch from 'cross-fetch';
import { BitbucketIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import path from 'path';
// TODO(blam): We should probably start to use a bitbucket client here that we can change
// the baseURL to point at on-prem or public bitbucket versions like we do for
@@ -45,7 +46,7 @@ export class BitbucketPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner: project, name } = parseGitUrl(values.storePath);
@@ -58,7 +59,7 @@ export class BitbucketPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl: result.remoteUrl,
auth: {
username: this.config.username ? this.config.username : 'x-token-auth',
plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts
+10 -10
View File
@@ -64,7 +64,7 @@ describe('GitHub Publisher', () => {
owner: 'bob',
access: 'blam/team',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -89,7 +89,7 @@ describe('GitHub Publisher', () => {
permission: 'admin',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -122,7 +122,7 @@ describe('GitHub Publisher', () => {
owner: 'bob',
access: 'blam',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -140,7 +140,7 @@ describe('GitHub Publisher', () => {
expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled();
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -175,7 +175,7 @@ describe('GitHub Publisher', () => {
access: 'bob',
description: 'description',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -198,7 +198,7 @@ describe('GitHub Publisher', () => {
permission: 'admin',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -233,7 +233,7 @@ describe('GitHub Publisher', () => {
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -249,7 +249,7 @@ describe('GitHub Publisher', () => {
visibility: 'internal',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
@@ -283,7 +283,7 @@ describe('GitHub Publisher', () => {
storePath: 'https://github.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -299,7 +299,7 @@ describe('GitHub Publisher', () => {
private: true,
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'https://github.com/backstage/backstage.git',
auth: { username: 'fake-token', password: 'x-oauth-basic' },
logger,
plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts
+5 -3
View File
@@ -19,8 +19,10 @@ import { initRepoAndPush } from './helpers';
import { GitHubIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { Octokit } from '@octokit/rest';
import path from 'path';
export type RepoVisibilityOptions = 'private' | 'internal' | 'public';
export class GithubPublisher implements PublisherBase {
static async fromConfig(
config: GitHubIntegrationConfig,
@@ -41,6 +43,7 @@ export class GithubPublisher implements PublisherBase {
repoVisibility,
});
}
constructor(
private readonly config: {
token: string;
@@ -51,7 +54,7 @@ export class GithubPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name } = parseGitUrl(values.storePath);
@@ -66,7 +69,7 @@ export class GithubPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: this.config.token,
@@ -79,7 +82,6 @@ export class GithubPublisher implements PublisherBase {
/\.git$/,
'/blob/master/catalog-info.yaml',
);
return { remoteUrl, catalogInfoUrl };
}
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.test.ts
+4 -4
View File
@@ -68,7 +68,7 @@ describe('GitLab Publisher', () => {
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -85,7 +85,7 @@ describe('GitLab Publisher', () => {
name: 'test',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'mockclone',
auth: { username: 'oauth2', password: 'fake-token' },
logger,
@@ -111,7 +111,7 @@ describe('GitLab Publisher', () => {
storePath: 'https://gitlab.com/blam/test',
owner: 'bob',
},
directory: '/tmp/test',
workspacePath: '/tmp/test',
logger,
});
@@ -125,7 +125,7 @@ describe('GitLab Publisher', () => {
name: 'test',
});
expect(initRepoAndPush).toHaveBeenCalledWith({
dir: '/tmp/test',
dir: '/tmp/test/result',
remoteUrl: 'mockclone',
auth: { username: 'oauth2', password: 'fake-token' },
logger,
plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts
+3 -3
View File
@@ -19,7 +19,7 @@ import { Gitlab } from '@gitbeaker/node';
import { Gitlab as GitlabClient } from '@gitbeaker/core';
import { initRepoAndPush } from './helpers';
import parseGitUrl from 'git-url-parse';
import path from 'path';
import { GitLabIntegrationConfig } from '@backstage/integration';
export class GitlabPublisher implements PublisherBase {
@@ -38,7 +38,7 @@ export class GitlabPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name } = parseGitUrl(values.storePath);
@@ -49,7 +49,7 @@ export class GitlabPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: 'oauth2',
plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts
+1 -1
View File
@@ -32,7 +32,7 @@ export type PublisherBase = {
export type PublisherOptions = {
values: TemplaterValues;
directory: string;
workspacePath: string;
logger: Logger;
};
plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts
+73 -100
View File
@@ -13,48 +13,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('./helpers', () => ({
runDockerContainer: jest.fn(),
runCommand: jest.fn(),
}));
jest.mock('command-exists-promise', () => jest.fn());
const runDockerContainer = jest.fn();
const runCommand = jest.fn();
const commandExists = jest.fn();
jest.mock('./helpers', () => ({ runDockerContainer, runCommand }));
jest.mock('command-exists-promise', () => commandExists);
jest.mock('fs-extra');
import { CookieCutter } from './cookiecutter';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { RunDockerContainerOptions, RunCommandOptions } from './helpers';
import { PassThrough } from 'stream';
import Docker from 'dockerode';
import parseGitUrl from 'git-url-parse';
const commandExists = require('command-exists-promise');
describe('CookieCutter Templater', () => {
const cookie = new CookieCutter();
const mockDocker = {} as Docker;
const {
runDockerContainer,
}: {
runDockerContainer: jest.Mock<RunDockerContainerOptions>;
} = require('./helpers');
jest
.spyOn(fs, 'readdir')
.mockImplementation(() => Promise.resolve(['newthing']));
beforeEach(async () => {
beforeEach(() => {
jest.clearAllMocks();
});
const mkTemp = async () => {
const tempDir = os.tmpdir();
return await fs.promises.mkdtemp(path.join(tempDir, 'temp'));
};
it('should write a cookiecutter.json file with the values from the entity', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -65,22 +46,30 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(cookieCutterJson).toEqual(
expect.objectContaining(JSON.parse(JSON.stringify(values))),
expect(fs.writeJson).toBeCalledWith(
'tempdir/template/cookiecutter.json',
expect.objectContaining(values),
);
});
it('should merge any value that is in the cookiecutter.json path already', async () => {
const tempdir = await mkTemp();
const existingJson = {
_copy_without_render: ['./github/workflows/*'],
};
await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson);
jest
.spyOn(fs, 'readJSON')
.mockImplementationOnce(() => Promise.resolve(existingJson));
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const values = {
owner: 'blobby',
@@ -91,11 +80,14 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
const cookieCutterJson = await fs.readJSON(`${tempdir}/cookiecutter.json`);
expect(cookieCutterJson).toEqual({
expect(fs.writeJSON).toBeCalledWith('tempdir/template/cookiecutter.json', {
...existingJson,
...values,
destination: {
@@ -105,9 +97,9 @@ describe('CookieCutter Templater', () => {
});
it('should throw an error if the cookiecutter json is malformed and not missing', async () => {
const tempdir = await mkTemp();
await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'");
jest.spyOn(fs, 'readJSON').mockImplementationOnce(() => {
throw new Error('BAM');
});
const values = {
owner: 'blobby',
@@ -117,14 +109,17 @@ describe('CookieCutter Templater', () => {
},
};
const templater = new CookieCutter();
await expect(
cookie.run({ directory: tempdir, values, dockerClient: mockDocker }),
).rejects.toThrow(/Unexpected token ' in JSON at position 1/);
templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
}),
).rejects.toThrow('BAM');
});
it('should run the correct docker container with the correct bindings for the volumes', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -134,7 +129,14 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({ directory: tempdir, values, dockerClient: mockDocker });
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(runDockerContainer).toHaveBeenCalledWith({
imageName: 'spotify/backstage-cookiecutter',
@@ -146,39 +148,16 @@ describe('CookieCutter Templater', () => {
'/template',
'--verbose',
],
templateDir: tempdir,
resultDir: expect.stringContaining(`${tempdir}-result`),
templateDir: 'tempdir/template',
resultDir: 'tempdir/intermediate',
logStream: undefined,
dockerClient: mockDocker,
});
});
it('should return the result path to the end templated folder', async () => {
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
component_id: 'newthing',
destination: {
git: parseGitUrl('https://github.com/org/repo'),
},
};
const { resultDir } = await cookie.run({
directory: tempdir,
values,
dockerClient: mockDocker,
});
expect(resultDir.startsWith(`${tempdir}-result`)).toBeTruthy();
});
it('should pass through the streamer to the run docker helper', async () => {
const stream = new PassThrough();
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -188,8 +167,11 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({
directory: tempdir,
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
@@ -205,29 +187,17 @@ describe('CookieCutter Templater', () => {
'/template',
'--verbose',
],
templateDir: tempdir,
resultDir: expect.stringContaining(`${tempdir}-result`),
templateDir: 'tempdir/template',
resultDir: 'tempdir/intermediate',
logStream: stream,
dockerClient: mockDocker,
});
});
describe('when cookiecutter is available', () => {
beforeAll(() => {
commandExists.mockImplementation(() => () => true);
});
it('use the binary', async () => {
const {
runCommand,
}: {
runCommand: jest.Mock<RunCommandOptions>;
} = require('./helpers');
const stream = new PassThrough();
const tempdir = await mkTemp();
const values = {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -237,8 +207,12 @@ describe('CookieCutter Templater', () => {
},
};
await cookie.run({
directory: tempdir,
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing']);
commandExists.mockImplementationOnce(() => () => true);
const templater = new CookieCutter();
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
@@ -249,8 +223,8 @@ describe('CookieCutter Templater', () => {
args: expect.arrayContaining([
'--no-input',
'-o',
tempdir,
expect.stringContaining(`${tempdir}-result`),
'tempdir/intermediate',
'tempdir/template',
'--verbose',
]),
logStream: stream,
@@ -259,18 +233,17 @@ describe('CookieCutter Templater', () => {
});
describe('when nothing was generated', () => {
beforeEach(() => {
jest.spyOn(fs, 'readdir').mockImplementation(() => Promise.resolve([]));
});
it('throws an error', async () => {
const stream = new PassThrough();
const tempdir = await mkTemp();
jest
.spyOn(fs, 'readdir')
.mockImplementationOnce(() => Promise.resolve([]));
return expect(
cookie.run({
directory: tempdir,
const templater = new CookieCutter();
await expect(
templater.run({
workspacePath: 'tempdir',
values: {
owner: 'blobby',
storePath: 'https://github.com/org/repo',
@@ -281,7 +254,7 @@ describe('CookieCutter Templater', () => {
logStream: stream,
dockerClient: mockDocker,
}),
).rejects.toThrow(/Cookie Cutter did not generate anything/);
).rejects.toThrow(/No data generated by cookiecutter/);
});
});
});
plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts
+22 -22
View File
@@ -19,8 +19,6 @@ import { runDockerContainer, runCommand } from './helpers';
import { TemplaterBase, TemplaterRunOptions } from '.';
import path from 'path';
import { TemplaterRunResult } from './types';
const commandExists = require('command-exists-promise');
export class CookieCutter implements TemplaterBase {
@@ -38,28 +36,32 @@ export class CookieCutter implements TemplaterBase {
}
}
public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult> {
public async run({
workspacePath,
dockerClient,
values,
logStream,
}: TemplaterRunOptions): Promise<void> {
const templateDir = path.join(workspacePath, 'template');
const intermediateDir = path.join(workspacePath, 'intermediate');
const resultDir = path.join(workspacePath, 'result');
// First lets grab the default cookiecutter.json file
const cookieCutterJson = await this.fetchTemplateCookieCutter(
options.directory,
);
const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir);
const cookieInfo = {
...cookieCutterJson,
...options.values,
...values,
};
await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
const templateDir = options.directory;
const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
await fs.writeJSON(`${templateDir}/cookiecutter.json`, cookieInfo);
const cookieCutterInstalled = await commandExists('cookiecutter');
if (cookieCutterInstalled) {
await runCommand({
command: 'cookiecutter',
args: ['--no-input', '-o', resultDir, templateDir, '--verbose'],
logStream: options.logStream,
args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'],
logStream,
});
} else {
await runDockerContainer({
@@ -73,22 +75,20 @@ export class CookieCutter implements TemplaterBase {
'--verbose',
],
templateDir,
resultDir,
logStream: options.logStream,
dockerClient: options.dockerClient,
resultDir: intermediateDir,
logStream,
dockerClient,
});
}
// if cookiecutter was successful, resultDir will contain
// if cookiecutter was successful, intermediateDir will contain
// exactly one directory.
const [generated] = await fs.readdir(resultDir);
const [generated] = await fs.readdir(intermediateDir);
if (generated === undefined) {
throw new Error('Cookie Cutter did not generate anything');
throw new Error('No data generated by cookiecutter');
}
return {
resultDir: path.resolve(resultDir, generated),
};
await fs.move(path.join(intermediateDir, generated), resultDir);
}
}
plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/index.ts
+74 -28
View File
@@ -17,25 +17,30 @@ import fs from 'fs-extra';
import { runDockerContainer } from '../helpers';
import { TemplaterBase, TemplaterRunOptions } from '..';
import path from 'path';
import { TemplaterRunResult } from '../types';
import * as yaml from 'yaml';
import { resolvePackagePath } from '@backstage/backend-common';
// TODO(blam): Replace with the universal import from github-actions after a release
// As it will break the E2E without it
const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug';
export class CreateReactAppTemplater implements TemplaterBase {
public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult> {
public async run({
workspacePath,
values,
logStream,
dockerClient,
}: TemplaterRunOptions): Promise<void> {
const {
component_id: componentName,
use_typescript: withTypescript,
use_github_actions: withGithubActions,
description,
owner,
} = options.values;
} = values;
const intermediateDir = path.join(workspacePath, 'template');
await fs.ensureDir(intermediateDir);
const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
const resultDir = path.join(workspacePath, 'result');
await runDockerContainer({
imageName: 'node:lts-alpine',
@@ -44,35 +49,80 @@ export class CreateReactAppTemplater implements TemplaterBase {
componentName as string,
withTypescript ? ' --template typescript' : '',
],
templateDir: options.directory,
resultDir,
logStream: options.logStream,
dockerClient: options.dockerClient,
templateDir: intermediateDir,
resultDir: intermediateDir,
logStream: logStream,
dockerClient: dockerClient,
createOptions: {
Entrypoint: ['npx'],
WorkingDir: '/result',
},
});
const extraAnnotations: Record<string, string> = {};
const finalDir = path.resolve(
resultDir,
options.values.component_id as string,
);
// if cookiecutter was successful, intermediateDir will contain
// exactly one directory.
const [generated] = await fs.readdir(intermediateDir);
if (generated === undefined) {
throw new Error('No data generated by cookiecutter');
}
await fs.move(path.join(intermediateDir, generated), resultDir);
const extraAnnotations: Record<string, string> = {};
if (withGithubActions) {
await fs.promises.mkdir(`${finalDir}/.github`);
await fs.promises.mkdir(`${finalDir}/.github/workflows`);
await fs.promises.copyFile(
`${resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
)}/templates/.github/workflows/main.yml`,
`${finalDir}/.github/workflows/main.yml`,
await fs.mkdir(`${resultDir}/.github`);
await fs.mkdir(`${resultDir}/.github/workflows`);
const githubActionsYaml = `
name: CRA Build
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [12.x]
steps:
- name: checkout code
uses: actions/checkout@v1
- name: get yarn cache
id: yarn-cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: \${{ steps.yarn-cache.outputs.dir }}
key: \${{ runner.os }}-yarn-\${{ hashFiles('**/yarn.lock') }}
restore-keys: |
\${{ runner.os }}-yarn-
- name: use node.js \${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: \${{ matrix.node-version }}
- name: yarn install, build, and test
working-directory: .
run: |
yarn install
yarn build
yarn test
env:
CI: true
`;
await fs.writeFile(
`${resultDir}/.github/workflows/main.yml`,
githubActionsYaml,
);
extraAnnotations[
GITHUB_ACTIONS_ANNOTATION
] = `${options.values?.destination?.git?.owner}/${options.values?.destination?.git?.name}`;
] = `${values?.destination?.git?.owner}/${values?.destination?.git?.name}`;
}
const componentInfo = {
@@ -92,13 +142,9 @@ export class CreateReactAppTemplater implements TemplaterBase {
},
};
await fs.promises.writeFile(
`${finalDir}/catalog-info.yaml`,
await fs.writeFile(
`${resultDir}/catalog-info.yaml`,
yaml.stringify(componentInfo),
);
return {
resultDir: finalDir,
};
}
}
plugins/scaffolder-backend/src/scaffolder/stages/templater/cra/templates/.github/workflows/main.yml
Vendored
-41
View File
@@ -1,41 +0,0 @@
name: CRA Build
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [12.x]
steps:
- name: checkout code
uses: actions/checkout@v1
- name: get yarn cache
id: yarn-cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
with:
path: ${{ steps.yarn-cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: use node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: yarn install, build, and test
working-directory: .
run: |
yarn install
yarn build
yarn test
env:
CI: true
plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.test.ts
+2 -90
View File
@@ -14,52 +14,13 @@
* limitations under the License.
*/
import { Templaters } from '.';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { CookieCutter } from './cookiecutter';
describe('Templaters', () => {
const mockTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'file:/Users/bingo/backstage/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml',
},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
templater: 'cookiecutter',
path: '.',
type: 'website',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
it('should throw an error when the templater is not registered', () => {
const templaters = new Templaters();
expect(() => templaters.get(mockTemplate)).toThrow(
expect(() => templaters.get('cookiecutter')).toThrow(
expect.objectContaining({
message: 'No templater registered for template: "cookiecutter"',
}),
@@ -71,55 +32,6 @@ describe('Templaters', () => {
templaters.register('cookiecutter', templater);
expect(templaters.get(mockTemplate)).toBe(templater);
});
it('should throw an error if the templater does not exist in the entity', () => {
const brokenTemplate: TemplateEntityV1alpha1 = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Template',
metadata: {
annotations: {},
name: 'react-ssr-template',
title: 'React SSR Template',
description:
'Next.js application skeleton for creating isomorphic web applications.',
uid: '7357f4c5-aa58-4a1e-9670-18931eef771f',
etag: 'YWUxZWQyY2EtZDkxMC00MDM0LWI0ODAtMDgwMWY0YzdlMWIw',
generation: 1,
},
spec: {
type: 'website',
path: '.',
templater: '',
schema: {
$schema: 'http://json-schema.org/draft-07/schema#',
required: ['storePath', 'owner'],
properties: {
owner: {
type: 'string',
title: 'Owner',
description: 'Who is going to own this component',
},
storePath: {
type: 'string',
title: 'Store path',
description: 'GitHub store path in org/repo format',
},
},
},
},
};
const templaters = new Templaters();
expect(() => templaters.get(brokenTemplate)).toThrow(
expect.objectContaining({
name: 'InputError',
message: expect.stringContaining(
'Template does not have a required templating key',
),
}),
);
expect(templaters.get('cookiecutter')).toBe(templater);
});
});
plugins/scaffolder-backend/src/scaffolder/stages/templater/templaters.ts
+7 -13
View File
@@ -20,26 +20,20 @@ import {
TemplaterBuilder,
} from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { getTemplaterKey } from './helpers';
export class Templaters implements TemplaterBuilder {
private preparerMap = new Map<SupportedTemplatingKey, TemplaterBase>();
private templaterMap = new Map<SupportedTemplatingKey, TemplaterBase>();
register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) {
this.preparerMap.set(templaterKey, templater);
this.templaterMap.set(templaterKey, templater);
}
get(template: TemplateEntityV1alpha1): TemplaterBase {
const templaterKey = getTemplaterKey(template);
const preparer = this.preparerMap.get(templaterKey);
get(templaterId: string): TemplaterBase {
const templater = this.templaterMap.get(templaterId);
if (!preparer) {
throw new Error(
`No templater registered for template: "${templaterKey}"`,
);
if (!templater) {
throw new Error(`No templater registered for template: "${templaterId}"`);
}
return preparer;
return templater;
}
}
plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts
+3 -4
View File
@@ -15,7 +15,6 @@
*/
import type { Writable } from 'stream';
import Docker from 'dockerode';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import gitUrlParse from 'git-url-parse';
/**
@@ -46,7 +45,7 @@ export type TemplaterRunResult = {
* client to run any templater on top of your directory.
*/
export type TemplaterRunOptions = {
directory: string;
workspacePath: string;
values: TemplaterValues;
logStream?: Writable;
dockerClient: Docker;
@@ -54,7 +53,7 @@ export type TemplaterRunOptions = {
export type TemplaterBase = {
// runs the templating with the values and returns the directory to push the VCS
run(opts: TemplaterRunOptions): Promise<TemplaterRunResult>;
run(opts: TemplaterRunOptions): Promise<void>;
};
export type TemplaterConfig = {
@@ -71,5 +70,5 @@ export type SupportedTemplatingKey = 'cookiecutter' | string;
*/
export type TemplaterBuilder = {
register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void;
get(template: TemplateEntityV1alpha1): TemplaterBase;
get(templater: string): TemplaterBase;
};
plugins/scaffolder-backend/src/service/router.test.ts
+8 -3
View File
@@ -23,6 +23,8 @@ jest.doMock('fs-extra', () => ({
F_OK: 0,
W_OK: 1,
},
mkdir: jest.fn(),
remove: jest.fn(),
}));
import { getVoidLogger } from '@backstage/backend-common';
@@ -116,9 +118,10 @@ describe('createRouter - working directory', () => {
},
});
expect(mockPrepare).toBeCalledWith(expect.anything(), {
expect(mockPrepare).toBeCalledWith({
logger: expect.anything(),
workingDirectory: '/path',
workspacePath: expect.stringContaining('path'),
url: expect.anything(),
});
});
@@ -143,8 +146,10 @@ describe('createRouter - working directory', () => {
},
});
expect(mockPrepare).toBeCalledWith(expect.anything(), {
expect(mockPrepare).toBeCalledWith({
logger: expect.anything(),
workspacePath: expect.anything(),
url: expect.anything(),
});
});
});
plugins/scaffolder-backend/src/service/router.ts
+40 -42
View File
@@ -15,19 +15,19 @@
*/
import { Config } from '@backstage/config';
import fs from 'fs-extra';
import Docker from 'dockerode';
import express from 'express';
import { resolve as resolvePath } from 'path';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import {
JobProcessor,
PreparerBuilder,
StageContext,
TemplaterBuilder,
TemplaterValues,
PublisherBuilder,
parseLocationAnnotation,
joinGitUrlPath,
FilePreparer,
} from '../scaffolder';
import { CatalogEntityClient } from '../lib/catalog';
@@ -62,27 +62,7 @@ export async function createRouter(
} = options;
const logger = parentLogger.child({ plugin: 'scaffolder' });
const jobProcessor = new JobProcessor();
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
}
const jobProcessor = await JobProcessor.fromConfig({ config, logger });
router
.get('/v1/job/:jobId', ({ params }, res) => {
@@ -128,51 +108,69 @@ export async function createRouter(
res.status(400).json({ errors: validationResult.errors });
return;
}
const job = jobProcessor.create({
entity: template,
values,
stages: [
{
name: 'Prepare the skeleton',
handler: async ctx => {
const { protocol, location: pullPath } = parseLocationAnnotation(
ctx.entity,
async handler(ctx) {
const {
protocol,
location: templateEntityLocation,
} = parseLocationAnnotation(ctx.entity);
if (protocol === 'file') {
const preparer = new FilePreparer();
const path = resolvePath(
templateEntityLocation,
template.spec.path || '.',
);
await preparer.prepare({
url: `file://${path}`,
logger: ctx.logger,
workspacePath: ctx.workspacePath,
});
return;
}
const preparer = preparers.get(templateEntityLocation);
const url = joinGitUrlPath(
templateEntityLocation,
template.spec.path,
);
const preparer =
protocol === 'file'
? new FilePreparer()
: preparers.get(pullPath);
const skeletonDir = await preparer.prepare(ctx.entity, {
await preparer.prepare({
url,
logger: ctx.logger,
workingDirectory,
workspacePath: ctx.workspacePath,
});
return { skeletonDir };
},
},
{
name: 'Run the templater',
handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
const templater = templaters.get(ctx.entity);
const { resultDir } = await templater.run({
directory: ctx.skeletonDir,
async handler(ctx) {
const templater = templaters.get(ctx.entity.spec.templater);
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
logStream: ctx.logStream,
values: ctx.values,
});
return { resultDir };
},
},
{
name: 'Publish template',
handler: async (ctx: StageContext<{ resultDir: string }>) => {
handler: async ctx => {
const publisher = publishers.get(ctx.values.storePath);
ctx.logger.info('Will now store the template');
const result = await publisher.publish({
values: ctx.values,
directory: ctx.resultDir,
workspacePath: ctx.workspacePath,
logger: ctx.logger,
});
return result;