Refactor scaffolder

Co-authored-by: Fredrik Adelöw <freben@gmail.com>
Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Johan Haals
2021-01-26 16:42:41 +01:00
parent 5190af378d
commit 92d39a0c9f
14 changed files with 218 additions and 229 deletions
@@ -13,13 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import { Processor, Job, StageContext, StageInput } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import * as uuid from 'uuid';
import Docker from 'dockerode';
import path from 'path';
import { TemplaterValues, TemplaterBase } from '../stages/templater';
import { PreparerBuilder } from '../stages/prepare';
import { makeLogStream } from './logger';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
export type JobProcessorArguments = {
preparers: PreparerBuilder;
@@ -33,7 +39,45 @@ export type JobAndDirectoryTuple = {
};
export class JobProcessor implements Processor {
private jobs = new Map<string, Job>();
private readonly workingDirectory: string;
private readonly jobs: Map<string, Job>;
static async fromConfig({
config,
logger,
}: {
config: Config;
logger: Logger;
}) {
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
} else {
workingDirectory = os.tmpdir();
}
return new JobProcessor(workingDirectory);
}
constructor(workingDirectory: string) {
this.workingDirectory = workingDirectory;
this.jobs = new Map<string, Job>();
}
create({
entity,
@@ -52,6 +96,7 @@ export class JobProcessor implements Processor {
values,
logger,
logStream: stream,
workspacePath: path.join(this.workingDirectory, id),
};
const job: Job = {
@@ -80,6 +125,8 @@ export class JobProcessor implements Processor {
throw new Error("Job is not in a 'PENDING' state");
}
await fs.mkdir(job.context.workspacePath);
job.status = 'STARTED';
try {
@@ -134,6 +181,8 @@ export class JobProcessor implements Processor {
// If something went wrong, fail the job, and set the error property on the job.
job.error = { name: error.name, message: error.message };
job.status = 'FAILED';
} finally {
await fs.remove(job.context.workspacePath);
}
}
}
@@ -26,6 +26,7 @@ export type StageContext<T = {}> = {
entity: TemplateEntityV1alpha1;
logger: Logger;
logStream: Writable;
workspacePath: string;
} & T;
export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,25 +27,13 @@ export class AzurePreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
// Username can be anything but the empty string according to:
@@ -62,10 +47,16 @@ export class AzurePreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import { BitbucketIntegrationConfig } from '@backstage/integration';
@@ -40,43 +37,29 @@ export class BitbucketPreparer implements PreparerBase {
},
) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const repo = parseGitUrl(location);
const repositoryCheckoutUrl = repo.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(repo.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const auth = this.getAuth();
const git = auth
? Git.fromAuth({
...auth,
logger,
})
: Git.fromAuth({ logger });
const git = Git.fromAuth({ logger, ...this.getAuth() });
await git.clone({
url: repositoryCheckoutUrl,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
private getAuth(): { username: string; password: string } | undefined {
@@ -13,43 +13,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { InputError } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
export class FilePreparer implements PreparerBase {
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { protocol, location } = parseLocationAnnotation(template);
const workingDirectory = opts?.workingDirectory ?? os.tmpdir();
if (protocol !== 'file') {
throw new InputError(
`Wrong location protocol: ${protocol}, should be 'file'`,
);
async prepare({ url, workspacePath }: PreparerOptions) {
if (!url.startsWith('file:///')) {
throw new InputError(`Wrong location protocol, should be 'file', ${url}`);
}
const templateId = template.metadata.name;
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const checkoutDir = path.join(workspacePath, 'checkout');
await fs.ensureDir(checkoutDir);
const parentDirectory = path.resolve(
path.dirname(location),
template.spec.path ?? '.',
);
const templatePath = url.slice('file://'.length);
await fs.copy(parentDirectory, tempDir, {
filter: src => src !== location,
await fs.copy(templatePath, checkoutDir, {
recursive: true,
});
return tempDir;
}
}
@@ -13,11 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import os from 'os';
import fs from 'fs-extra';
import path from 'path';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { parseLocationAnnotation } from '../helpers';
import { Git } from '@backstage/backend-common';
import { PreparerBase, PreparerOptions } from './types';
import parseGitUrl from 'git-url-parse';
@@ -30,29 +27,15 @@ export class GithubPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const logger = opts.logger;
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
);
const checkoutLocation = path.resolve(tempDir, templateDirectory);
const git = this.config.token
? Git.fromAuth({
username: this.config.token,
@@ -62,10 +45,16 @@ export class GithubPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
});
return checkoutLocation;
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -14,13 +14,10 @@
* limitations under the License.
*/
import { Git } from '@backstage/backend-common';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { GitLabIntegrationConfig } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
import path from 'path';
import { parseLocationAnnotation } from '../helpers';
import { PreparerBase, PreparerOptions } from './types';
export class GitlabPreparer implements PreparerBase {
@@ -30,25 +27,13 @@ export class GitlabPreparer implements PreparerBase {
constructor(private readonly config: { token?: string }) {}
async prepare(
template: TemplateEntityV1alpha1,
opts: PreparerOptions,
): Promise<string> {
const { location } = parseLocationAnnotation(template);
const logger = opts.logger;
const workingDirectory = opts.workingDirectory ?? os.tmpdir();
const templateId = template.metadata.name;
const parsedGitLocation = parseGitUrl(location);
const repositoryCheckoutUrl = parsedGitLocation.toString('https');
const tempDir = await fs.promises.mkdtemp(
path.join(workingDirectory, templateId),
);
const templateDirectory = path.join(
`${path.dirname(parsedGitLocation.filepath)}`,
template.spec.path ?? '.',
async prepare({ url, workspacePath, logger }: PreparerOptions) {
const parsedGitUrl = parseGitUrl(url);
const checkoutPath = path.join(workspacePath, 'checkout');
const targetPath = path.join(workspacePath, 'template');
const fullPathToTemplate = path.resolve(
checkoutPath,
parsedGitUrl.filepath,
);
const git = this.config.token
@@ -60,10 +45,16 @@ export class GitlabPreparer implements PreparerBase {
: Git.fromAuth({ logger });
await git.clone({
url: repositoryCheckoutUrl,
dir: tempDir,
url: parsedGitUrl.toString('https'),
dir: checkoutPath,
});
return path.resolve(tempDir, templateDirectory);
await fs.move(fullPathToTemplate, targetPath);
try {
await fs.rmdir(path.join(targetPath, '.git'));
} catch {
// Ignore intentionally
}
}
}
@@ -13,24 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
export type PreparerOptions = {
workingDirectory?: string;
/**
* Full URL to the directory containg template data
*/
url: string;
/**
* The workspace path that will eventually be the the root of the new repo
*/
workspacePath: string;
logger: Logger;
};
export interface PreparerBase {
/**
* Given an Entity definition from the Service Catalog, go and prepare a directory
* with contents from the remote location in temporary storage and return the path
* @param template The template entity from the Service Catalog
* Prepare a directory with contents from the remote location
*/
prepare(
template: TemplateEntityV1alpha1,
opts?: PreparerOptions,
): Promise<string>;
prepare(opts: PreparerOptions): Promise<void>;
}
export type PreparerBuilder = {
@@ -19,8 +19,10 @@ import { initRepoAndPush } from './helpers';
import { GitHubIntegrationConfig } from '@backstage/integration';
import parseGitUrl from 'git-url-parse';
import { Octokit } from '@octokit/rest';
import path from 'path';
export type RepoVisibilityOptions = 'private' | 'internal' | 'public';
export class GithubPublisher implements PublisherBase {
static async fromConfig(
config: GitHubIntegrationConfig,
@@ -41,6 +43,7 @@ export class GithubPublisher implements PublisherBase {
repoVisibility,
});
}
constructor(
private readonly config: {
token: string;
@@ -51,7 +54,7 @@ export class GithubPublisher implements PublisherBase {
async publish({
values,
directory,
workspacePath,
logger,
}: PublisherOptions): Promise<PublisherResult> {
const { owner, name } = parseGitUrl(values.storePath);
@@ -66,7 +69,7 @@ export class GithubPublisher implements PublisherBase {
});
await initRepoAndPush({
dir: directory,
dir: path.join(workspacePath, 'result'),
remoteUrl,
auth: {
username: this.config.token,
@@ -79,7 +82,6 @@ export class GithubPublisher implements PublisherBase {
/\.git$/,
'/blob/master/catalog-info.yaml',
);
return { remoteUrl, catalogInfoUrl };
}
@@ -32,7 +32,7 @@ export type PublisherBase = {
export type PublisherOptions = {
values: TemplaterValues;
directory: string;
workspacePath: string;
logger: Logger;
};
@@ -19,8 +19,6 @@ import { runDockerContainer, runCommand } from './helpers';
import { TemplaterBase, TemplaterRunOptions } from '.';
import path from 'path';
import { TemplaterRunResult } from './types';
const commandExists = require('command-exists-promise');
export class CookieCutter implements TemplaterBase {
@@ -38,28 +36,32 @@ export class CookieCutter implements TemplaterBase {
}
}
public async run(options: TemplaterRunOptions): Promise<TemplaterRunResult> {
public async run({
workspacePath,
dockerClient,
values,
logStream,
}: TemplaterRunOptions): Promise<void> {
const templateDir = path.join(workspacePath, 'template');
const intermediateDir = path.join(workspacePath, 'intermediate');
const resultDir = path.join(workspacePath, 'result');
// First lets grab the default cookiecutter.json file
const cookieCutterJson = await this.fetchTemplateCookieCutter(
options.directory,
);
const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir);
const cookieInfo = {
...cookieCutterJson,
...options.values,
...values,
};
await fs.writeJSON(`${options.directory}/cookiecutter.json`, cookieInfo);
const templateDir = options.directory;
const resultDir = await fs.promises.mkdtemp(`${options.directory}-result`);
await fs.writeJSON(`${templateDir}/cookiecutter.json`, cookieInfo);
const cookieCutterInstalled = await commandExists('cookiecutter');
if (cookieCutterInstalled) {
await runCommand({
command: 'cookiecutter',
args: ['--no-input', '-o', resultDir, templateDir, '--verbose'],
logStream: options.logStream,
args: ['--no-input', '-o', intermediateDir, templateDir, '--verbose'],
logStream,
});
} else {
await runDockerContainer({
@@ -73,22 +75,20 @@ export class CookieCutter implements TemplaterBase {
'--verbose',
],
templateDir,
resultDir,
logStream: options.logStream,
dockerClient: options.dockerClient,
resultDir: intermediateDir,
logStream,
dockerClient,
});
}
// if cookiecutter was successful, resultDir will contain
// if cookiecutter was successful, intermediateDir will contain
// exactly one directory.
const [generated] = await fs.readdir(resultDir);
const [generated] = await fs.readdir(intermediateDir);
if (generated === undefined) {
throw new Error('Cookie Cutter did not generate anything');
throw new Error('No data generated by cookiecutter');
}
return {
resultDir: path.resolve(resultDir, generated),
};
await fs.move(path.join(intermediateDir, generated), resultDir);
}
}
@@ -20,26 +20,20 @@ import {
TemplaterBuilder,
} from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { getTemplaterKey } from './helpers';
export class Templaters implements TemplaterBuilder {
private preparerMap = new Map<SupportedTemplatingKey, TemplaterBase>();
private templaterMap = new Map<SupportedTemplatingKey, TemplaterBase>();
register(templaterKey: SupportedTemplatingKey, templater: TemplaterBase) {
this.preparerMap.set(templaterKey, templater);
this.templaterMap.set(templaterKey, templater);
}
get(template: TemplateEntityV1alpha1): TemplaterBase {
const templaterKey = getTemplaterKey(template);
const preparer = this.preparerMap.get(templaterKey);
get(templaterId: string): TemplaterBase {
const templater = this.templaterMap.get(templaterId);
if (!preparer) {
throw new Error(
`No templater registered for template: "${templaterKey}"`,
);
if (!templater) {
throw new Error(`No templater registered for template: "${templaterId}"`);
}
return preparer;
return templater;
}
}
@@ -15,7 +15,6 @@
*/
import type { Writable } from 'stream';
import Docker from 'dockerode';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import gitUrlParse from 'git-url-parse';
/**
@@ -46,7 +45,7 @@ export type TemplaterRunResult = {
* client to run any templater on top of your directory.
*/
export type TemplaterRunOptions = {
directory: string;
workspacePath: string;
values: TemplaterValues;
logStream?: Writable;
dockerClient: Docker;
@@ -54,7 +53,7 @@ export type TemplaterRunOptions = {
export type TemplaterBase = {
// runs the templating with the values and returns the directory to push the VCS
run(opts: TemplaterRunOptions): Promise<TemplaterRunResult>;
run(opts: TemplaterRunOptions): Promise<void>;
};
export type TemplaterConfig = {
@@ -71,5 +70,5 @@ export type SupportedTemplatingKey = 'cookiecutter' | string;
*/
export type TemplaterBuilder = {
register(protocol: SupportedTemplatingKey, templater: TemplaterBase): void;
get(template: TemplateEntityV1alpha1): TemplaterBase;
get(templater: string): TemplaterBase;
};
@@ -15,7 +15,6 @@
*/
import { Config } from '@backstage/config';
import fs from 'fs-extra';
import Docker from 'dockerode';
import express from 'express';
import Router from 'express-promise-router';
@@ -33,6 +32,7 @@ import {
import { CatalogEntityClient } from '../lib/catalog';
import { validate, ValidatorResult } from 'jsonschema';
import parseGitUrl from 'git-url-parse';
import { PreparerBase } from '../scaffolder/stages/prepare';
export interface RouterOptions {
preparers: PreparerBuilder;
@@ -62,27 +62,7 @@ export async function createRouter(
} = options;
const logger = parentLogger.child({ plugin: 'scaffolder' });
const jobProcessor = new JobProcessor();
let workingDirectory: string;
if (config.has('backend.workingDirectory')) {
workingDirectory = config.getString('backend.workingDirectory');
try {
// Check if working directory exists and is writable
await fs.promises.access(
workingDirectory,
fs.constants.F_OK | fs.constants.W_OK,
);
logger.info(`using working directory: ${workingDirectory}`);
} catch (err) {
logger.error(
`working directory ${workingDirectory} ${
err.code === 'ENOENT' ? 'does not exist' : 'is not writable'
}`,
);
throw err;
}
}
const jobProcessor = await JobProcessor.fromConfig({ config, logger });
router
.get('/v1/job/:jobId', ({ params }, res) => {
@@ -128,51 +108,79 @@ export async function createRouter(
res.status(400).json({ errors: validationResult.errors });
return;
}
const job = jobProcessor.create({
entity: template,
values,
stages: [
{
name: 'Prepare the skeleton',
handler: async ctx => {
const { protocol, location: pullPath } = parseLocationAnnotation(
ctx.entity,
);
async handler(ctx) {
const {
protocol,
location: templateEntityLocation,
} = parseLocationAnnotation(ctx.entity);
const preparer =
if (protocol === 'file') {
const preparer: PreparerBase =
protocol === 'file'
? new FilePreparer()
: preparers.get(templateEntityLocation);
const url = new URL(
template.spec.path || '.',
templateEntityLocation,
)
.toString()
.replace(/\/$/, '');
await preparer.prepare({
url,
logger: ctx.logger,
workspacePath: ctx.workspacePath,
});
return;
}
const preparer: PreparerBase =
protocol === 'file'
? new FilePreparer()
: preparers.get(pullPath);
: preparers.get(templateEntityLocation);
const skeletonDir = await preparer.prepare(ctx.entity, {
const url = new URL(
template.spec.path || '.',
templateEntityLocation,
)
.toString()
.replace(/\/$/, '');
await preparer.prepare({
url,
logger: ctx.logger,
workingDirectory,
workspacePath: ctx.workspacePath,
});
return { skeletonDir };
},
},
{
name: 'Run the templater',
handler: async (ctx: StageContext<{ skeletonDir: string }>) => {
const templater = templaters.get(ctx.entity);
const { resultDir } = await templater.run({
directory: ctx.skeletonDir,
async handler(ctx) {
const templater = templaters.get(ctx.entity.spec.templater);
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
logStream: ctx.logStream,
values: ctx.values,
});
return { resultDir };
},
},
{
name: 'Publish template',
handler: async (ctx: StageContext<{ resultDir: string }>) => {
handler: async ctx => {
const publisher = publishers.get(ctx.values.storePath);
ctx.logger.info('Will now store the template');
const result = await publisher.publish({
values: ctx.values,
directory: ctx.resultDir,
workspacePath: ctx.workspacePath,
logger: ctx.logger,
});
return result;