paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

Merge pull request #5849 from SDA-SE/feat/scaffolder-conditional

Browse Source 
Conditional scaffolder steps
This commit is contained in:
Fredrik Adelöw
2021-06-08 14:25:49 +02:00
committed by GitHub
parent fcb3c7216a eca511cb37
commit 28fc26ddf4
26 changed files with 554 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/scaffolder-backend/sample-templates/v1beta2-demo/template.yaml
+22 -4
View File
@@ -35,13 +35,14 @@ spec:
description: System of the component
ui:field: EntityPicker
ui:options:
allowedKinds:
allowedKinds:
- System
defaultKind: System
- title: Choose a location
required:
- repoUrl
- dryRun
properties:
repoUrl:
title: Repository Location
@@ -50,6 +51,10 @@ spec:
ui:options:
allowedHosts:
- github.com
dryRun:
title: Only perform a dry run, don't publish anything
type: boolean
default: false
steps:
- id: fetch-base
@@ -59,7 +64,8 @@ spec:
url: ./template
values:
name: '{{ parameters.name }}'
owner: '{{ parameters.owner }}'
owner: '{{ parameters.owner }}'
system: '{{ parameters.system }}'
destination: '{{ parseRepoUrl parameters.repoUrl }}'
- id: fetch-docs
@@ -70,6 +76,7 @@ spec:
url: https://github.com/backstage/community/tree/main/backstage-community-sessions
- id: publish
if: '{{ not parameters.dryRun }}'
name: Publish
action: publish:github
input:
@@ -78,12 +85,23 @@ spec:
repoUrl: '{{ parameters.repoUrl }}'
- id: register
if: '{{ not parameters.dryRun }}'
name: Register
action: catalog:register
input:
repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}'
catalogInfoPath: '/catalog-info.yaml'
- name: Results
if: '{{ parameters.dryRun }}'
action: debug:log
input:
listWorkspace: true
output:
remoteUrl: '{{ steps.publish.output.remoteUrl }}'
entityRef: '{{ steps.register.output.entityRef }}'
links:
- title: Repository
url: '{{ steps.publish.output.remoteUrl }}'
- title: Open in catalog
icon: 'catalog'
entityRef: '{{ steps.register.output.entityRef }}'
plugins/scaffolder-backend/sample-templates/v1beta2-demo/template/catalog-info.yaml
+1 -1
View File
@@ -7,6 +7,6 @@ spec:
type: website
lifecycle: experimental
owner: {{cookiecutter.owner | jsonify}}
{%- if cookiecutter.backstage_system != "" %}
{%- if 'system' in cookiecutter %}
system: {{ cookiecutter.system | jsonify }}
{%- endif %}
plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts
+2
View File
@@ -19,6 +19,7 @@ import { CatalogApi } from '@backstage/catalog-client';
import { ScmIntegrations } from '@backstage/integration';
import { TemplaterBuilder } from '../../stages';
import { createCatalogRegisterAction } from './catalog';
import { createDebugLogAction } from './debug';
import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch';
import {
createPublishAzureAction,
@@ -61,6 +62,7 @@ export const createBuiltinActions = (options: {
createPublishAzureAction({
integrations,
}),
createDebugLogAction(),
createCatalogRegisterAction({ catalogClient, integrations }),
];
};
plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/index.ts
+17
View File
@@ -0,0 +1,17 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { createDebugLogAction } from './log';
plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.test.ts
+94
View File
@@ -0,0 +1,94 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import mock from 'mock-fs';
import os from 'os';
import { Writable } from 'stream';
import { createDebugLogAction } from './log';
import { join } from 'path';
describe('debug:log', () => {
const logStream = ({
write: jest.fn(),
} as jest.Mocked<Partial<Writable>>) as jest.Mocked<Writable>;
const mockTmpDir = os.tmpdir();
const mockContext = {
input: {},
baseUrl: 'somebase',
workspacePath: mockTmpDir,
logger: getVoidLogger(),
logStream,
output: jest.fn(),
createTemporaryDirectory: jest.fn().mockResolvedValue(mockTmpDir),
};
const action = createDebugLogAction();
beforeEach(() => {
mock({
[`${mockContext.workspacePath}/README.md`]: '',
[`${mockContext.workspacePath}/a-directory/index.md`]: '',
});
jest.resetAllMocks();
});
afterEach(() => {
mock.restore();
});
it('should do nothing', async () => {
await action.handler(mockContext);
expect(logStream.write).toBeCalledTimes(0);
});
it('should log the workspace content, if active', async () => {
const context = {
...mockContext,
input: {
listWorkspace: 'true',
},
};
await action.handler(context);
expect(logStream.write).toBeCalledTimes(1);
expect(logStream.write).toBeCalledWith(
expect.stringContaining('README.md'),
);
expect(logStream.write).toBeCalledWith(
expect.stringContaining(join('a-directory', 'index.md')),
);
});
it('should log message', async () => {
const context = {
...mockContext,
input: {
message: 'Hello Backstage!',
},
};
await action.handler(context);
expect(logStream.write).toBeCalledTimes(1);
expect(logStream.write).toBeCalledWith(
expect.stringContaining('Hello Backstage!'),
);
});
});
plugins/scaffolder-backend/src/scaffolder/actions/builtin/debug/log.ts
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { readdir, stat } from 'fs-extra';
import { relative, resolve } from 'path';
import { createTemplateAction } from '../../createTemplateAction';
/**
* This task is useful for local development and testing of both the scaffolder
* and scaffolder templates.
*/
export function createDebugLogAction() {
return createTemplateAction<{ message?: string; listWorkspace?: boolean }>({
id: 'debug:log',
description:
'Writes a message into the log or list all files in the workspace.',
schema: {
input: {
type: 'object',
properties: {
message: {
title: 'Message to output.',
type: 'string',
},
listWorkspace: {
title: 'List all files in the workspace, if true.',
type: 'boolean',
},
},
},
},
async handler(ctx) {
if (ctx.input?.message) {
ctx.logStream.write(ctx.input.message);
}
if (ctx.input?.listWorkspace) {
const files = await recursiveReadDir(ctx.workspacePath);
ctx.logStream.write(
`Workspace:\n${files
.map(f => ` - ${relative(ctx.workspacePath, f)}`)
.join('\n')}`,
);
}
},
});
}
export async function recursiveReadDir(dir: string): Promise<string[]> {
const subdirs = await readdir(dir);
const files = await Promise.all(
subdirs.map(async subdir => {
const res = resolve(dir, subdir);
return (await stat(res)).isDirectory() ? recursiveReadDir(res) : [res];
}),
);
return files.reduce((a, f) => a.concat(f), []);
}
plugins/scaffolder-backend/src/scaffolder/actions/builtin/index.ts
+2 -1
View File
@@ -15,6 +15,7 @@
*/
export * from './catalog';
export { createBuiltinActions } from './createBuiltinActions';
export * from './debug';
export * from './fetch';
export * from './publish';
export { createBuiltinActions } from './createBuiltinActions';
plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/index.ts
+3 -3
View File
@@ -14,9 +14,9 @@
* limitations under the License.
*/
export { createPublishGithubAction } from './github';
export { createPublishGithubPullRequestAction } from './githubPullRequest';
export { createPublishAzureAction } from './azure';
export { createPublishGitlabAction } from './gitlab';
export { createPublishBitbucketAction } from './bitbucket';
export { createPublishFileAction } from './file';
export { createPublishGithubAction } from './github';
export { createPublishGithubPullRequestAction } from './githubPullRequest';
export { createPublishGitlabAction } from './gitlab';
plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.test.ts
+71 -4
View File
@@ -18,13 +18,13 @@ import {
getVoidLogger,
SingleConnectionDatabaseManager,
} from '@backstage/backend-common';
import { TaskWorker } from './TaskWorker';
import os from 'os';
import { ConfigReader, JsonObject } from '@backstage/config';
import { StorageTaskBroker } from './StorageTaskBroker';
import { DatabaseTaskStore } from './DatabaseTaskStore';
import os from 'os';
import { createTemplateAction, TemplateActionRegistry } from '../actions';
import { RepoSpec } from '../actions/builtin/publish/util';
import { DatabaseTaskStore } from './DatabaseTaskStore';
import { StorageTaskBroker } from './StorageTaskBroker';
import { TaskWorker } from './TaskWorker';
async function createStore(): Promise<DatabaseTaskStore> {
const manager = SingleConnectionDatabaseManager.fromConfig(
@@ -54,6 +54,7 @@ describe('TaskWorker', () => {
id: 'test-action',
handler: async ctx => {
ctx.output('testOutput', 'winning');
ctx.output('badOutput', false);
},
});
});
@@ -172,6 +173,72 @@ describe('TaskWorker', () => {
expect((event?.body?.output as JsonObject).result).toBe('winning');
});
it('should execute steps conditionally', async () => {
const broker = new StorageTaskBroker(storage, logger);
const taskWorker = new TaskWorker({
logger,
workingDirectory: os.tmpdir(),
actionRegistry,
taskBroker: broker,
});
const { taskId } = await broker.dispatch({
steps: [
{ id: 'test', name: 'test', action: 'test-action' },
{
id: 'conditional',
name: 'conditional',
action: 'test-action',
if: '{{ steps.test.output.testOutput }}',
},
],
output: {
result: '{{ steps.conditional.output.testOutput }}',
},
values: {},
});
const task = await broker.claim();
await taskWorker.runOneTask(task);
const { events } = await storage.listEvents({ taskId });
const event = events.find(e => e.type === 'completion');
expect((event?.body?.output as JsonObject).result).toBe('winning');
});
it('should skip steps conditionally', async () => {
const broker = new StorageTaskBroker(storage, logger);
const taskWorker = new TaskWorker({
logger,
workingDirectory: os.tmpdir(),
actionRegistry,
taskBroker: broker,
});
const { taskId } = await broker.dispatch({
steps: [
{ id: 'test', name: 'test', action: 'test-action' },
{
id: 'conditional',
name: 'conditional',
action: 'test-action',
if: '{{ steps.test.output.badOutput }}',
},
],
output: {
result: '{{ steps.conditional.output.testOutput }}',
},
values: {},
});
const task = await broker.claim();
await taskWorker.runOneTask(task);
const { events } = await storage.listEvents({ taskId });
const event = events.find(e => e.type === 'completion');
expect((event?.body?.output as JsonObject).result).toBeUndefined();
});
it('should parse strings as objects if possible', async () => {
const inputAction = createTemplateAction<{
address: { line1: string };
plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
+86 -13
View File
@@ -14,18 +14,19 @@
* limitations under the License.
*/
import { PassThrough } from 'stream';
import { Logger } from 'winston';
import * as winston from 'winston';
import { JsonValue, JsonObject } from '@backstage/config';
import { validate as validateJsonSchema } from 'jsonschema';
import { TaskBroker, Task } from './types';
import fs from 'fs-extra';
import path from 'path';
import { TemplateActionRegistry } from '../actions/TemplateActionRegistry';
import * as Handlebars from 'handlebars';
import { JsonObject, JsonValue } from '@backstage/config';
import { InputError } from '@backstage/errors';
import fs from 'fs-extra';
import * as Handlebars from 'handlebars';
import { validate as validateJsonSchema } from 'jsonschema';
import path from 'path';
import { PassThrough } from 'stream';
import * as winston from 'winston';
import { Logger } from 'winston';
import { parseRepoUrl } from '../actions/builtin/publish/util';
import { TemplateActionRegistry } from '../actions/TemplateActionRegistry';
import { isTruthy } from './helper';
import { Task, TaskBroker } from './types';
type Options = {
logger: Logger;
@@ -48,6 +49,8 @@ export class TaskWorker {
});
this.handlebars.registerHelper('json', obj => JSON.stringify(obj));
this.handlebars.registerHelper('not', value => !isTruthy(value));
}
start() {
@@ -102,6 +105,51 @@ export class TaskWorker {
});
taskLogger.add(new winston.transports.Stream({ stream }));
if (step.if !== undefined) {
// Support passing values like false to disable steps
let skip = !step.if;
// Evaluate strings as handlebar templates
if (typeof step.if === 'string') {
const condition = JSON.parse(
JSON.stringify(step.if),
(_key, value) => {
if (typeof value === 'string') {
const templated = this.handlebars.compile(value, {
noEscape: true,
data: false,
preventIndent: true,
})(templateCtx);
// If it's just an empty string, treat it as undefined
if (templated === '') {
return undefined;
}
try {
return JSON.parse(templated);
} catch {
return templated;
}
}
return value;
},
);
skip = !isTruthy(condition);
}
if (skip) {
await task.emitLog(`Skipped step ${step.name}`, {
...metadata,
status: 'skipped',
});
continue;
}
}
await task.emitLog(`Beginning step ${step.name}`, {
...metadata,
status: 'processing',
@@ -118,13 +166,18 @@ export class TaskWorker {
if (typeof value === 'string') {
const templated = this.handlebars.compile(value, {
noEscape: true,
strict: true,
data: false,
preventIndent: true,
})(templateCtx);
// If it's just an empty string, treat it as undefined
if (templated === '') {
return undefined;
}
// If it smells like a JSON object then give it a parse as an object and if it fails return the string
if (
(templated.startsWith('"') && templated.endsWith('"')) ||
(templated.startsWith('{') && templated.endsWith('}')) ||
(templated.startsWith('[') && templated.endsWith(']'))
) {
@@ -203,12 +256,32 @@ export class TaskWorker {
JSON.stringify(task.spec.output),
(_key, value) => {
if (typeof value === 'string') {
return this.handlebars.compile(value, {
const templated = this.handlebars.compile(value, {
noEscape: true,
strict: true,
data: false,
preventIndent: true,
})(templateCtx);
// If it's just an empty string, treat it as undefined
if (templated === '') {
return undefined;
}
// If it smells like a JSON object then give it a parse as an object and if it fails return the string
if (
(templated.startsWith('"') && templated.endsWith('"')) ||
(templated.startsWith('{') && templated.endsWith('}')) ||
(templated.startsWith('[') && templated.endsWith(']'))
) {
try {
// Don't recursively JSON parse the values of this string.
// Shouldn't need to, don't want to encourage the use of returning handlebars from somewhere else
return JSON.parse(templated);
} catch {
return templated;
}
}
return templated;
}
return value;
},
plugins/scaffolder-backend/src/scaffolder/tasks/helper.test.ts
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { isTruthy } from './helper';
describe('isTruthy', () => {
it.each`
value | result
${'string'} | ${true}
${true} | ${true}
${1} | ${true}
${['1']} | ${true}
${{}} | ${true}
${false} | ${false}
${''} | ${false}
${undefined} | ${false}
${null} | ${false}
${0} | ${false}
${[]} | ${false}
`('should be $result for $value', async ({ value, result }) => {
expect(isTruthy(value)).toEqual(result);
});
});
plugins/scaffolder-backend/src/scaffolder/tasks/helper.ts
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { isArray } from 'lodash';
/**
* Returns true if the input is not `false`, `undefined`, `null`, `""`, `0`, or
* `[]`. This behavior is based on the behavior of handlebars, see
* https://handlebarsjs.com/guide/builtin-helpers.html#if
*/
export function isTruthy(value: any): boolean {
return isArray(value) ? value.length > 0 : !!value;
}
plugins/scaffolder-backend/src/scaffolder/tasks/types.ts
+1
View File
@@ -51,6 +51,7 @@ export type TaskSpec = {
name: string;
action: string;
input?: JsonObject;
if?: string | boolean;
}>;
output: { [name: string]: string };
};