paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 40 Packages Projects Releases Wiki Activity

Merge pull request #4547 from backstage/mob/scaffolder-frontend

Browse Source 
Stateless scaffolding
This commit is contained in:
Johan Haals
2021-02-19 10:39:15 +01:00
committed by GitHub
parent dedee41f57 10e08f9adb
commit fe2a804742
52 changed files with 1362 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog/package.json
-1
View File
@@ -34,7 +34,6 @@
"@backstage/catalog-model": "^0.7.1",
"@backstage/core": "^0.6.2",
"@backstage/plugin-catalog-react": "^0.0.4",
"@backstage/plugin-scaffolder": "^0.5.1",
"@backstage/theme": "^0.2.3",
"@material-ui/core": "^4.11.0",
"@material-ui/icons": "^4.9.1",
plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx
+6
View File
@@ -33,6 +33,7 @@ import { MockStorageApi, wrapInTestApp } from '@backstage/test-utils';
import { fireEvent, render, waitFor } from '@testing-library/react';
import React from 'react';
import { EntityFilterGroupsProvider } from '../../filter';
import { createComponentRouteRef } from '../../routes';
import { CatalogPage } from './CatalogPage';
describe('CatalogPage', () => {
@@ -116,6 +117,11 @@ describe('CatalogPage', () => {
>
<EntityFilterGroupsProvider>{children}</EntityFilterGroupsProvider>,
</ApiProvider>,
{
mountedRoutes: {
'/create': createComponentRouteRef,
},
},
),
);
plugins/catalog/src/components/CatalogPage/CatalogPage.tsx
+4 -3
View File
@@ -21,9 +21,9 @@ import {
errorApiRef,
SupportButton,
useApi,
useRouteRef,
} from '@backstage/core';
import { catalogApiRef, isOwnerOf } from '@backstage/plugin-catalog-react';
import { rootRoute as scaffolderRootRoute } from '@backstage/plugin-scaffolder';
import { Button, makeStyles } from '@material-ui/core';
import SettingsIcon from '@material-ui/icons/Settings';
import StarIcon from '@material-ui/icons/Star';
@@ -31,6 +31,7 @@ import React, { useCallback, useMemo, useState } from 'react';
import { Link as RouterLink } from 'react-router-dom';
import { EntityFilterGroupsProvider, useFilteredEntities } from '../../filter';
import { useStarredEntities } from '../../hooks/useStarredEntities';
import { createComponentRouteRef } from '../../routes';
import {
ButtonGroup,
CatalogFilter,
@@ -73,7 +74,7 @@ const CatalogPageContents = () => {
CatalogFilterType
>();
const orgName = configApi.getOptionalString('organization.name') ?? 'Company';
const createComponentLink = useRouteRef(createComponentRouteRef);
const addMockData = useCallback(async () => {
try {
const promises: Promise<unknown>[] = [];
@@ -166,7 +167,7 @@ const CatalogPageContents = () => {
component={RouterLink}
variant="contained"
color="primary"
to={scaffolderRootRoute.path}
to={createComponentLink()}
>
Create Component
</Button>
plugins/catalog/src/plugin.ts
+4
View File
@@ -29,6 +29,7 @@ import {
entityRouteRef,
} from '@backstage/plugin-catalog-react';
import { CatalogClientWrapper } from './CatalogClientWrapper';
import { createComponentRouteRef } from './routes';
export const catalogPlugin = createPlugin({
id: 'catalog',
@@ -47,6 +48,9 @@ export const catalogPlugin = createPlugin({
catalogIndex: catalogRouteRef,
catalogEntity: entityRouteRef,
},
externalRoutes: {
createComponent: createComponentRouteRef,
},
});
export const CatalogIndexPage = catalogPlugin.provide(
plugins/scaffolder/src/components/JobStatusModal/index.ts → plugins/catalog/src/routes.ts
+7 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,4 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { JobStatusModal } from './JobStatusModal';
import { createExternalRouteRef } from '@backstage/core';
export const createComponentRouteRef = createExternalRouteRef({
id: 'create-component',
});
plugins/scaffolder-backend/package.json
+4 -2
View File
@@ -51,10 +51,12 @@
"fs-extra": "^9.0.0",
"git-url-parse": "^11.4.4",
"globby": "^11.0.0",
"handlebars": "^4.7.6",
"helmet": "^4.0.0",
"isomorphic-git": "^1.8.0",
"jsonschema": "^1.2.6",
"knex": "^0.21.6",
"luxon": "^1.26.0",
"morgan": "^1.10.0",
"uuid": "^8.2.0",
"winston": "^3.2.1",
@@ -67,9 +69,9 @@
"@types/mock-fs": "^4.13.0",
"@types/supertest": "^2.0.8",
"mock-fs": "^4.13.0",
"msw": "^0.21.2",
"supertest": "^4.0.2",
"yaml": "^1.10.0",
"msw": "^0.21.2"
"yaml": "^1.10.0"
},
"files": [
"dist",
plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts
+3 -13
View File
@@ -15,24 +15,14 @@
*/
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { CatalogClient } from '@backstage/catalog-client';
import {
ConflictError,
NotFoundError,
PluginEndpointDiscovery,
} from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
import { ConflictError, NotFoundError } from '@backstage/backend-common';
/**
* A catalog client tailored for reading out entity data from the catalog.
*/
export class CatalogEntityClient {
private readonly catalogClient: CatalogClient;
constructor(options: { discovery: PluginEndpointDiscovery }) {
this.catalogClient = new CatalogClient({
discoveryApi: options.discovery,
});
}
constructor(private readonly catalogClient: CatalogApi) {}
/**
* Looks up a single template using a template name.
plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts
+29 -6
View File
@@ -19,28 +19,37 @@ import { FilePreparer, PreparerBuilder } from './prepare';
import Docker from 'dockerode';
import { TemplaterBuilder, TemplaterValues } from './templater';
import { PublisherBuilder } from './publish';
import { CatalogApi } from '@backstage/catalog-client';
import { getEntityName } from '@backstage/catalog-model';
type Options = {
dockerClient: Docker;
preparers: PreparerBuilder;
templaters: TemplaterBuilder;
publishers: PublisherBuilder;
catalogClient: CatalogApi;
};
export function registerLegacyActions(
registry: TemplateActionRegistry,
options: Options,
) {
const { dockerClient, preparers, templaters, publishers } = options;
const {
dockerClient,
preparers,
templaters,
publishers,
catalogClient,
} = options;
registry.register({
id: 'legacy:prepare',
async handler(ctx) {
ctx.logger.info('Preparing the skeleton');
const { protocol, url } = ctx.parameters;
const preparer =
protocol === 'file' ? new FilePreparer() : preparers.get(url as string);
ctx.logger.info('Prepare the skeleton');
await preparer.prepare({
url: url as string,
logger: ctx.logger,
@@ -52,11 +61,8 @@ export function registerLegacyActions(
registry.register({
id: 'legacy:template',
async handler(ctx) {
const { logger } = ctx;
ctx.logger.info('Running the templater');
const templater = templaters.get(ctx.parameters.templater as string);
logger.info('Run the templater');
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
@@ -107,4 +113,21 @@ export function registerLegacyActions(
}
},
});
registry.register({
id: 'catalog:register',
async handler(ctx) {
const { catalogInfoUrl } = ctx.parameters;
ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`);
const result = await catalogClient.addLocation({
type: 'url',
target: catalogInfoUrl as string,
});
if (result.entities.length >= 1) {
const { kind, name, namespace } = getEntityName(result.entities[0]);
ctx.output('entityRef', `${kind}:${namespace}/${name}`);
}
},
});
}
plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts
+7 -3
View File
@@ -32,6 +32,7 @@ import {
TaskStoreEmitOptions,
TaskStoreGetEventsOptions,
} from './types';
import { DateTime } from 'luxon';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-scaffolder-backend',
@@ -64,7 +65,7 @@ export class DatabaseTaskStore implements TaskStore {
constructor(private readonly db: Knex) {}
async get(taskId: string): Promise<DbTaskRow> {
async getTask(taskId: string): Promise<DbTaskRow> {
const [result] = await this.db<RawDbTaskRow>('tasks')
.where({ id: taskId })
.select();
@@ -255,11 +256,14 @@ export class DatabaseTaskStore implements TaskStore {
try {
const body = JSON.parse(event.body) as JsonObject;
return {
id: event.id,
id: Number(event.id),
taskId,
body,
type: event.event_type,
createdAt: event.created_at,
createdAt:
typeof event.created_at === 'string'
? DateTime.fromSQL(event.created_at, { zone: 'UTC' }).toISO()
: event.created_at,
};
} catch (error) {
throw new Error(
plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts
+12 -12
View File
@@ -47,7 +47,7 @@ describe('StorageTaskBroker', () => {
const logger = getVoidLogger();
it('should claim a dispatched work item', async () => {
const broker = new StorageTaskBroker(storage, logger);
await broker.dispatch({ steps: [] });
await broker.dispatch({} as TaskSpec);
await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent));
});
@@ -57,7 +57,7 @@ describe('StorageTaskBroker', () => {
await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting');
await broker.dispatch({ steps: [] });
await broker.dispatch({} as TaskSpec);
await expect(promise).resolves.toEqual(expect.any(TaskAgent));
});
@@ -80,19 +80,19 @@ describe('StorageTaskBroker', () => {
it('should complete a task', async () => {
const broker = new StorageTaskBroker(storage, logger);
const dispatchResult = await broker.dispatch({ steps: [] });
const dispatchResult = await broker.dispatch({} as TaskSpec);
const task = await broker.claim();
await task.complete('completed');
const taskRow = await storage.get(dispatchResult.taskId);
const taskRow = await storage.getTask(dispatchResult.taskId);
expect(taskRow.status).toBe('completed');
}, 10000);
it('should fail a task', async () => {
const broker = new StorageTaskBroker(storage, logger);
const dispatchResult = await broker.dispatch({ steps: [] });
const dispatchResult = await broker.dispatch({} as TaskSpec);
const task = await broker.claim();
await task.complete('failed');
const taskRow = await storage.get(dispatchResult.taskId);
const taskRow = await storage.getTask(dispatchResult.taskId);
expect(taskRow.status).toBe('failed');
});
@@ -100,7 +100,7 @@ describe('StorageTaskBroker', () => {
const broker1 = new StorageTaskBroker(storage, logger);
const broker2 = new StorageTaskBroker(storage, logger);
const { taskId } = await broker1.dispatch({ steps: [] });
const { taskId } = await broker1.dispatch({} as TaskSpec);
const logPromise = new Promise<DbTaskEventRow[]>(resolve => {
const observedEvents = new Array<DbTaskEventRow>();
@@ -139,13 +139,13 @@ describe('StorageTaskBroker', () => {
it('should heartbeat', async () => {
const broker = new StorageTaskBroker(storage, logger);
const { taskId } = await broker.dispatch({ steps: [] });
const { taskId } = await broker.dispatch({} as TaskSpec);
const task = await broker.claim();
const initialTask = await storage.get(taskId);
const initialTask = await storage.getTask(taskId);
for (;;) {
const maybeTask = await storage.get(taskId);
const maybeTask = await storage.getTask(taskId);
if (maybeTask.lastHeartbeatAt !== initialTask.lastHeartbeatAt) {
break;
}
@@ -157,7 +157,7 @@ describe('StorageTaskBroker', () => {
it('should be update the status to failed if heartbeat fails', async () => {
const broker = new StorageTaskBroker(storage, logger);
const { taskId } = await broker.dispatch({ steps: [] });
const { taskId } = await broker.dispatch({} as TaskSpec);
const task = await broker.claim();
jest
@@ -169,7 +169,7 @@ describe('StorageTaskBroker', () => {
}, 500);
for (;;) {
const maybeTask = await storage.get(taskId);
const maybeTask = await storage.getTask(taskId);
if (maybeTask.status === 'failed') {
break;
}
plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts
+16 -4
View File
@@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonObject } from '@backstage/config';
import { Logger } from 'winston';
import {
CompletedTaskState,
@@ -22,6 +23,7 @@ import {
TaskBroker,
DispatchResult,
DbTaskEventRow,
DbTaskRow,
} from './types';
export class TaskAgent implements Task {
@@ -54,18 +56,24 @@ export class TaskAgent implements Task {
return this.isDone;
}
async emitLog(message: string): Promise<void> {
async emitLog(message: string, metadata?: JsonObject): Promise<void> {
await this.storage.emitLogEvent({
taskId: this.state.taskId,
body: { message },
body: { message, ...metadata },
});
}
async complete(result: CompletedTaskState): Promise<void> {
async complete(
result: CompletedTaskState,
metadata?: JsonObject,
): Promise<void> {
await this.storage.completeTask({
taskId: this.state.taskId,
status: result === 'failed' ? 'failed' : 'completed',
eventBody: { message: `Run completed with status: ${result}` },
eventBody: {
message: `Run completed with status: ${result}`,
...metadata,
},
});
this.isDone = true;
if (this.heartbeatTimeoutId) {
@@ -136,6 +144,10 @@ export class StorageTaskBroker implements TaskBroker {
};
}
async get(taskId: string): Promise<DbTaskRow> {
return this.storage.getTask(taskId);
}
observe(
options: {
taskId: string;
plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts
+97 -47
View File
@@ -22,6 +22,7 @@ import { TaskBroker, Task } from './types';
import fs from 'fs-extra';
import path from 'path';
import { TemplateActionRegistry } from './TemplateConverter';
import * as handlebars from 'handlebars';
type Options = {
logger: Logger;
@@ -44,67 +45,116 @@ export class TaskWorker {
async runOneTask(task: Task) {
try {
const { actionRegistry, logger } = this.options;
const { actionRegistry } = this.options;
const workspacePath = path.join(
this.options.workingDirectory,
await task.getWorkspaceName(),
);
await fs.ensureDir(workspacePath);
await task.emitLog(
`Starting up task with ${task.spec.steps.length} steps`,
);
const taskLogger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.colorize(),
winston.format.timestamp(),
winston.format.simple(),
),
defaultMeta: {},
});
const stream = new PassThrough();
stream.on('data', data => {
const message = data.toString().trim();
if (message?.length > 1) task.emitLog(message);
});
taskLogger.add(new winston.transports.Stream({ stream }));
// Give us some time to curl observe
task.emitLog('Task claimed, waiting ...');
await new Promise(resolve => setTimeout(resolve, 5000));
task.emitLog(`Starting up work with ${task.spec.steps.length} steps`);
const outputs: { [name: string]: JsonValue } = {};
const templateCtx: {
steps: {
[stepName: string]: { output: { [outputName: string]: JsonValue } };
};
} = { steps: {} };
for (const step of task.spec.steps) {
task.emitLog(`Beginning step ${step.name}`);
const metadata = { stepId: step.id };
try {
const taskLogger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.colorize(),
winston.format.timestamp(),
winston.format.simple(),
),
defaultMeta: {},
});
const action = actionRegistry.get(step.action);
if (!action) {
throw new Error(`Action '${step.action}' does not exist`);
const stream = new PassThrough();
stream.on('data', async data => {
const message = data.toString().trim();
if (message?.length > 1) {
await task.emitLog(message, metadata);
}
});
taskLogger.add(new winston.transports.Stream({ stream }));
await task.emitLog(`Beginning step ${step.name}`, {
...metadata,
status: 'processing',
});
const action = actionRegistry.get(step.action);
if (!action) {
throw new Error(`Action '${step.action}' does not exist`);
}
const parameters: { [name: string]: JsonValue } = {};
for (const [name, maybeTemplateStr] of Object.entries(
step.parameters ?? {},
)) {
if (typeof maybeTemplateStr === 'string') {
const value = handlebars.compile(maybeTemplateStr, {
noEscape: true,
strict: true,
data: false,
preventIndent: true,
})(templateCtx);
parameters[name] = value;
} else {
parameters[name] = maybeTemplateStr;
}
}
const stepOutputs: { [name: string]: JsonValue } = {};
await action.handler({
logger: taskLogger,
logStream: stream,
parameters,
workspacePath,
output(name: string, value: JsonValue) {
stepOutputs[name] = value;
},
});
templateCtx.steps[step.id] = { output: stepOutputs };
await task.emitLog(`Finished step ${step.name}`, {
...metadata,
status: 'completed',
});
} catch (error) {
await task.emitLog(String(error.stack), {
...metadata,
status: 'failed',
});
throw error;
}
// TODO: substitute any placeholders with output from previous steps
const parameters = step.parameters!;
await action.handler({
logger,
logStream: stream,
parameters,
workspacePath,
output(name: string, value: JsonValue) {
outputs[name] = value;
},
});
task.emitLog(`Finished step ${step.name}`);
}
await task.complete('completed');
const output = Object.fromEntries(
Object.entries(task.spec.output).map(([name, templateStr]) => {
const value = handlebars.compile(templateStr, {
noEscape: true,
strict: true,
data: false,
preventIndent: true,
})(templateCtx);
return [name, value];
}),
);
await task.complete('completed', { output });
} catch (error) {
task.emitLog(String(error.stack));
await task.complete('failed');
await task.complete('failed', {
error: { name: error.name, message: error.message },
});
}
}
}
plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts
+19 -3
View File
@@ -18,7 +18,7 @@ import { resolve as resolvePath } from 'path';
import { JsonValue } from '@backstage/config';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import { Logger } from 'winston';
import type { Writable } from 'stream';
import { Writable } from 'stream';
import { TaskSpec } from './types';
import { ConflictError, NotFoundError } from '@backstage/backend-common';
@@ -69,14 +69,30 @@ export function templateEntityToSpec(
steps.push({
id: 'publish',
name: 'Publishing',
name: 'Publish',
action: 'legacy:publish',
parameters: {
values,
},
});
return { steps };
steps.push({
id: 'register',
name: 'Register',
action: 'catalog:register',
parameters: {
catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}',
},
});
return {
steps,
output: {
remoteUrl: '{{ steps.publish.output.remoteUrl }}',
catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}',
entityRef: '{{ steps.register.output.entityRef }}',
},
};
}
type ActionContext = {
plugins/scaffolder-backend/src/scaffolder/tasks/types.ts
+4 -2
View File
@@ -49,6 +49,7 @@ export type TaskSpec = {
action: string;
parameters?: { [name: string]: JsonValue };
}>;
output: { [name: string]: string };
};
export type DispatchResult = {
@@ -58,8 +59,8 @@ export type DispatchResult = {
export interface Task {
spec: TaskSpec;
done: boolean;
emitLog(message: string): Promise<void>;
complete(result: CompletedTaskState): Promise<void>;
emitLog(message: string, metadata?: JsonValue): Promise<void>;
complete(result: CompletedTaskState, metadata?: JsonValue): Promise<void>;
getWorkspaceName(): Promise<string>;
}
@@ -90,6 +91,7 @@ export type TaskStoreGetEventsOptions = {
};
export interface TaskStore {
createTask(task: TaskSpec): Promise<{ taskId: string }>;
getTask(taskId: string): Promise<DbTaskRow>;
claimTask(): Promise<DbTaskRow | undefined>;
completeTask(options: {
taskId: string;
plugins/scaffolder-backend/src/service/router.test.ts
+44 -8
View File
@@ -39,12 +39,14 @@ import request from 'supertest';
import { createRouter } from './router';
import { Templaters, Preparers, Publishers } from '../scaffolder';
import Docker from 'dockerode';
import { CatalogApi } from '@backstage/catalog-client';
jest.mock('dockerode');
const generateEntityClient: any = (template: any) => ({
findTemplate: () => Promise.resolve(template),
});
const createCatalogClient = (templates: any[] = []) =>
({
getEntities: async () => ({ items: templates }),
} as CatalogApi);
function createDatabase(): PluginDatabaseManager {
return SingleConnectionDatabaseManager.fromConfig(
@@ -95,7 +97,6 @@ describe('createRouter - working directory', () => {
},
};
const mockedEntityClient = generateEntityClient(template);
it('should throw an error when working directory does not exist or is not writable', async () => {
mockAccess.mockImplementation(() => {
throw new Error('access error');
@@ -109,8 +110,8 @@ describe('createRouter - working directory', () => {
publishers: new Publishers(),
config: new ConfigReader(workDirConfig('/path')),
dockerClient: new Docker(),
entityClient: mockedEntityClient,
database: createDatabase(),
catalogClient: createCatalogClient([template]),
}),
).rejects.toThrow('access error');
});
@@ -123,8 +124,8 @@ describe('createRouter - working directory', () => {
publishers: new Publishers(),
config: new ConfigReader(workDirConfig('/path')),
dockerClient: new Docker(),
entityClient: mockedEntityClient,
database: createDatabase(),
catalogClient: createCatalogClient([template]),
});
const app = express().use(router);
@@ -152,8 +153,8 @@ describe('createRouter - working directory', () => {
publishers: new Publishers(),
config: new ConfigReader({}),
dockerClient: new Docker(),
entityClient: mockedEntityClient,
database: createDatabase(),
catalogClient: createCatalogClient([template]),
});
const app = express().use(router);
@@ -184,6 +185,9 @@ describe('createRouter', () => {
name: 'create-react-app-template',
tags: ['experimental', 'react', 'cra'],
title: 'Create React App Template',
annotations: {
'backstage.io/managed-by-location': 'url:https://dev.azure.com',
},
},
spec: {
owner: 'web@example.com',
@@ -222,8 +226,8 @@ describe('createRouter', () => {
publishers: new Publishers(),
config: new ConfigReader({}),
dockerClient: new Docker(),
entityClient: generateEntityClient(template),
database: createDatabase(),
catalogClient: createCatalogClient([template]),
});
app = express().use(router);
});
@@ -246,4 +250,36 @@ describe('createRouter', () => {
expect(response.status).toEqual(400);
});
});
describe('POST /v2/tasks', () => {
it('rejects template values which do not match the template schema definition', async () => {
const response = await request(app)
.post('/v2/tasks')
.send({
templateName: '',
values: {
storePath: 'https://github.com/backstage/backstage',
},
});
expect(response.status).toEqual(400);
});
it('return the template id', async () => {
const response = await request(app)
.post('/v2/tasks')
.send({
templateName: 'create-react-app-template',
values: {
storePath: 'https://github.com/backstage/backstage',
component_id: '123',
name: 'test',
use_typescript: false,
},
});
expect(response.body.id).toBeDefined();
expect(response.status).toEqual(201);
});
});
});
plugins/scaffolder-backend/src/service/router.ts
+17 -3
View File
@@ -44,7 +44,11 @@ import {
} from '../scaffolder/tasks/TemplateConverter';
import { registerLegacyActions } from '../scaffolder/stages/legacy';
import { getWorkingDirectory } from './helpers';
import { PluginDatabaseManager } from '@backstage/backend-common';
import {
NotFoundError,
PluginDatabaseManager,
} from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
export interface RouterOptions {
preparers: PreparerBuilder;
@@ -54,8 +58,8 @@ export interface RouterOptions {
logger: Logger;
config: Config;
dockerClient: Docker;
entityClient: CatalogEntityClient;
database: PluginDatabaseManager;
catalogClient: CatalogApi;
}
export async function createRouter(
@@ -71,13 +75,14 @@ export async function createRouter(
logger: parentLogger,
config,
dockerClient,
entityClient,
database,
catalogClient,
} = options;
const logger = parentLogger.child({ plugin: 'scaffolder' });
const workingDirectory = await getWorkingDirectory(config, logger);
const jobProcessor = await JobProcessor.fromConfig({ config, logger });
const entityClient = new CatalogEntityClient(catalogClient);
const databaseTaskStore = await DatabaseTaskStore.create(
await database.getClient(),
@@ -96,6 +101,7 @@ export async function createRouter(
preparers,
publishers,
templaters,
catalogClient,
});
worker.start();
@@ -249,6 +255,14 @@ export async function createRouter(
res.status(201).json({ id: result.taskId });
})
.get('/v2/tasks/:taskId', async (req, res) => {
const { taskId } = req.params;
const task = await taskBroker.get(taskId);
if (!task) {
throw new NotFoundError(`Task with id ${taskId} does not exist`);
}
res.status(200).json(task);
})
.get('/v2/tasks/:taskId/eventstream', async (req, res) => {
const { taskId } = req.params;
const after = Number(req.query.after) || undefined;
plugins/scaffolder/dev/index.tsx
+4 -8
View File
@@ -19,8 +19,8 @@ import { createDevApp } from '@backstage/dev-utils';
import { discoveryApiRef, identityApiRef } from '@backstage/core';
import { CatalogClient } from '@backstage/catalog-client';
import { catalogApiRef } from '@backstage/plugin-catalog-react';
import { TemplateIndexPage, TemplatePage } from '../src/plugin';
import { ScaffolderApi, scaffolderApiRef } from '../src';
import { ScaffolderPage } from '../src/plugin';
import { ScaffolderClient, scaffolderApiRef } from '../src';
createDevApp()
.registerApi({
@@ -32,15 +32,11 @@ createDevApp()
api: scaffolderApiRef,
deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef },
factory: ({ discoveryApi, identityApi }) =>
new ScaffolderApi({ discoveryApi, identityApi }),
new ScaffolderClient({ discoveryApi, identityApi }),
})
.addPage({
path: '/create',
title: 'Create',
element: <TemplateIndexPage />,
})
.addPage({
path: '/create/:templateName',
element: <TemplatePage />,
element: <ScaffolderPage />,
})
.render();
plugins/scaffolder/package.json
+8 -2
View File
@@ -30,7 +30,9 @@
"clean": "backstage-cli clean"
},
"dependencies": {
"@backstage/catalog-client": "^0.3.6",
"@backstage/catalog-model": "^0.7.1",
"@backstage/config": "^0.1.2",
"@backstage/core": "^0.6.2",
"@backstage/plugin-catalog-react": "^0.0.4",
"@backstage/theme": "^0.2.3",
@@ -41,6 +43,8 @@
"@rjsf/material-ui": "^2.4.0",
"classnames": "^2.2.6",
"git-url-parse": "^11.4.4",
"humanize-duration": "^3.25.1",
"luxon": "^1.25.0",
"moment": "^2.26.0",
"react": "^16.13.1",
"react-dom": "^16.13.1",
@@ -48,16 +52,18 @@
"react-router": "6.0.0-beta.0",
"react-router-dom": "6.0.0-beta.0",
"react-use": "^15.3.3",
"swr": "^0.3.0"
"swr": "^0.3.0",
"use-immer": "^0.4.2",
"zen-observable": "^0.8.15"
},
"devDependencies": {
"@backstage/cli": "^0.6.1",
"@backstage/dev-utils": "^0.1.11",
"@backstage/test-utils": "^0.1.7",
"@backstage/catalog-client": "^0.3.6",
"@testing-library/jest-dom": "^5.10.1",
"@testing-library/react": "^10.4.1",
"@testing-library/user-event": "^12.0.7",
"@types/humanize-duration": "^3.18.1",
"@testing-library/react-hooks": "^3.3.0",
"@types/jest": "^26.0.7",
"@types/node": "^12.0.0",
plugins/scaffolder/src/api.ts
+98 -7
View File
@@ -14,14 +14,53 @@
* limitations under the License.
*/
import { createApiRef, DiscoveryApi, IdentityApi } from '@backstage/core';
import {
createApiRef,
DiscoveryApi,
Observable,
IdentityApi,
} from '@backstage/core';
import ObservableImpl from 'zen-observable';
import { ScaffolderTask, Status } from './types';
export const scaffolderApiRef = createApiRef<ScaffolderApi>({
id: 'plugin.scaffolder.service',
description: 'Used to make requests towards the scaffolder backend',
});
export class ScaffolderApi {
export type LogEvent = {
type: 'log' | 'completion';
body: {
message: string;
stepId?: string;
status?: Status;
};
createdAt: string;
id: string;
taskId: string;
};
export interface ScaffolderApi {
/**
* Executes the scaffolding of a component, given a template and its
* parameter values.
*
* @param templateName Name of the Template entity for the scaffolder to use. New project is going to be created out of this template.
* @param values Parameters for the template, e.g. name, description
*/
scaffold(templateName: string, values: Record<string, any>): Promise<string>;
getTask(taskId: string): Promise<ScaffolderTask>;
streamLogs({
taskId,
after,
}: {
taskId: string;
after?: number;
}): Observable<LogEvent>;
}
export class ScaffolderClient implements ScaffolderApi {
private readonly discoveryApi: DiscoveryApi;
private readonly identityApi: IdentityApi;
@@ -40,9 +79,12 @@ export class ScaffolderApi {
* @param templateName Template name for the scaffolder to use. New project is going to be created out of this template.
* @param values Parameters for the template, e.g. name, description
*/
async scaffold(templateName: string, values: Record<string, any>) {
async scaffold(
templateName: string,
values: Record<string, any>,
): Promise<string> {
const token = await this.identityApi.getIdToken();
const url = `${await this.discoveryApi.getBaseUrl('scaffolder')}/v1/jobs`;
const url = `${await this.discoveryApi.getBaseUrl('scaffolder')}/v2/tasks`;
const response = await fetch(url, {
method: 'POST',
headers: {
@@ -58,16 +100,65 @@ export class ScaffolderApi {
throw new Error(`Backend request failed, ${status} ${body.trim()}`);
}
const { id } = await response.json();
const { id } = (await response.json()) as { id: string };
return id;
}
async getJob(jobId: string) {
async getTask(taskId: string) {
const token = await this.identityApi.getIdToken();
const baseUrl = await this.discoveryApi.getBaseUrl('scaffolder');
const url = `${baseUrl}/v1/job/${encodeURIComponent(jobId)}`;
const url = `${baseUrl}/v2/tasks/${encodeURIComponent(taskId)}`;
return fetch(url, {
headers: token ? { Authorization: `Bearer ${token}` } : {},
}).then(x => x.json());
}
streamLogs({
taskId,
after,
}: {
taskId: string;
after?: number;
}): Observable<LogEvent> {
return new ObservableImpl(subscriber => {
const params = new URLSearchParams();
if (after !== undefined) {
params.set('after', String(Number(after)));
}
this.discoveryApi.getBaseUrl('scaffolder').then(
baseUrl => {
const url = `${baseUrl}/v2/tasks/${encodeURIComponent(
taskId,
)}/eventstream`;
const eventSource = new EventSource(url);
eventSource.addEventListener('log', (event: any) => {
if (event.data) {
try {
subscriber.next(JSON.parse(event.data));
} catch (ex) {
subscriber.error(ex);
}
}
});
eventSource.addEventListener('completion', (event: any) => {
if (event.data) {
try {
subscriber.next(JSON.parse(event.data));
} catch (ex) {
subscriber.error(ex);
}
}
subscriber.complete();
});
eventSource.addEventListener('error', event => {
subscriber.error(event);
});
},
error => {
subscriber.error(error);
},
);
});
}
}
plugins/scaffolder/src/components/JobStage/JobStage.tsx
-169
View File
@@ -1,169 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Accordion,
AccordionDetails,
AccordionSummary,
AccordionActions,
Box,
CircularProgress,
LinearProgress,
Typography,
Button,
} from '@material-ui/core';
import { makeStyles } from '@material-ui/core/styles';
import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
import ExpandLessIcon from '@material-ui/icons/ExpandLess';
import cn from 'classnames';
import moment from 'moment';
import React, { Suspense, useEffect, useState } from 'react';
import { LogModal } from './LogModal';
import { Job } from '../../types';
const LazyLog = React.lazy(() => import('react-lazylog/build/LazyLog'));
moment.relativeTimeThreshold('ss', 0);
const useStyles = makeStyles(theme => ({
accordionDetails: {
padding: 0,
},
button: {
order: -1,
margin: '0 1em 0 -20px',
},
cardContent: {
backgroundColor: theme.palette.background.default,
},
accordion: {
position: 'relative',
'&:after': {
pointerEvents: 'none',
content: '""',
position: 'absolute',
top: 0,
right: 0,
left: 0,
bottom: 0,
},
},
neutral: {},
failed: {
'&:after': {
boxShadow: `inset 4px 0px 0px ${theme.palette.error.main}`,
},
},
started: {
'&:after': {
boxShadow: `inset 4px 0px 0px ${theme.palette.info.main}`,
},
},
completed: {
'&:after': {
boxShadow: `inset 4px 0px 0px ${theme.palette.success.main}`,
},
},
jobStatusTitle: {
display: 'flex',
width: '100%',
alignItems: 'center',
flexDirection: 'row',
justifyContent: 'space-between',
[theme.breakpoints.down('xs')]: {
flexDirection: 'column',
alignItems: 'flex-start',
justifyContent: 'flex-start',
},
},
}));
type Props = {
name: string;
className?: string;
log: string[];
startedAt: string;
endedAt?: string;
status: Job['status'];
};
export const JobStage = ({ endedAt, startedAt, name, log, status }: Props) => {
const classes = useStyles();
const [expanded, setExpanded] = useState(false);
useEffect(() => {
if (status === 'FAILED') setExpanded(true);
}, [status, setExpanded]);
const timeElapsed =
status !== 'PENDING'
? moment
.duration(moment(endedAt ?? moment()).diff(moment(startedAt)))
.humanize()
: null;
const [logsFullScreen, setLogsFullScreen] = useState(false);
const toggleLogsFullScreen = () => setLogsFullScreen(!logsFullScreen);
return (
<Accordion
TransitionProps={{ unmountOnExit: true }}
className={cn(
classes.accordion,
classes[status.toLowerCase() as keyof ReturnType<typeof useStyles>] ??
classes.neutral,
)}
expanded={expanded}
onChange={(_, newState) => setExpanded(newState)}
>
<AccordionSummary
expandIcon={expanded ? <ExpandLessIcon /> : <ExpandMoreIcon />}
aria-controls={`panel-${name}-content`}
id={`panel-${name}-header`}
IconButtonProps={{
className: classes.button,
}}
>
<Typography variant="button" className={classes.jobStatusTitle}>
{name} {timeElapsed && `(${timeElapsed})`}{' '}
{startedAt && !endedAt && <CircularProgress size="1em" />}
</Typography>
</AccordionSummary>
<AccordionDetails className={classes.accordionDetails}>
{log.length === 0 ? (
<Box px={9} pb={2} width="100%">
No logs available for this step
</Box>
) : (
<Suspense fallback={<LinearProgress />}>
<LogModal
open={logsFullScreen}
onClose={toggleLogsFullScreen}
log={log}
/>
<div style={{ height: '20vh', width: '100%' }}>
<LazyLog text={`${log.join('\n')}`} extraLines={1} follow />
</div>
</Suspense>
)}
</AccordionDetails>
<AccordionActions>
<Button color="primary" onClick={toggleLogsFullScreen}>
Open in fullscreen
</Button>
</AccordionActions>
</Accordion>
);
};
plugins/scaffolder/src/components/JobStage/LogModal.tsx
-67
View File
@@ -1,67 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import React from 'react';
import {
Dialog,
DialogTitle,
DialogContent,
IconButton,
} from '@material-ui/core';
import { makeStyles } from '@material-ui/core/styles';
import Close from '@material-ui/icons/Close';
import LazyLog from 'react-lazylog/build/LazyLog';
type Props = {
log: string[];
open?: boolean;
onClose(): void;
};
const useStyles = makeStyles(theme => ({
header: {
width: '100%',
padding: theme.spacing(1, 4),
},
closeIcon: {
float: 'right',
padding: theme.spacing(0.5, 0),
},
logs: {
boxShadow: '-3px -1px 7px 0px rgba(50, 50, 50, 0.59)',
height: '100%',
width: '100%',
},
}));
export const LogModal = ({ log, open = false, onClose }: Props) => {
const classes = useStyles();
return (
<Dialog open={open} onClose={onClose} fullScreen>
<DialogTitle id="responsive-dialog-title" className={classes.header}>
Logs
<IconButton onClick={onClose} className={classes.closeIcon}>
<Close />
</IconButton>
</DialogTitle>
<DialogContent>
<div className={classes.logs}>
<LazyLog text={log.join('\n')} extraLines={1} follow />
</div>
</DialogContent>
</Dialog>
);
};
plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx
-95
View File
@@ -1,95 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button } from '@backstage/core';
import {
Button as Action,
Dialog,
DialogActions,
DialogContent,
DialogTitle,
LinearProgress,
} from '@material-ui/core';
import React, { useCallback } from 'react';
import { Job } from '../../types';
import { JobStage } from '../JobStage/JobStage';
type Props = {
job: Job | null;
toCatalogLink?: string;
open: boolean;
onModalClose: () => void;
};
export const JobStatusModal = ({
job,
toCatalogLink,
open,
onModalClose,
}: Props) => {
const renderTitle = () => {
switch (job?.status) {
case 'COMPLETED':
return 'Successfully created component';
case 'FAILED':
return 'Failed to create component';
default:
return 'Create component';
}
};
const onClose = useCallback(() => {
if (!job) {
return;
}
// Disallow closing modal if the job is in progress.
if (job.status === 'COMPLETED' || job.status === 'FAILED') {
onModalClose();
}
}, [job, onModalClose]);
return (
<Dialog open={open} onClose={onClose} fullWidth>
<DialogTitle id="responsive-dialog-title">{renderTitle()}</DialogTitle>
<DialogContent>
{!job ? (
<LinearProgress />
) : (
(job?.stages ?? []).map(step => (
<JobStage
log={step.log}
name={step.name}
key={step.name}
startedAt={step.startedAt}
endedAt={step.endedAt}
status={step.status}
/>
))
)}
</DialogContent>
{job?.status && toCatalogLink && (
<DialogActions>
<Button to={toCatalogLink}>View in catalog</Button>
</DialogActions>
)}
{job?.status === 'FAILED' && (
<DialogActions>
<Action onClick={onClose}>Close</Action>
</DialogActions>
)}
</Dialog>
);
};
plugins/scaffolder/src/components/Router.tsx
+29
View File
@@ -0,0 +1,29 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import React from 'react';
import { Routes, Route } from 'react-router';
import { ScaffolderPage } from './ScaffolderPage';
import { TemplatePage } from './TemplatePage';
import { TaskPage } from './TaskPage';
export const Router = () => (
<Routes>
<Route path="/" element={<ScaffolderPage />} />
<Route path="/templates/:templateName" element={<TemplatePage />} />
<Route path="/tasks/:taskId" element={<TaskPage />} />
</Routes>
);
plugins/scaffolder/src/components/TaskPage/TaskPage.tsx
+338
View File
@@ -0,0 +1,338 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Page, Header, Lifecycle, Content, ErrorPage } from '@backstage/core';
import React, { useState, useEffect, memo, useMemo } from 'react';
import { makeStyles, Theme, createStyles } from '@material-ui/core/styles';
import Stepper from '@material-ui/core/Stepper';
import Step from '@material-ui/core/Step';
import StepLabel from '@material-ui/core/StepLabel';
import Grid from '@material-ui/core/Grid';
import Typography from '@material-ui/core/Typography';
import { generatePath, useParams } from 'react-router';
import { useTaskEventStream } from '../hooks/useEventStream';
import LazyLog from 'react-lazylog/build/LazyLog';
import { Link } from 'react-router-dom';
import {
Box,
Button,
CircularProgress,
Paper,
StepButton,
StepIconProps,
} from '@material-ui/core';
import { Status } from '../../types';
import { DateTime, Interval } from 'luxon';
import { useInterval } from 'react-use';
import Check from '@material-ui/icons/Check';
import Cancel from '@material-ui/icons/Cancel';
import FiberManualRecordIcon from '@material-ui/icons/FiberManualRecord';
import { entityRoute } from '@backstage/plugin-catalog-react';
import { parseEntityName } from '@backstage/catalog-model';
import classNames from 'classnames';
import { BackstageTheme } from '@backstage/theme';
// typings are wrong for this library, so fallback to not parsing types.
const humanizeDuration = require('humanize-duration');
const useStyles = makeStyles((theme: Theme) =>
createStyles({
root: {
width: '100%',
},
button: {
marginTop: theme.spacing(1),
marginRight: theme.spacing(1),
},
actionsContainer: {
marginBottom: theme.spacing(2),
},
resetContainer: {
padding: theme.spacing(3),
},
labelWrapper: {
display: 'flex',
flex: 1,
flexDirection: 'row',
justifyContent: 'space-between',
},
stepWrapper: {
width: '100%',
},
}),
);
type TaskStep = {
id: string;
name: string;
status: Status;
startedAt?: string;
endedAt?: string;
};
const StepTimeTicker = ({ step }: { step: TaskStep }) => {
const [time, setTime] = useState('');
useInterval(() => {
if (!step.startedAt) {
setTime('');
return;
}
const end = step.endedAt
? DateTime.fromISO(step.endedAt)
: DateTime.local();
const startedAt = DateTime.fromISO(step.startedAt);
const formatted = Interval.fromDateTimes(startedAt, end)
.toDuration()
.valueOf();
setTime(humanizeDuration(formatted, { round: true }));
}, 1000);
return <Typography variant="caption">{time}</Typography>;
};
const useStepIconStyles = makeStyles((theme: BackstageTheme) =>
createStyles({
root: {
color: theme.palette.text.disabled,
display: 'flex',
height: 22,
alignItems: 'center',
},
completed: {
color: theme.palette.status.ok,
},
error: {
color: theme.palette.status.error,
},
}),
);
function TaskStepIconComponent(props: StepIconProps) {
const classes = useStepIconStyles();
const { active, completed, error } = props;
const getMiddle = () => {
if (active) {
return <CircularProgress size="24px" />;
}
if (completed) {
return <Check />;
}
if (error) {
return <Cancel />;
}
return <FiberManualRecordIcon />;
};
return (
<div
className={classNames(classes.root, {
[classes.completed]: completed,
[classes.error]: error,
})}
>
{getMiddle()}
</div>
);
}
export const TaskStatusStepper = memo(
({
steps,
currentStepId,
onUserStepChange,
}: {
steps: TaskStep[];
currentStepId: string | undefined;
onUserStepChange: (id: string) => void;
}) => {
const classes = useStyles();
return (
<div className={classes.root}>
<Stepper
activeStep={steps.findIndex(s => s.id === currentStepId)}
orientation="vertical"
nonLinear
>
{steps.map((step, index) => {
const isCompleted = step.status === 'completed';
const isFailed = step.status === 'failed';
const isActive = step.status === 'processing';
return (
<Step key={String(index)} expanded>
<StepButton onClick={() => onUserStepChange(step.id)}>
<StepLabel
StepIconProps={{
completed: isCompleted,
error: isFailed,
active: isActive,
}}
StepIconComponent={TaskStepIconComponent}
className={classes.stepWrapper}
>
<div className={classes.labelWrapper}>
<Typography variant="subtitle2">{step.name}</Typography>
<StepTimeTicker step={step} />
</div>
</StepLabel>
</StepButton>
</Step>
);
})}
</Stepper>
</div>
);
},
);
const TaskLogger = memo(({ log }: { log: string }) => {
return (
<div style={{ height: '80vh' }}>
<LazyLog text={log} extraLines={1} follow selectableLines enableSearch />
</div>
);
});
export const TaskPage = () => {
const [userSelectedStepId, setUserSelectedStepId] = useState<
string | undefined
>(undefined);
const [lastActiveStepId, setLastActiveStepId] = useState<string | undefined>(
undefined,
);
const { taskId } = useParams();
const taskStream = useTaskEventStream(taskId);
const completed = taskStream.completed;
const steps = useMemo(
() =>
taskStream.task?.spec.steps.map(step => ({
...step,
...taskStream?.steps?.[step.id],
})) ?? [],
[taskStream],
);
useEffect(() => {
const mostRecentFailedOrActiveStep = steps.find(step =>
['failed', 'processing'].includes(step.status),
);
if (completed && !mostRecentFailedOrActiveStep) {
setLastActiveStepId(steps[steps.length - 1]?.id);
return;
}
setLastActiveStepId(mostRecentFailedOrActiveStep?.id);
}, [steps, completed]);
const currentStepId = userSelectedStepId ?? lastActiveStepId;
const logAsString = useMemo(() => {
if (!currentStepId) {
return 'Loading...';
}
const log = taskStream.stepLogs[currentStepId];
if (!log?.length) {
return 'Waiting for logs...';
}
return log.join('\n');
}, [taskStream.stepLogs, currentStepId]);
const taskNotFound =
taskStream.completed === true &&
taskStream.loading === false &&
!taskStream.task;
const entityRef = taskStream.output?.entityRef;
const remoteUrl = taskStream.output?.remoteUrl;
return (
<Page themeId="home">
<Header
pageTitleOverride={`Task ${taskId}`}
title={
<>
Task Activity <Lifecycle alpha shorthand />
</>
}
subtitle={`Activity for task: ${taskId}`}
/>
<Content>
{taskNotFound ? (
<ErrorPage
status="404"
statusMessage="Task not found"
additionalInfo="No task found with this ID"
/>
) : (
<div>
<Grid container>
<Grid item xs={3}>
<Paper>
<TaskStatusStepper
steps={steps}
currentStepId={currentStepId}
onUserStepChange={setUserSelectedStepId}
/>
{(entityRef || remoteUrl) && (
<Box
px={3}
pb={3}
display="flex"
flex={1}
justifyContent="space-between"
flexDirection="row"
>
{entityRef && (
<Button
size="small"
variant="outlined"
component={Link}
to={generatePath(
`/catalog/${entityRoute.path}`,
parseEntityName(entityRef),
)}
>
Open in catalog
</Button>
)}
{remoteUrl && (
<Button
size="small"
variant="outlined"
href={remoteUrl}
>
Repo
</Button>
)}
</Box>
)}
</Paper>
</Grid>
<Grid item xs={9}>
<TaskLogger log={logAsString} />
</Grid>
</Grid>
</div>
)}
</Content>
</Page>
);
};
plugins/scaffolder/src/components/JobStage/index.ts → plugins/scaffolder/src/components/TaskPage/index.ts
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Spotify AB
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,4 +13,4 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { JobStage } from './JobStage';
export { TaskPage } from './TaskPage';
plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx
+7 -4
View File
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Button } from '@backstage/core';
import { Button, useRouteRef } from '@backstage/core';
import { BackstageTheme, pageTheme } from '@backstage/theme';
import {
Card,
@@ -25,8 +25,8 @@ import {
useTheme,
} from '@material-ui/core';
import React from 'react';
import { generatePath } from 'react-router-dom';
import { templateRoute } from '../../routes';
import { generatePath } from 'react-router';
import { rootRouteRef } from '../../routes';
const useStyles = makeStyles(theme => ({
header: {
@@ -59,11 +59,14 @@ export const TemplateCard = ({
name,
}: TemplateCardProps) => {
const backstageTheme = useTheme<BackstageTheme>();
const rootLink = useRouteRef(rootRouteRef);
const themeId = pageTheme[type] ? type : 'other';
const theme = backstageTheme.getPageTheme({ themeId });
const classes = useStyles({ backgroundImage: theme.backgroundImage });
const href = generatePath(templateRoute.path, { templateName: name });
const href = generatePath(`${rootLink()}/templates/:templateName`, {
templateName: name,
});
return (
<Card>
plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx
+15 -6
View File
@@ -22,7 +22,7 @@ import React from 'react';
import { act } from 'react-dom/test-utils';
import { MemoryRouter, Route } from 'react-router';
import { ScaffolderApi, scaffolderApiRef } from '../../api';
import { rootRoute } from '../../routes';
import { rootRouteRef } from '../../routes';
import { TemplatePage } from './TemplatePage';
const templateMock = {
@@ -97,11 +97,15 @@ describe('TemplatePage', () => {
<ApiProvider apis={apis}>
<TemplatePage />
</ApiProvider>,
{
mountedRoutes: {
'/create': rootRouteRef,
},
},
);
expect(rendered.queryByText('Create a New Component')).toBeInTheDocument();
expect(rendered.queryByText('React SSR Template')).toBeInTheDocument();
// await act(async () => await mutate('templates/test'));
});
it('renders spinner while loading', async () => {
@@ -114,13 +118,18 @@ describe('TemplatePage', () => {
<ApiProvider apis={apis}>
<TemplatePage />
</ApiProvider>,
{
mountedRoutes: {
'/create': rootRouteRef,
},
},
);
expect(rendered.queryByText('Create a New Component')).toBeInTheDocument();
expect(rendered.queryByTestId('loading-progress')).toBeInTheDocument();
// Need to cleanup the promise or will timeout
act(() => {
resolve!({ items: [] });
await act(async () => {
resolve!({ items: [templateMock] });
});
});
@@ -134,7 +143,7 @@ describe('TemplatePage', () => {
<Route path="/create/test">
<TemplatePage />
</Route>
<Route path={rootRoute.path} element={<>This is root</>} />
<Route path="/create" element={<>This is root</>} />
</MemoryRouter>
</ThemeProvider>
</ApiProvider>,
plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx
+13 -53
View File
@@ -22,23 +22,18 @@ import {
Lifecycle,
Page,
useApi,
useRouteRef,
} from '@backstage/core';
import {
catalogApiRef,
entityRoute,
entityRouteParams,
} from '@backstage/plugin-catalog-react';
import { catalogApiRef } from '@backstage/plugin-catalog-react';
import { LinearProgress } from '@material-ui/core';
import { IChangeEvent } from '@rjsf/core';
import parseGitUrl from 'git-url-parse';
import React, { useCallback, useState } from 'react';
import { generatePath, Navigate } from 'react-router';
import { generatePath, useNavigate, Navigate } from 'react-router';
import { useParams } from 'react-router-dom';
import { useAsync } from 'react-use';
import { scaffolderApiRef } from '../../api';
import { rootRoute } from '../../routes';
import { useJobPolling } from '../hooks/useJobPolling';
import { JobStatusModal } from '../JobStatusModal';
import { rootRouteRef } from '../../routes';
import { MultistepJsonForm } from '../MultistepJsonForm';
const useTemplate = (
@@ -50,7 +45,7 @@ const useTemplate = (
filter: { kind: 'Template', 'metadata.name': templateName },
});
return response.items as TemplateEntityV1alpha1[];
});
}, [catalogApi, templateName]);
return { template: value?.[0], loading, error };
};
@@ -76,57 +71,28 @@ const OWNER_REPO_SCHEMA = {
},
},
};
export const TemplatePage = () => {
const errorApi = useApi(errorApiRef);
const catalogApi = useApi(catalogApiRef);
const scaffolderApi = useApi(scaffolderApiRef);
const { templateName } = useParams();
const [catalogLink, setCatalogLink] = useState<string | undefined>();
const navigate = useNavigate();
const rootLink = useRouteRef(rootRouteRef);
const { template, loading } = useTemplate(templateName, catalogApi);
const [formState, setFormState] = useState({});
const [modalOpen, setModalOpen] = useState(false);
const handleFormReset = () => setFormState({});
const handleChange = useCallback(
(e: IChangeEvent) => setFormState({ ...formState, ...e.formData }),
[setFormState, formState],
);
const [jobId, setJobId] = useState<string | null>(null);
const job = useJobPolling(jobId, async jobItem => {
if (!jobItem.metadata.catalogInfoUrl) {
errorApi.post(
new Error(`No catalogInfoUrl returned from the scaffolder`),
);
return;
}
try {
const {
entities: [createdEntity],
} = await catalogApi.addLocation({
target: jobItem.metadata.catalogInfoUrl,
});
const resolvedPath = generatePath(
`/catalog/${entityRoute.path}`,
entityRouteParams(createdEntity),
);
setCatalogLink(resolvedPath);
} catch (ex) {
errorApi.post(
new Error(
`Something went wrong trying to add the new 'catalog-info.yaml' to the catalog`,
),
);
}
});
const handleCreate = async () => {
try {
const id = await scaffolderApi.scaffold(templateName, formState);
setJobId(id);
setModalOpen(true);
navigate(generatePath(`${rootLink()}/tasks/:taskId`, { taskId: id }));
} catch (e) {
errorApi.post(e);
}
@@ -134,7 +100,7 @@ export const TemplatePage = () => {
if (!loading && !template) {
errorApi.post(new Error('Template was not found.'));
return <Navigate to={rootRoute.path} />;
return <Navigate to={rootLink()} />;
}
if (template && !template?.spec?.schema) {
@@ -143,7 +109,7 @@ export const TemplatePage = () => {
'Template schema is corrupted, please check the template.yaml file.',
),
);
return <Navigate to={rootRoute.path} />;
return <Navigate to={rootLink()} />;
}
return (
@@ -159,12 +125,6 @@ export const TemplatePage = () => {
/>
<Content>
{loading && <LinearProgress data-testid="loading-progress" />}
<JobStatusModal
job={job}
toCatalogLink={catalogLink}
open={modalOpen}
onModalClose={() => setModalOpen(false)}
/>
{template && (
<InfoCard title={template.metadata.title} noPadding>
<MultistepJsonForm
plugins/scaffolder/src/components/hooks/useEventStream.ts
+211
View File
@@ -0,0 +1,211 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { useImmerReducer } from 'use-immer';
import { useEffect } from 'react';
import { scaffolderApiRef, LogEvent } from '../../api';
import { ScaffolderTask, Status } from '../../types';
import { Subscription, useApi } from '@backstage/core';
type Step = {
id: string;
status: Status;
endedAt?: string;
startedAt?: string;
};
type TaskOutput = { entityRef?: string } & { [key in string]: string };
export type TaskStream = {
loading: boolean;
error?: Error;
stepLogs: { [stepId in string]: string[] };
completed: boolean;
task?: ScaffolderTask;
steps: { [stepId in string]: Step };
output?: TaskOutput;
};
type ReducerLogEntry = {
createdAt: string;
body: {
stepId?: string;
status?: Status;
message: string;
output?: TaskOutput;
};
};
type ReducerAction =
| { type: 'INIT'; data: ScaffolderTask }
| { type: 'LOGS'; data: ReducerLogEntry[] }
| { type: 'COMPLETED'; data: ReducerLogEntry }
| { type: 'ERROR'; data: Error };
function reducer(draft: TaskStream, action: ReducerAction) {
switch (action.type) {
case 'INIT': {
draft.steps = action.data.spec.steps.reduce((current, next) => {
current[next.id] = { status: 'open', id: next.id };
return current;
}, {} as { [stepId in string]: Step });
draft.stepLogs = action.data.spec.steps.reduce((current, next) => {
current[next.id] = [];
return current;
}, {} as { [stepId in string]: string[] });
draft.loading = false;
draft.error = undefined;
draft.completed = false;
draft.task = action.data;
return;
}
case 'LOGS': {
const entries = action.data;
const logLines = [];
for (const entry of entries) {
const logLine = `${entry.createdAt} ${entry.body.message}`;
logLines.push(logLine);
if (!entry.body.stepId || !draft.steps?.[entry.body.stepId]) {
continue;
}
const currentStepLog = draft.stepLogs?.[entry.body.stepId];
const currentStep = draft.steps?.[entry.body.stepId];
if (entry.body.status && entry.body.status !== currentStep.status) {
currentStep.status = entry.body.status;
if (currentStep.status === 'processing') {
currentStep.startedAt = entry.createdAt;
}
if (
['cancelled', 'failed', 'completed'].includes(currentStep.status)
) {
currentStep.endedAt = entry.createdAt;
}
}
currentStepLog?.push(logLine);
}
return;
}
case 'COMPLETED': {
draft.completed = true;
draft.output = action.data.body.output;
return;
}
case 'ERROR': {
draft.error = action.data;
draft.loading = false;
draft.completed = true;
return;
}
default:
return;
}
}
export const useTaskEventStream = (taskId: string): TaskStream => {
const scaffolderApi = useApi(scaffolderApiRef);
const [state, dispatch] = useImmerReducer(reducer, {
loading: true,
completed: false,
stepLogs: {} as { [stepId in string]: string[] },
steps: {} as { [stepId in string]: Step },
});
useEffect(() => {
let didCancel = false;
let subscription: Subscription | undefined;
let logPusher: NodeJS.Timeout | undefined;
scaffolderApi.getTask(taskId).then(
task => {
if (didCancel) {
return;
}
dispatch({ type: 'INIT', data: task });
// TODO(blam): Use a normal fetch to fetch the current log for the event stream
// and use that for an INIT_EVENTs dispatch event, and then
// use the last event ID to subscribe using after option to
// stream logs. Without this, if you have a lot of logs, it can look like the
// task is being rebuilt on load as it progresses through the steps at a slower
// rate whilst it builds the status from the event logs
const observable = scaffolderApi.streamLogs({ taskId });
const collectedLogEvents = new Array<LogEvent>();
function emitLogs() {
if (collectedLogEvents.length) {
const logs = collectedLogEvents.splice(
0,
collectedLogEvents.length,
);
dispatch({ type: 'LOGS', data: logs });
}
}
logPusher = setInterval(emitLogs, 500);
subscription = observable.subscribe({
next: event => {
switch (event.type) {
case 'log':
return collectedLogEvents.push(event);
case 'completion':
emitLogs();
dispatch({ type: 'COMPLETED', data: event });
return undefined;
default:
throw new Error(
`Unhandled event type ${event.type} in observer`,
);
}
},
error: error => {
emitLogs();
dispatch({ type: 'ERROR', data: error });
},
});
},
error => {
if (!didCancel) {
dispatch({ type: 'ERROR', data: error });
}
},
);
return () => {
didCancel = true;
if (subscription) {
subscription.unsubscribe();
}
if (logPusher) {
clearInterval(logPusher);
}
};
}, [scaffolderApi, dispatch, taskId]);
return state;
};
plugins/scaffolder/src/components/hooks/useJobPolling.ts
-62
View File
@@ -1,62 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { useEffect, useState } from 'react';
import { Job } from '../../types';
import { useApi } from '@backstage/core';
import { scaffolderApiRef } from '../../api';
import { useInterval } from 'react-use';
const DEFAULT_POLLING_INTERVAL = 1000;
export const useJobPolling = (
jobId: string | null,
onFinish?: (j: Job) => void,
pollingInterval = DEFAULT_POLLING_INTERVAL,
) => {
const scaffolderApi = useApi(scaffolderApiRef);
const [currentJob, setCurrentJob] = useState<Job | null>(null);
useEffect(() => {
const resetCurrentJob = async () => {
if (jobId) {
const job = await scaffolderApi.getJob(jobId);
setCurrentJob(job);
}
};
resetCurrentJob();
}, [jobId, scaffolderApi]);
const shouldBeRunningInterval =
jobId &&
currentJob?.status !== 'COMPLETED' &&
currentJob?.status !== 'FAILED';
useInterval(
async () => {
if (jobId) {
const job = await scaffolderApi.getJob(jobId);
if (job?.status === 'COMPLETED' || job?.status === 'FAILED') {
onFinish?.(job);
}
setCurrentJob(job);
}
},
shouldBeRunningInterval ? pollingInterval : null,
);
return currentJob;
};
plugins/scaffolder/src/index.ts
+3 -4
View File
@@ -17,8 +17,7 @@
export {
scaffolderPlugin,
scaffolderPlugin as plugin,
TemplateIndexPage,
TemplatePage,
ScaffolderPage,
} from './plugin';
export { ScaffolderApi, scaffolderApiRef } from './api';
export { rootRoute, templateRoute } from './routes';
export type { ScaffolderApi } from './api';
export { ScaffolderClient, scaffolderApiRef } from './api';
plugins/scaffolder/src/plugin.ts
+7 -23
View File
@@ -21,10 +21,8 @@ import {
identityApiRef,
createRoutableExtension,
} from '@backstage/core';
import { ScaffolderPage as ScaffolderPageComponent } from './components/ScaffolderPage';
import { TemplatePage as TemplatePageComponent } from './components/TemplatePage';
import { rootRoute, templateRoute } from './routes';
import { scaffolderApiRef, ScaffolderApi } from './api';
import { rootRouteRef } from './routes';
import { scaffolderApiRef, ScaffolderClient } from './api';
export const scaffolderPlugin = createPlugin({
id: 'scaffolder',
@@ -33,31 +31,17 @@ export const scaffolderPlugin = createPlugin({
api: scaffolderApiRef,
deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef },
factory: ({ discoveryApi, identityApi }) =>
new ScaffolderApi({ discoveryApi, identityApi }),
new ScaffolderClient({ discoveryApi, identityApi }),
}),
],
register({ router }) {
router.addRoute(rootRoute, ScaffolderPageComponent);
router.addRoute(templateRoute, TemplatePageComponent);
},
routes: {
templateIndex: rootRoute,
template: templateRoute,
root: rootRouteRef,
},
});
export const TemplateIndexPage = scaffolderPlugin.provide(
export const ScaffolderPage = scaffolderPlugin.provide(
createRoutableExtension({
component: () =>
import('./components/ScaffolderPage').then(m => m.ScaffolderPage),
mountPoint: rootRoute,
}),
);
export const TemplatePage = scaffolderPlugin.provide(
createRoutableExtension({
component: () =>
import('./components/TemplatePage').then(m => m.TemplatePage),
mountPoint: templateRoute,
component: () => import('./components/Router').then(m => m.Router),
mountPoint: rootRouteRef,
}),
);
plugins/scaffolder/src/routes.ts
+1 -7
View File
@@ -15,12 +15,6 @@
*/
import { createRouteRef } from '@backstage/core';
export const rootRoute = createRouteRef({
path: '/create',
export const rootRouteRef = createRouteRef({
title: 'Create new entity',
});
export const templateRoute = createRouteRef({
path: '/create/:templateName',
title: 'Entity creation',
});
plugins/scaffolder/src/types.ts
+19
View File
@@ -13,7 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { JsonValue } from '@backstage/config';
export type Status = 'open' | 'processing' | 'failed' | 'completed';
export type JobStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED';
export type Job = {
id: string;
@@ -35,3 +37,20 @@ export type Stage = {
startedAt: string;
endedAt?: string;
};
export type ScaffolderStep = {
id: string;
name: string;
action: string;
parameters?: { [name: string]: JsonValue };
};
export type ScaffolderTask = {
id: string;
spec: {
steps: ScaffolderStep[];
};
status: 'failed' | 'completed' | 'processing' | 'open' | 'cancelled';
lastHeartbeatAt: string;
createdAt: string;
};