diff --git a/.changeset/clever-tomatoes-change.md b/.changeset/clever-tomatoes-change.md new file mode 100644 index 0000000000..c57953efb9 --- /dev/null +++ b/.changeset/clever-tomatoes-change.md @@ -0,0 +1,53 @@ +--- +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': minor +--- + +# Stateless scaffolding + +The scaffolder has been redesigned to be horizontally scalable and to persistently store task state and execution logs in the database. + +Each scaffolder task is given a unique task ID which is persisted in the database. +Tasks are then picked up by a `TaskWorker` which performs the scaffolding steps. +Execution logs are also persisted in the database meaning you can now refresh the scaffolder task status page without losing information. + +The task status page is now dynamically created based on the step information stored in the database. +This allows for custom steps to be displayed once the next version of the scaffolder template schema is available. + +The task page is updated to display links to both the git repository and to the newly created catalog entity. + +Component registration has moved from the frontend into a separate registration step executed by the `TaskWorker`. This requires that a `CatalogClient` is passed to the scaffolder backend instead of the old `CatalogEntityClient`. + +Make sure to update `plugins/scaffolder.ts` + +```diff + import { + CookieCutter, + createRouter, + Preparers, + Publishers, + CreateReactAppTemplater, + Templaters, +- CatalogEntityClient, + } from '@backstage/plugin-scaffolder-backend'; + ++import { CatalogClient } from '@backstage/catalog-client'; + + const discovery = SingleHostDiscovery.fromConfig(config); +-const entityClient = new CatalogEntityClient({ discovery }); ++const catalogClient = new CatalogClient({ discoveryApi: discovery }) + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, +- entityClient, + database, ++ catalogClient, + }); +``` + +As well as adding the `@backstage/catalog-client` packages as a dependency of your backend package. diff --git a/.changeset/cyan-dingos-watch.md b/.changeset/cyan-dingos-watch.md new file mode 100644 index 0000000000..7b15d30112 --- /dev/null +++ b/.changeset/cyan-dingos-watch.md @@ -0,0 +1,5 @@ +--- +'@backstage/test-utils': patch +--- + +Allow `ExternalRouteRef` instances to be passed as a route ref to `mountedRoutes`. diff --git a/.changeset/dingo-dongo.md b/.changeset/dingo-dongo.md new file mode 100644 index 0000000000..424c88f61c --- /dev/null +++ b/.changeset/dingo-dongo.md @@ -0,0 +1,39 @@ +--- +'@backstage/plugin-catalog': minor +'@backstage/plugin-scaffolder': minor +--- + +The Scaffolder and Catalog plugins have been migrated to partially require use of the [new composability API](https://backstage.io/docs/plugins/composability). The Scaffolder used to register its pages using the deprecated route registration plugin API, but those registrations have been removed. This means you now need to add the Scaffolder plugin page to the app directly. + +The page is imported from the Scaffolder plugin and added to the `` component: + +```tsx +} /> +``` + +The Catalog plugin has also been migrated to use an [external route reference](https://backstage.io/docs/plugins/composability#binding-external-routes-in-the-app) to dynamically link to the create component page. This means you need to migrate the catalog plugin to use the new extension components, as well as bind the external route. + +To use the new extension components, replace existing usage of the `CatalogRouter` with the following: + +```tsx +} /> +}> + + +``` + +And to bind the external route from the catalog plugin to the scaffolder template index page, make sure you have the appropriate imports and add the following to the `createApp` call: + +```ts +import { catalogPlugin } from '@backstage/plugin-catalog'; +import { scaffolderPlugin } from '@backstage/plugin-scaffolder'; + +const app = createApp({ + // ... + bindRoutes({ bind }) { + bind(catalogPlugin.externalRoutes, { + createComponent: scaffolderPlugin.routes.root, + }); + }, +}); +``` diff --git a/.changeset/weak-foxes-explain.md b/.changeset/weak-foxes-explain.md new file mode 100644 index 0000000000..5536e1982c --- /dev/null +++ b/.changeset/weak-foxes-explain.md @@ -0,0 +1,86 @@ +--- +'@backstage/create-app': patch +--- + +**BREAKING CHANGE** + +The Scaffolder and Catalog plugins have been migrated to partially require use of the [new composability API](https://backstage.io/docs/plugins/composability). The Scaffolder used to register its pages using the deprecated route registration plugin API, but those registrations have been removed. This means you now need to add the Scaffolder plugin page to the app directly. + +The Catalog plugin has also been migrated to use an [external route reference](https://backstage.io/docs/plugins/composability#binding-external-routes-in-the-app) to dynamically link to the create component page. This means you need to migrate the catalog plugin to use the new extension components, as well as bind the external route. + +Apply the following changes to `packages/app/src/App.tsx`: + +```diff +-import { Router as CatalogRouter } from '@backstage/plugin-catalog'; ++import { ++ catalogPlugin, ++ CatalogIndexPage, ++ CatalogEntityPage, ++} from '@backstage/plugin-catalog'; ++import { scaffolderPlugin, ScaffolderPage } from '@backstage/plugin-scaffolder'; + +# The following addition to the app config allows the catalog plugin to link to the +# component creation page, i.e. the scaffolder. You can chose a different target if you want to. + const app = createApp({ + apis, + plugins: Object.values(plugins), ++ bindRoutes({ bind }) { ++ bind(catalogPlugin.externalRoutes, { ++ createComponent: scaffolderPlugin.routes.root, ++ }); ++ } + }); + +# Apply these changes within FlatRoutes. It is important to have migrated to using FlatRoutes +# for this to work, if you haven't done that yet, see the previous entries in this changelog. +- } +- /> ++ } /> ++ } ++ > ++ ++ + } /> ++ } /> +``` + +The scaffolder has been redesigned to be horizontally scalable and to persistently store task state and execution logs in the database. Component registration has moved from the frontend into a separate registration step executed by the `TaskWorker`. This requires that a `CatalogClient` is passed to the scaffolder backend instead of the old `CatalogEntityClient`. + +The default catalog client comes from the `@backstage/catalog-client`, which you need to add as a dependency in `packages/backend/package.json`. + +Once the dependency has been added, apply the following changes to`packages/backend/src/plugins/scaffolder.ts`: + +```diff + import { + CookieCutter, + createRouter, + Preparers, + Publishers, + CreateReactAppTemplater, + Templaters, +- CatalogEntityClient, + } from '@backstage/plugin-scaffolder-backend'; ++import { CatalogClient } from '@backstage/catalog-client'; + + const discovery = SingleHostDiscovery.fromConfig(config); +-const entityClient = new CatalogEntityClient({ discovery }); ++const catalogClient = new CatalogClient({ discoveryApi: discovery }) + + return await createRouter({ + preparers, + templaters, + publishers, + logger, + config, + dockerClient, +- entityClient, + database, ++ catalogClient, + }); +``` + +See the `@backstage/scaffolder-backend` changelog for more information about this change. diff --git a/docs/features/software-catalog/installation.md b/docs/features/software-catalog/installation.md index b2afd62878..ae8c671ed1 100644 --- a/docs/features/software-catalog/installation.md +++ b/docs/features/software-catalog/installation.md @@ -30,33 +30,59 @@ it doesn't. Add the following entry to the head of your `packages/app/src/plugins.ts`: ```ts -export { plugin as CatalogPlugin } from '@backstage/plugin-catalog'; +export { catalogPlugin } from '@backstage/plugin-catalog'; ``` -Add the following to your `packages/app/src/apis.ts`: +Next we need to install the two pages that the catalog plugin provides. You can +choose any name for these routes, but we recommend the following: + +```tsx +import { + catalogPlugin, + CatalogIndexPage, + CatalogEntityPage, +} from '@backstage/plugin-catalog'; + +// Add to the top-level routes, directly within +} /> +}> + {/* + This is the root of the custom entity pages for your app, refer to the example app + in the main repo or the output of @backstage/create-app for an example + */} + + +``` + +The catalog plugin also has one external route that needs to be bound for it to +function: the `createComponent` route which should link to the page where the +user can create components. In a typical setup the create component route will +be linked to the Scaffolder plugin's template index page: ```ts -import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; +import { catalogPlugin } from '@backstage/plugin-catalog'; +import { scaffolderPlugin } from '@backstage/plugin-scaffolder'; -// Inside the ApiRegistry builder function ... - -builder.add( - catalogApiRef, - new CatalogClient({ - apiOrigin: backendUrl, - basePath: '/catalog', - }), -); +const app = createApp({ + // ... + bindRoutes({ bind }) { + bind(catalogPlugin.externalRoutes, { + createComponent: scaffolderPlugin.routes.root, + }); + }, +}); ``` -Where `backendUrl` is the `backend.baseUrl` from config, i.e. -`const backendUrl = config.getString('backend.baseUrl')`. +You may also want to add a link to the catalog index page to your sidebar: -The catalog components depend on a number of other -[Utility APIs](../../api/utility-apis.md) to function, including at least the -`ErrorApi` and `StorageApi`. You can find an example of how to install these in -your app -[here](https://github.com/backstage/backstage/blob/61c3a7e5b750dc7c059ef16b188594d31b2c04c2/packages/app/src/apis.ts#L80). +```tsx +import HomeIcon from '@material-ui/icons/Home'; + +// Somewhere within the +; +``` + +This is all that is needed for the frontend part of the Catalog plugin to work! ## Gotchas that we will fix diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 8d2fa7727d..9aa1d3bc8f 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -33,27 +33,27 @@ it doesn't. Add the following entry to the head of your `packages/app/src/plugins.ts`: ```ts -export { plugin as ScaffolderPlugin } from '@backstage/plugin-scaffolder'; +export { scaffolderPlugin } from '@backstage/plugin-scaffolder'; ``` -Add the following to your `packages/app/src/apis.ts`: +Next we need to install the root page that the Scaffolder plugin provides. You +can choose any path for the route, but we recommend the following: -```ts -import { scaffolderApiRef, ScaffolderApi } from '@backstage/plugin-scaffolder'; +```tsx +import { ScaffolderPage } from '@backstage/plugin-scaffolder'; -// Inside the ApiRegistry builder function ... - -builder.add( - scaffolderApiRef, - new ScaffolderApi({ - apiOrigin: backendUrl, - basePath: '/scaffolder/v1', - }), -); +// Add to the top-level routes, directly within +} />; ``` -Where `backendUrl` is the `backend.baseUrl` from config, i.e. -`const backendUrl = config.getString('backend.baseUrl')`. +You may also want to add a link to the template index page to your sidebar: + +```tsx +import CreateComponentIcon from '@material-ui/icons/AddCircleOutline'; + +// Somewhere within the +; +``` This is all that is needed for the frontend part of the Scaffolder plugin to work! @@ -85,29 +85,25 @@ following contents to get you up and running quickly. import { CookieCutter, createRouter, - FilePreparer, - GithubPreparer, - GitlabPreparer, Preparers, Publishers, - GithubPublisher, - GitlabPublisher, CreateReactAppTemplater, Templaters, - RepoVisibilityOptions, } from '@backstage/plugin-scaffolder-backend'; -import { Octokit } from '@octokit/rest'; -import { Gitlab } from '@gitbeaker/node'; +import { SingleHostDiscovery } from '@backstage/backend-common'; import type { PluginEnvironment } from '../types'; import Docker from 'dockerode'; +import { CatalogClient } from '@backstage/catalog-client'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment) { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); const templaters = new Templaters(); + templaters.register('cookiecutter', cookiecutterTemplater); templaters.register('cra', craTemplater); @@ -115,12 +111,19 @@ export default async function createPlugin({ const publishers = await Publishers.fromConfig(config, { logger }); const dockerClient = new Docker(); + + const discovery = SingleHostDiscovery.fromConfig(config); + const catalogClient = new CatalogClient({ discoveryApi: discovery }); + return await createRouter({ preparers, templaters, publishers, logger, + config, dockerClient, + database, + catalogClient, }); } ``` diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index cc673c7769..4a49db02d5 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -22,12 +22,17 @@ import { OAuthRequestDialog, SignInPage, } from '@backstage/core'; -import { Router as CatalogRouter } from '@backstage/plugin-catalog'; +import { + catalogPlugin, + CatalogIndexPage, + CatalogEntityPage, +} from '@backstage/plugin-catalog'; import { CatalogImportPage } from '@backstage/plugin-catalog-import'; import { ExplorePage } from '@backstage/plugin-explore'; import { Router as GraphiQLRouter } from '@backstage/plugin-graphiql'; import { Router as LighthouseRouter } from '@backstage/plugin-lighthouse'; import { Router as RegisterComponentRouter } from '@backstage/plugin-register-component'; +import { ScaffolderPage, scaffolderPlugin } from '@backstage/plugin-scaffolder'; import { Router as TechRadarRouter } from '@backstage/plugin-tech-radar'; import { Router as DocsRouter } from '@backstage/plugin-techdocs'; import { Router as SettingsRouter } from '@backstage/plugin-user-settings'; @@ -60,6 +65,11 @@ const app = createApp({ ); }, }, + bindRoutes({ bind }) { + bind(catalogPlugin.externalRoutes, { + createComponent: scaffolderPlugin.routes.root, + }); + }, }); const AppProvider = app.getProvider(); @@ -74,12 +84,16 @@ const catalogRouteRef = createRouteRef({ const routes = ( - } /> + } /> } - /> + path="/catalog/:namespace/:kind/:name" + element={} + > + + + } /> } /> + } /> } /> ( + } /> } - /> + path="/catalog/:namespace/:kind/:name" + element={} + > + + } /> + } /> } diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index 1af8276316..8c8cfae0b3 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -20,6 +20,7 @@ "app": "0.0.0", "@backstage/backend-common": "^{{version '@backstage/backend-common'}}", "@backstage/catalog-model": "^{{version '@backstage/catalog-model'}}", + "@backstage/catalog-client": "^{{version '@backstage/catalog-client'}}", "@backstage/config": "^{{version '@backstage/config'}}", "@backstage/plugin-app-backend": "^{{version '@backstage/plugin-app-backend'}}", "@backstage/plugin-auth-backend": "^{{version '@backstage/plugin-auth-backend'}}", diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts index d68f90ce08..6f42aaa327 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/scaffolder.ts @@ -5,11 +5,11 @@ import { Publishers, CreateReactAppTemplater, Templaters, - CatalogEntityClient, } from '@backstage/plugin-scaffolder-backend'; import { SingleHostDiscovery } from '@backstage/backend-common'; import type { PluginEnvironment } from '../types'; import Docker from 'dockerode'; +import { CatalogClient } from '@backstage/catalog-client'; export default async function createPlugin({ logger, @@ -29,7 +29,7 @@ export default async function createPlugin({ const dockerClient = new Docker(); const discovery = SingleHostDiscovery.fromConfig(config); - const entityClient = new CatalogEntityClient({ discovery }); + const catalogClient = new CatalogClient({ discoveryApi: discovery }); return await createRouter({ preparers, @@ -38,7 +38,7 @@ export default async function createPlugin({ logger, config, dockerClient, - entityClient, database, + catalogClient, }); } diff --git a/packages/test-utils/src/testUtils/appWrappers.tsx b/packages/test-utils/src/testUtils/appWrappers.tsx index cc67a5b707..05cd0b8fd4 100644 --- a/packages/test-utils/src/testUtils/appWrappers.tsx +++ b/packages/test-utils/src/testUtils/appWrappers.tsx @@ -22,8 +22,8 @@ import privateExports, { defaultSystemIcons, BootErrorPageProps, RouteRef, - createPlugin, - createRoutableExtension, + ExternalRouteRef, + attachComponentData, } from '@backstage/core-api'; import { RenderResult } from '@testing-library/react'; import { renderWithEffects } from '@backstage/test-utils-core'; @@ -62,7 +62,7 @@ type TestAppOptions = { * // ... * const link = useRouteRef(myRouteRef) */ - mountedRoutes?: { [path: string]: RouteRef }; + mountedRoutes?: { [path: string]: RouteRef | ExternalRouteRef }; }; /** @@ -109,16 +109,10 @@ export function wrapInTestApp( Wrapper = () => Component as React.ReactElement; } - const routePlugin = createPlugin({ id: 'mock-route-plugin' }); const routeElements = Object.entries(options.mountedRoutes ?? {}).map( ([path, routeRef]) => { - const PageComponent = () =>
Mounted at {path}
; - const Page = routePlugin.provide( - createRoutableExtension({ - component: async () => PageComponent, - mountPoint: routeRef, - }), - ); + const Page = () =>
Mounted at {path}
; + attachComponentData(Page, 'core.mountPoint', routeRef); return } />; }, ); diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index a03be16632..6efb450597 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -34,7 +34,6 @@ "@backstage/catalog-model": "^0.7.1", "@backstage/core": "^0.6.2", "@backstage/plugin-catalog-react": "^0.0.4", - "@backstage/plugin-scaffolder": "^0.5.1", "@backstage/theme": "^0.2.3", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx index 6d4d18e4f2..accfe6eda4 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.test.tsx @@ -33,6 +33,7 @@ import { MockStorageApi, wrapInTestApp } from '@backstage/test-utils'; import { fireEvent, render, waitFor } from '@testing-library/react'; import React from 'react'; import { EntityFilterGroupsProvider } from '../../filter'; +import { createComponentRouteRef } from '../../routes'; import { CatalogPage } from './CatalogPage'; describe('CatalogPage', () => { @@ -116,6 +117,11 @@ describe('CatalogPage', () => { > {children}, , + { + mountedRoutes: { + '/create': createComponentRouteRef, + }, + }, ), ); diff --git a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx index a2ebc63825..ed51316dcd 100644 --- a/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx +++ b/plugins/catalog/src/components/CatalogPage/CatalogPage.tsx @@ -21,9 +21,9 @@ import { errorApiRef, SupportButton, useApi, + useRouteRef, } from '@backstage/core'; import { catalogApiRef, isOwnerOf } from '@backstage/plugin-catalog-react'; -import { rootRoute as scaffolderRootRoute } from '@backstage/plugin-scaffolder'; import { Button, makeStyles } from '@material-ui/core'; import SettingsIcon from '@material-ui/icons/Settings'; import StarIcon from '@material-ui/icons/Star'; @@ -31,6 +31,7 @@ import React, { useCallback, useMemo, useState } from 'react'; import { Link as RouterLink } from 'react-router-dom'; import { EntityFilterGroupsProvider, useFilteredEntities } from '../../filter'; import { useStarredEntities } from '../../hooks/useStarredEntities'; +import { createComponentRouteRef } from '../../routes'; import { ButtonGroup, CatalogFilter, @@ -73,7 +74,7 @@ const CatalogPageContents = () => { CatalogFilterType >(); const orgName = configApi.getOptionalString('organization.name') ?? 'Company'; - + const createComponentLink = useRouteRef(createComponentRouteRef); const addMockData = useCallback(async () => { try { const promises: Promise[] = []; @@ -166,7 +167,7 @@ const CatalogPageContents = () => { component={RouterLink} variant="contained" color="primary" - to={scaffolderRootRoute.path} + to={createComponentLink()} > Create Component diff --git a/plugins/catalog/src/plugin.ts b/plugins/catalog/src/plugin.ts index a0932a8dba..19dc778280 100644 --- a/plugins/catalog/src/plugin.ts +++ b/plugins/catalog/src/plugin.ts @@ -29,6 +29,7 @@ import { entityRouteRef, } from '@backstage/plugin-catalog-react'; import { CatalogClientWrapper } from './CatalogClientWrapper'; +import { createComponentRouteRef } from './routes'; export const catalogPlugin = createPlugin({ id: 'catalog', @@ -47,6 +48,9 @@ export const catalogPlugin = createPlugin({ catalogIndex: catalogRouteRef, catalogEntity: entityRouteRef, }, + externalRoutes: { + createComponent: createComponentRouteRef, + }, }); export const CatalogIndexPage = catalogPlugin.provide( diff --git a/plugins/scaffolder/src/components/JobStatusModal/index.ts b/plugins/catalog/src/routes.ts similarity index 75% rename from plugins/scaffolder/src/components/JobStatusModal/index.ts rename to plugins/catalog/src/routes.ts index 5598999fe3..40e1784235 100644 --- a/plugins/scaffolder/src/components/JobStatusModal/index.ts +++ b/plugins/catalog/src/routes.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,4 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { JobStatusModal } from './JobStatusModal'; + +import { createExternalRouteRef } from '@backstage/core'; + +export const createComponentRouteRef = createExternalRouteRef({ + id: 'create-component', +}); diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 06cac907de..24636ce20e 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -51,10 +51,12 @@ "fs-extra": "^9.0.0", "git-url-parse": "^11.4.4", "globby": "^11.0.0", + "handlebars": "^4.7.6", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "jsonschema": "^1.2.6", "knex": "^0.21.6", + "luxon": "^1.26.0", "morgan": "^1.10.0", "uuid": "^8.2.0", "winston": "^3.2.1", @@ -67,9 +69,9 @@ "@types/mock-fs": "^4.13.0", "@types/supertest": "^2.0.8", "mock-fs": "^4.13.0", + "msw": "^0.21.2", "supertest": "^4.0.2", - "yaml": "^1.10.0", - "msw": "^0.21.2" + "yaml": "^1.10.0" }, "files": [ "dist", diff --git a/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts b/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts index 827ed0559a..cea0e85148 100644 --- a/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts +++ b/plugins/scaffolder-backend/src/lib/catalog/CatalogEntityClient.ts @@ -15,24 +15,14 @@ */ import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; -import { CatalogClient } from '@backstage/catalog-client'; -import { - ConflictError, - NotFoundError, - PluginEndpointDiscovery, -} from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; +import { ConflictError, NotFoundError } from '@backstage/backend-common'; /** * A catalog client tailored for reading out entity data from the catalog. */ export class CatalogEntityClient { - private readonly catalogClient: CatalogClient; - - constructor(options: { discovery: PluginEndpointDiscovery }) { - this.catalogClient = new CatalogClient({ - discoveryApi: options.discovery, - }); - } + constructor(private readonly catalogClient: CatalogApi) {} /** * Looks up a single template using a template name. diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts index f151780ba9..efba57a063 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/legacy.ts @@ -19,28 +19,37 @@ import { FilePreparer, PreparerBuilder } from './prepare'; import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from './templater'; import { PublisherBuilder } from './publish'; +import { CatalogApi } from '@backstage/catalog-client'; +import { getEntityName } from '@backstage/catalog-model'; type Options = { dockerClient: Docker; preparers: PreparerBuilder; templaters: TemplaterBuilder; publishers: PublisherBuilder; + catalogClient: CatalogApi; }; export function registerLegacyActions( registry: TemplateActionRegistry, options: Options, ) { - const { dockerClient, preparers, templaters, publishers } = options; + const { + dockerClient, + preparers, + templaters, + publishers, + catalogClient, + } = options; registry.register({ id: 'legacy:prepare', async handler(ctx) { + ctx.logger.info('Preparing the skeleton'); const { protocol, url } = ctx.parameters; const preparer = protocol === 'file' ? new FilePreparer() : preparers.get(url as string); - ctx.logger.info('Prepare the skeleton'); await preparer.prepare({ url: url as string, logger: ctx.logger, @@ -52,11 +61,8 @@ export function registerLegacyActions( registry.register({ id: 'legacy:template', async handler(ctx) { - const { logger } = ctx; - + ctx.logger.info('Running the templater'); const templater = templaters.get(ctx.parameters.templater as string); - - logger.info('Run the templater'); await templater.run({ workspacePath: ctx.workspacePath, dockerClient, @@ -107,4 +113,21 @@ export function registerLegacyActions( } }, }); + + registry.register({ + id: 'catalog:register', + async handler(ctx) { + const { catalogInfoUrl } = ctx.parameters; + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); + + const result = await catalogClient.addLocation({ + type: 'url', + target: catalogInfoUrl as string, + }); + if (result.entities.length >= 1) { + const { kind, name, namespace } = getEntityName(result.entities[0]); + ctx.output('entityRef', `${kind}:${namespace}/${name}`); + } + }, + }); } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 5717f3c8c1..d069008282 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -32,6 +32,7 @@ import { TaskStoreEmitOptions, TaskStoreGetEventsOptions, } from './types'; +import { DateTime } from 'luxon'; const migrationsDir = resolvePackagePath( '@backstage/plugin-scaffolder-backend', @@ -64,7 +65,7 @@ export class DatabaseTaskStore implements TaskStore { constructor(private readonly db: Knex) {} - async get(taskId: string): Promise { + async getTask(taskId: string): Promise { const [result] = await this.db('tasks') .where({ id: taskId }) .select(); @@ -255,11 +256,14 @@ export class DatabaseTaskStore implements TaskStore { try { const body = JSON.parse(event.body) as JsonObject; return { - id: event.id, + id: Number(event.id), taskId, body, type: event.event_type, - createdAt: event.created_at, + createdAt: + typeof event.created_at === 'string' + ? DateTime.fromSQL(event.created_at, { zone: 'UTC' }).toISO() + : event.created_at, }; } catch (error) { throw new Error( diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts index c45fa46335..fd2de30332 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.test.ts @@ -47,7 +47,7 @@ describe('StorageTaskBroker', () => { const logger = getVoidLogger(); it('should claim a dispatched work item', async () => { const broker = new StorageTaskBroker(storage, logger); - await broker.dispatch({ steps: [] }); + await broker.dispatch({} as TaskSpec); await expect(broker.claim()).resolves.toEqual(expect.any(TaskAgent)); }); @@ -57,7 +57,7 @@ describe('StorageTaskBroker', () => { await expect(Promise.race([promise, 'waiting'])).resolves.toBe('waiting'); - await broker.dispatch({ steps: [] }); + await broker.dispatch({} as TaskSpec); await expect(promise).resolves.toEqual(expect.any(TaskAgent)); }); @@ -80,19 +80,19 @@ describe('StorageTaskBroker', () => { it('should complete a task', async () => { const broker = new StorageTaskBroker(storage, logger); - const dispatchResult = await broker.dispatch({ steps: [] }); + const dispatchResult = await broker.dispatch({} as TaskSpec); const task = await broker.claim(); await task.complete('completed'); - const taskRow = await storage.get(dispatchResult.taskId); + const taskRow = await storage.getTask(dispatchResult.taskId); expect(taskRow.status).toBe('completed'); }, 10000); it('should fail a task', async () => { const broker = new StorageTaskBroker(storage, logger); - const dispatchResult = await broker.dispatch({ steps: [] }); + const dispatchResult = await broker.dispatch({} as TaskSpec); const task = await broker.claim(); await task.complete('failed'); - const taskRow = await storage.get(dispatchResult.taskId); + const taskRow = await storage.getTask(dispatchResult.taskId); expect(taskRow.status).toBe('failed'); }); @@ -100,7 +100,7 @@ describe('StorageTaskBroker', () => { const broker1 = new StorageTaskBroker(storage, logger); const broker2 = new StorageTaskBroker(storage, logger); - const { taskId } = await broker1.dispatch({ steps: [] }); + const { taskId } = await broker1.dispatch({} as TaskSpec); const logPromise = new Promise(resolve => { const observedEvents = new Array(); @@ -139,13 +139,13 @@ describe('StorageTaskBroker', () => { it('should heartbeat', async () => { const broker = new StorageTaskBroker(storage, logger); - const { taskId } = await broker.dispatch({ steps: [] }); + const { taskId } = await broker.dispatch({} as TaskSpec); const task = await broker.claim(); - const initialTask = await storage.get(taskId); + const initialTask = await storage.getTask(taskId); for (;;) { - const maybeTask = await storage.get(taskId); + const maybeTask = await storage.getTask(taskId); if (maybeTask.lastHeartbeatAt !== initialTask.lastHeartbeatAt) { break; } @@ -157,7 +157,7 @@ describe('StorageTaskBroker', () => { it('should be update the status to failed if heartbeat fails', async () => { const broker = new StorageTaskBroker(storage, logger); - const { taskId } = await broker.dispatch({ steps: [] }); + const { taskId } = await broker.dispatch({} as TaskSpec); const task = await broker.claim(); jest @@ -169,7 +169,7 @@ describe('StorageTaskBroker', () => { }, 500); for (;;) { - const maybeTask = await storage.get(taskId); + const maybeTask = await storage.getTask(taskId); if (maybeTask.status === 'failed') { break; } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts index 48c0d5e5ba..fb1f4ab422 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/StorageTaskBroker.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { JsonObject } from '@backstage/config'; import { Logger } from 'winston'; import { CompletedTaskState, @@ -22,6 +23,7 @@ import { TaskBroker, DispatchResult, DbTaskEventRow, + DbTaskRow, } from './types'; export class TaskAgent implements Task { @@ -54,18 +56,24 @@ export class TaskAgent implements Task { return this.isDone; } - async emitLog(message: string): Promise { + async emitLog(message: string, metadata?: JsonObject): Promise { await this.storage.emitLogEvent({ taskId: this.state.taskId, - body: { message }, + body: { message, ...metadata }, }); } - async complete(result: CompletedTaskState): Promise { + async complete( + result: CompletedTaskState, + metadata?: JsonObject, + ): Promise { await this.storage.completeTask({ taskId: this.state.taskId, status: result === 'failed' ? 'failed' : 'completed', - eventBody: { message: `Run completed with status: ${result}` }, + eventBody: { + message: `Run completed with status: ${result}`, + ...metadata, + }, }); this.isDone = true; if (this.heartbeatTimeoutId) { @@ -136,6 +144,10 @@ export class StorageTaskBroker implements TaskBroker { }; } + async get(taskId: string): Promise { + return this.storage.getTask(taskId); + } + observe( options: { taskId: string; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index c62495ca4c..e717fcae1b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -22,6 +22,7 @@ import { TaskBroker, Task } from './types'; import fs from 'fs-extra'; import path from 'path'; import { TemplateActionRegistry } from './TemplateConverter'; +import * as handlebars from 'handlebars'; type Options = { logger: Logger; @@ -44,67 +45,116 @@ export class TaskWorker { async runOneTask(task: Task) { try { - const { actionRegistry, logger } = this.options; + const { actionRegistry } = this.options; const workspacePath = path.join( this.options.workingDirectory, await task.getWorkspaceName(), ); await fs.ensureDir(workspacePath); + await task.emitLog( + `Starting up task with ${task.spec.steps.length} steps`, + ); - const taskLogger = winston.createLogger({ - level: process.env.LOG_LEVEL || 'info', - format: winston.format.combine( - winston.format.colorize(), - winston.format.timestamp(), - winston.format.simple(), - ), - defaultMeta: {}, - }); - - const stream = new PassThrough(); - stream.on('data', data => { - const message = data.toString().trim(); - if (message?.length > 1) task.emitLog(message); - }); - - taskLogger.add(new winston.transports.Stream({ stream })); - - // Give us some time to curl observe - task.emitLog('Task claimed, waiting ...'); - await new Promise(resolve => setTimeout(resolve, 5000)); - task.emitLog(`Starting up work with ${task.spec.steps.length} steps`); - - const outputs: { [name: string]: JsonValue } = {}; + const templateCtx: { + steps: { + [stepName: string]: { output: { [outputName: string]: JsonValue } }; + }; + } = { steps: {} }; for (const step of task.spec.steps) { - task.emitLog(`Beginning step ${step.name}`); + const metadata = { stepId: step.id }; + try { + const taskLogger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: winston.format.combine( + winston.format.colorize(), + winston.format.timestamp(), + winston.format.simple(), + ), + defaultMeta: {}, + }); - const action = actionRegistry.get(step.action); - if (!action) { - throw new Error(`Action '${step.action}' does not exist`); + const stream = new PassThrough(); + stream.on('data', async data => { + const message = data.toString().trim(); + if (message?.length > 1) { + await task.emitLog(message, metadata); + } + }); + + taskLogger.add(new winston.transports.Stream({ stream })); + await task.emitLog(`Beginning step ${step.name}`, { + ...metadata, + status: 'processing', + }); + + const action = actionRegistry.get(step.action); + if (!action) { + throw new Error(`Action '${step.action}' does not exist`); + } + + const parameters: { [name: string]: JsonValue } = {}; + for (const [name, maybeTemplateStr] of Object.entries( + step.parameters ?? {}, + )) { + if (typeof maybeTemplateStr === 'string') { + const value = handlebars.compile(maybeTemplateStr, { + noEscape: true, + strict: true, + data: false, + preventIndent: true, + })(templateCtx); + parameters[name] = value; + } else { + parameters[name] = maybeTemplateStr; + } + } + + const stepOutputs: { [name: string]: JsonValue } = {}; + + await action.handler({ + logger: taskLogger, + logStream: stream, + parameters, + workspacePath, + output(name: string, value: JsonValue) { + stepOutputs[name] = value; + }, + }); + + templateCtx.steps[step.id] = { output: stepOutputs }; + + await task.emitLog(`Finished step ${step.name}`, { + ...metadata, + status: 'completed', + }); + } catch (error) { + await task.emitLog(String(error.stack), { + ...metadata, + status: 'failed', + }); + throw error; } - - // TODO: substitute any placeholders with output from previous steps - const parameters = step.parameters!; - - await action.handler({ - logger, - logStream: stream, - parameters, - workspacePath, - output(name: string, value: JsonValue) { - outputs[name] = value; - }, - }); - - task.emitLog(`Finished step ${step.name}`); } - await task.complete('completed'); + const output = Object.fromEntries( + Object.entries(task.spec.output).map(([name, templateStr]) => { + const value = handlebars.compile(templateStr, { + noEscape: true, + strict: true, + data: false, + preventIndent: true, + })(templateCtx); + return [name, value]; + }), + ); + + await task.complete('completed', { output }); } catch (error) { - task.emitLog(String(error.stack)); - await task.complete('failed'); + await task.complete('failed', { + error: { name: error.name, message: error.message }, + }); } } } diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index 69788238cc..4139948513 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -18,7 +18,7 @@ import { resolve as resolvePath } from 'path'; import { JsonValue } from '@backstage/config'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { Logger } from 'winston'; -import type { Writable } from 'stream'; +import { Writable } from 'stream'; import { TaskSpec } from './types'; import { ConflictError, NotFoundError } from '@backstage/backend-common'; @@ -69,14 +69,30 @@ export function templateEntityToSpec( steps.push({ id: 'publish', - name: 'Publishing', + name: 'Publish', action: 'legacy:publish', parameters: { values, }, }); - return { steps }; + steps.push({ + id: 'register', + name: 'Register', + action: 'catalog:register', + parameters: { + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', + }, + }); + + return { + steps, + output: { + remoteUrl: '{{ steps.publish.output.remoteUrl }}', + catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', + entityRef: '{{ steps.register.output.entityRef }}', + }, + }; } type ActionContext = { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts index 0c2592109b..ae18a59e41 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/types.ts @@ -49,6 +49,7 @@ export type TaskSpec = { action: string; parameters?: { [name: string]: JsonValue }; }>; + output: { [name: string]: string }; }; export type DispatchResult = { @@ -58,8 +59,8 @@ export type DispatchResult = { export interface Task { spec: TaskSpec; done: boolean; - emitLog(message: string): Promise; - complete(result: CompletedTaskState): Promise; + emitLog(message: string, metadata?: JsonValue): Promise; + complete(result: CompletedTaskState, metadata?: JsonValue): Promise; getWorkspaceName(): Promise; } @@ -90,6 +91,7 @@ export type TaskStoreGetEventsOptions = { }; export interface TaskStore { createTask(task: TaskSpec): Promise<{ taskId: string }>; + getTask(taskId: string): Promise; claimTask(): Promise; completeTask(options: { taskId: string; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 8a0fa6cba6..8fd53e5fb6 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -39,12 +39,14 @@ import request from 'supertest'; import { createRouter } from './router'; import { Templaters, Preparers, Publishers } from '../scaffolder'; import Docker from 'dockerode'; +import { CatalogApi } from '@backstage/catalog-client'; jest.mock('dockerode'); -const generateEntityClient: any = (template: any) => ({ - findTemplate: () => Promise.resolve(template), -}); +const createCatalogClient = (templates: any[] = []) => + ({ + getEntities: async () => ({ items: templates }), + } as CatalogApi); function createDatabase(): PluginDatabaseManager { return SingleConnectionDatabaseManager.fromConfig( @@ -95,7 +97,6 @@ describe('createRouter - working directory', () => { }, }; - const mockedEntityClient = generateEntityClient(template); it('should throw an error when working directory does not exist or is not writable', async () => { mockAccess.mockImplementation(() => { throw new Error('access error'); @@ -109,8 +110,8 @@ describe('createRouter - working directory', () => { publishers: new Publishers(), config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), - entityClient: mockedEntityClient, database: createDatabase(), + catalogClient: createCatalogClient([template]), }), ).rejects.toThrow('access error'); }); @@ -123,8 +124,8 @@ describe('createRouter - working directory', () => { publishers: new Publishers(), config: new ConfigReader(workDirConfig('/path')), dockerClient: new Docker(), - entityClient: mockedEntityClient, database: createDatabase(), + catalogClient: createCatalogClient([template]), }); const app = express().use(router); @@ -152,8 +153,8 @@ describe('createRouter - working directory', () => { publishers: new Publishers(), config: new ConfigReader({}), dockerClient: new Docker(), - entityClient: mockedEntityClient, database: createDatabase(), + catalogClient: createCatalogClient([template]), }); const app = express().use(router); @@ -184,6 +185,9 @@ describe('createRouter', () => { name: 'create-react-app-template', tags: ['experimental', 'react', 'cra'], title: 'Create React App Template', + annotations: { + 'backstage.io/managed-by-location': 'url:https://dev.azure.com', + }, }, spec: { owner: 'web@example.com', @@ -222,8 +226,8 @@ describe('createRouter', () => { publishers: new Publishers(), config: new ConfigReader({}), dockerClient: new Docker(), - entityClient: generateEntityClient(template), database: createDatabase(), + catalogClient: createCatalogClient([template]), }); app = express().use(router); }); @@ -246,4 +250,36 @@ describe('createRouter', () => { expect(response.status).toEqual(400); }); }); + + describe('POST /v2/tasks', () => { + it('rejects template values which do not match the template schema definition', async () => { + const response = await request(app) + .post('/v2/tasks') + .send({ + templateName: '', + values: { + storePath: 'https://github.com/backstage/backstage', + }, + }); + + expect(response.status).toEqual(400); + }); + + it('return the template id', async () => { + const response = await request(app) + .post('/v2/tasks') + .send({ + templateName: 'create-react-app-template', + values: { + storePath: 'https://github.com/backstage/backstage', + component_id: '123', + name: 'test', + use_typescript: false, + }, + }); + + expect(response.body.id).toBeDefined(); + expect(response.status).toEqual(201); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index e8aa9e7c43..3d10bbd101 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -44,7 +44,11 @@ import { } from '../scaffolder/tasks/TemplateConverter'; import { registerLegacyActions } from '../scaffolder/stages/legacy'; import { getWorkingDirectory } from './helpers'; -import { PluginDatabaseManager } from '@backstage/backend-common'; +import { + NotFoundError, + PluginDatabaseManager, +} from '@backstage/backend-common'; +import { CatalogApi } from '@backstage/catalog-client'; export interface RouterOptions { preparers: PreparerBuilder; @@ -54,8 +58,8 @@ export interface RouterOptions { logger: Logger; config: Config; dockerClient: Docker; - entityClient: CatalogEntityClient; database: PluginDatabaseManager; + catalogClient: CatalogApi; } export async function createRouter( @@ -71,13 +75,14 @@ export async function createRouter( logger: parentLogger, config, dockerClient, - entityClient, database, + catalogClient, } = options; const logger = parentLogger.child({ plugin: 'scaffolder' }); const workingDirectory = await getWorkingDirectory(config, logger); const jobProcessor = await JobProcessor.fromConfig({ config, logger }); + const entityClient = new CatalogEntityClient(catalogClient); const databaseTaskStore = await DatabaseTaskStore.create( await database.getClient(), @@ -96,6 +101,7 @@ export async function createRouter( preparers, publishers, templaters, + catalogClient, }); worker.start(); @@ -249,6 +255,14 @@ export async function createRouter( res.status(201).json({ id: result.taskId }); }) + .get('/v2/tasks/:taskId', async (req, res) => { + const { taskId } = req.params; + const task = await taskBroker.get(taskId); + if (!task) { + throw new NotFoundError(`Task with id ${taskId} does not exist`); + } + res.status(200).json(task); + }) .get('/v2/tasks/:taskId/eventstream', async (req, res) => { const { taskId } = req.params; const after = Number(req.query.after) || undefined; diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index e75aadb03f..1501a1cd5e 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -19,8 +19,8 @@ import { createDevApp } from '@backstage/dev-utils'; import { discoveryApiRef, identityApiRef } from '@backstage/core'; import { CatalogClient } from '@backstage/catalog-client'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; -import { TemplateIndexPage, TemplatePage } from '../src/plugin'; -import { ScaffolderApi, scaffolderApiRef } from '../src'; +import { ScaffolderPage } from '../src/plugin'; +import { ScaffolderClient, scaffolderApiRef } from '../src'; createDevApp() .registerApi({ @@ -32,15 +32,11 @@ createDevApp() api: scaffolderApiRef, deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, factory: ({ discoveryApi, identityApi }) => - new ScaffolderApi({ discoveryApi, identityApi }), + new ScaffolderClient({ discoveryApi, identityApi }), }) .addPage({ path: '/create', title: 'Create', - element: , - }) - .addPage({ - path: '/create/:templateName', - element: , + element: , }) .render(); diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 13c9803667..c5b264154d 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -30,7 +30,9 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@backstage/catalog-client": "^0.3.6", "@backstage/catalog-model": "^0.7.1", + "@backstage/config": "^0.1.2", "@backstage/core": "^0.6.2", "@backstage/plugin-catalog-react": "^0.0.4", "@backstage/theme": "^0.2.3", @@ -41,6 +43,8 @@ "@rjsf/material-ui": "^2.4.0", "classnames": "^2.2.6", "git-url-parse": "^11.4.4", + "humanize-duration": "^3.25.1", + "luxon": "^1.25.0", "moment": "^2.26.0", "react": "^16.13.1", "react-dom": "^16.13.1", @@ -48,16 +52,18 @@ "react-router": "6.0.0-beta.0", "react-router-dom": "6.0.0-beta.0", "react-use": "^15.3.3", - "swr": "^0.3.0" + "swr": "^0.3.0", + "use-immer": "^0.4.2", + "zen-observable": "^0.8.15" }, "devDependencies": { "@backstage/cli": "^0.6.1", "@backstage/dev-utils": "^0.1.11", "@backstage/test-utils": "^0.1.7", - "@backstage/catalog-client": "^0.3.6", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^10.4.1", "@testing-library/user-event": "^12.0.7", + "@types/humanize-duration": "^3.18.1", "@testing-library/react-hooks": "^3.3.0", "@types/jest": "^26.0.7", "@types/node": "^12.0.0", diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 453b1b6a7e..20fca5b3f2 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -14,14 +14,53 @@ * limitations under the License. */ -import { createApiRef, DiscoveryApi, IdentityApi } from '@backstage/core'; +import { + createApiRef, + DiscoveryApi, + Observable, + IdentityApi, +} from '@backstage/core'; +import ObservableImpl from 'zen-observable'; +import { ScaffolderTask, Status } from './types'; export const scaffolderApiRef = createApiRef({ id: 'plugin.scaffolder.service', description: 'Used to make requests towards the scaffolder backend', }); -export class ScaffolderApi { +export type LogEvent = { + type: 'log' | 'completion'; + body: { + message: string; + stepId?: string; + status?: Status; + }; + createdAt: string; + id: string; + taskId: string; +}; + +export interface ScaffolderApi { + /** + * Executes the scaffolding of a component, given a template and its + * parameter values. + * + * @param templateName Name of the Template entity for the scaffolder to use. New project is going to be created out of this template. + * @param values Parameters for the template, e.g. name, description + */ + scaffold(templateName: string, values: Record): Promise; + + getTask(taskId: string): Promise; + + streamLogs({ + taskId, + after, + }: { + taskId: string; + after?: number; + }): Observable; +} +export class ScaffolderClient implements ScaffolderApi { private readonly discoveryApi: DiscoveryApi; private readonly identityApi: IdentityApi; @@ -40,9 +79,12 @@ export class ScaffolderApi { * @param templateName Template name for the scaffolder to use. New project is going to be created out of this template. * @param values Parameters for the template, e.g. name, description */ - async scaffold(templateName: string, values: Record) { + async scaffold( + templateName: string, + values: Record, + ): Promise { const token = await this.identityApi.getIdToken(); - const url = `${await this.discoveryApi.getBaseUrl('scaffolder')}/v1/jobs`; + const url = `${await this.discoveryApi.getBaseUrl('scaffolder')}/v2/tasks`; const response = await fetch(url, { method: 'POST', headers: { @@ -58,16 +100,65 @@ export class ScaffolderApi { throw new Error(`Backend request failed, ${status} ${body.trim()}`); } - const { id } = await response.json(); + const { id } = (await response.json()) as { id: string }; return id; } - async getJob(jobId: string) { + async getTask(taskId: string) { const token = await this.identityApi.getIdToken(); const baseUrl = await this.discoveryApi.getBaseUrl('scaffolder'); - const url = `${baseUrl}/v1/job/${encodeURIComponent(jobId)}`; + const url = `${baseUrl}/v2/tasks/${encodeURIComponent(taskId)}`; return fetch(url, { headers: token ? { Authorization: `Bearer ${token}` } : {}, }).then(x => x.json()); } + + streamLogs({ + taskId, + after, + }: { + taskId: string; + after?: number; + }): Observable { + return new ObservableImpl(subscriber => { + const params = new URLSearchParams(); + if (after !== undefined) { + params.set('after', String(Number(after))); + } + + this.discoveryApi.getBaseUrl('scaffolder').then( + baseUrl => { + const url = `${baseUrl}/v2/tasks/${encodeURIComponent( + taskId, + )}/eventstream`; + const eventSource = new EventSource(url); + eventSource.addEventListener('log', (event: any) => { + if (event.data) { + try { + subscriber.next(JSON.parse(event.data)); + } catch (ex) { + subscriber.error(ex); + } + } + }); + eventSource.addEventListener('completion', (event: any) => { + if (event.data) { + try { + subscriber.next(JSON.parse(event.data)); + } catch (ex) { + subscriber.error(ex); + } + } + subscriber.complete(); + }); + eventSource.addEventListener('error', event => { + subscriber.error(event); + }); + }, + error => { + subscriber.error(error); + }, + ); + }); + } } diff --git a/plugins/scaffolder/src/components/JobStage/JobStage.tsx b/plugins/scaffolder/src/components/JobStage/JobStage.tsx deleted file mode 100644 index f285cb98c0..0000000000 --- a/plugins/scaffolder/src/components/JobStage/JobStage.tsx +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - Accordion, - AccordionDetails, - AccordionSummary, - AccordionActions, - Box, - CircularProgress, - LinearProgress, - Typography, - Button, -} from '@material-ui/core'; -import { makeStyles } from '@material-ui/core/styles'; -import ExpandMoreIcon from '@material-ui/icons/ExpandMore'; -import ExpandLessIcon from '@material-ui/icons/ExpandLess'; -import cn from 'classnames'; -import moment from 'moment'; -import React, { Suspense, useEffect, useState } from 'react'; -import { LogModal } from './LogModal'; -import { Job } from '../../types'; - -const LazyLog = React.lazy(() => import('react-lazylog/build/LazyLog')); -moment.relativeTimeThreshold('ss', 0); - -const useStyles = makeStyles(theme => ({ - accordionDetails: { - padding: 0, - }, - button: { - order: -1, - margin: '0 1em 0 -20px', - }, - cardContent: { - backgroundColor: theme.palette.background.default, - }, - accordion: { - position: 'relative', - '&:after': { - pointerEvents: 'none', - content: '""', - position: 'absolute', - top: 0, - right: 0, - left: 0, - bottom: 0, - }, - }, - neutral: {}, - failed: { - '&:after': { - boxShadow: `inset 4px 0px 0px ${theme.palette.error.main}`, - }, - }, - started: { - '&:after': { - boxShadow: `inset 4px 0px 0px ${theme.palette.info.main}`, - }, - }, - completed: { - '&:after': { - boxShadow: `inset 4px 0px 0px ${theme.palette.success.main}`, - }, - }, - jobStatusTitle: { - display: 'flex', - width: '100%', - alignItems: 'center', - flexDirection: 'row', - justifyContent: 'space-between', - [theme.breakpoints.down('xs')]: { - flexDirection: 'column', - alignItems: 'flex-start', - justifyContent: 'flex-start', - }, - }, -})); - -type Props = { - name: string; - className?: string; - log: string[]; - startedAt: string; - endedAt?: string; - status: Job['status']; -}; - -export const JobStage = ({ endedAt, startedAt, name, log, status }: Props) => { - const classes = useStyles(); - - const [expanded, setExpanded] = useState(false); - useEffect(() => { - if (status === 'FAILED') setExpanded(true); - }, [status, setExpanded]); - - const timeElapsed = - status !== 'PENDING' - ? moment - .duration(moment(endedAt ?? moment()).diff(moment(startedAt))) - .humanize() - : null; - - const [logsFullScreen, setLogsFullScreen] = useState(false); - const toggleLogsFullScreen = () => setLogsFullScreen(!logsFullScreen); - - return ( - ] ?? - classes.neutral, - )} - expanded={expanded} - onChange={(_, newState) => setExpanded(newState)} - > - : } - aria-controls={`panel-${name}-content`} - id={`panel-${name}-header`} - IconButtonProps={{ - className: classes.button, - }} - > - - {name} {timeElapsed && `(${timeElapsed})`}{' '} - {startedAt && !endedAt && } - - - - {log.length === 0 ? ( - - No logs available for this step - - ) : ( - }> - -
- -
-
- )} -
- - - -
- ); -}; diff --git a/plugins/scaffolder/src/components/JobStage/LogModal.tsx b/plugins/scaffolder/src/components/JobStage/LogModal.tsx deleted file mode 100644 index 73b6253510..0000000000 --- a/plugins/scaffolder/src/components/JobStage/LogModal.tsx +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React from 'react'; -import { - Dialog, - DialogTitle, - DialogContent, - IconButton, -} from '@material-ui/core'; -import { makeStyles } from '@material-ui/core/styles'; -import Close from '@material-ui/icons/Close'; -import LazyLog from 'react-lazylog/build/LazyLog'; - -type Props = { - log: string[]; - open?: boolean; - onClose(): void; -}; - -const useStyles = makeStyles(theme => ({ - header: { - width: '100%', - padding: theme.spacing(1, 4), - }, - closeIcon: { - float: 'right', - padding: theme.spacing(0.5, 0), - }, - logs: { - boxShadow: '-3px -1px 7px 0px rgba(50, 50, 50, 0.59)', - height: '100%', - width: '100%', - }, -})); - -export const LogModal = ({ log, open = false, onClose }: Props) => { - const classes = useStyles(); - - return ( - - - Logs - - - - - -
- -
-
-
- ); -}; diff --git a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx b/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx deleted file mode 100644 index d35385ec3b..0000000000 --- a/plugins/scaffolder/src/components/JobStatusModal/JobStatusModal.tsx +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { Button } from '@backstage/core'; -import { - Button as Action, - Dialog, - DialogActions, - DialogContent, - DialogTitle, - LinearProgress, -} from '@material-ui/core'; - -import React, { useCallback } from 'react'; -import { Job } from '../../types'; -import { JobStage } from '../JobStage/JobStage'; - -type Props = { - job: Job | null; - toCatalogLink?: string; - open: boolean; - onModalClose: () => void; -}; - -export const JobStatusModal = ({ - job, - toCatalogLink, - open, - onModalClose, -}: Props) => { - const renderTitle = () => { - switch (job?.status) { - case 'COMPLETED': - return 'Successfully created component'; - case 'FAILED': - return 'Failed to create component'; - default: - return 'Create component'; - } - }; - - const onClose = useCallback(() => { - if (!job) { - return; - } - // Disallow closing modal if the job is in progress. - if (job.status === 'COMPLETED' || job.status === 'FAILED') { - onModalClose(); - } - }, [job, onModalClose]); - - return ( - - {renderTitle()} - - {!job ? ( - - ) : ( - (job?.stages ?? []).map(step => ( - - )) - )} - - {job?.status && toCatalogLink && ( - - - - )} - {job?.status === 'FAILED' && ( - - Close - - )} - - ); -}; diff --git a/plugins/scaffolder/src/components/Router.tsx b/plugins/scaffolder/src/components/Router.tsx new file mode 100644 index 0000000000..12d4ecce82 --- /dev/null +++ b/plugins/scaffolder/src/components/Router.tsx @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { Routes, Route } from 'react-router'; +import { ScaffolderPage } from './ScaffolderPage'; +import { TemplatePage } from './TemplatePage'; +import { TaskPage } from './TaskPage'; + +export const Router = () => ( + + } /> + } /> + } /> + +); diff --git a/plugins/scaffolder/src/components/TaskPage/TaskPage.tsx b/plugins/scaffolder/src/components/TaskPage/TaskPage.tsx new file mode 100644 index 0000000000..96a857dbb5 --- /dev/null +++ b/plugins/scaffolder/src/components/TaskPage/TaskPage.tsx @@ -0,0 +1,338 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Page, Header, Lifecycle, Content, ErrorPage } from '@backstage/core'; +import React, { useState, useEffect, memo, useMemo } from 'react'; +import { makeStyles, Theme, createStyles } from '@material-ui/core/styles'; +import Stepper from '@material-ui/core/Stepper'; +import Step from '@material-ui/core/Step'; +import StepLabel from '@material-ui/core/StepLabel'; +import Grid from '@material-ui/core/Grid'; +import Typography from '@material-ui/core/Typography'; +import { generatePath, useParams } from 'react-router'; +import { useTaskEventStream } from '../hooks/useEventStream'; +import LazyLog from 'react-lazylog/build/LazyLog'; +import { Link } from 'react-router-dom'; +import { + Box, + Button, + CircularProgress, + Paper, + StepButton, + StepIconProps, +} from '@material-ui/core'; +import { Status } from '../../types'; +import { DateTime, Interval } from 'luxon'; +import { useInterval } from 'react-use'; +import Check from '@material-ui/icons/Check'; +import Cancel from '@material-ui/icons/Cancel'; +import FiberManualRecordIcon from '@material-ui/icons/FiberManualRecord'; +import { entityRoute } from '@backstage/plugin-catalog-react'; +import { parseEntityName } from '@backstage/catalog-model'; +import classNames from 'classnames'; +import { BackstageTheme } from '@backstage/theme'; + +// typings are wrong for this library, so fallback to not parsing types. +const humanizeDuration = require('humanize-duration'); + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + root: { + width: '100%', + }, + button: { + marginTop: theme.spacing(1), + marginRight: theme.spacing(1), + }, + actionsContainer: { + marginBottom: theme.spacing(2), + }, + resetContainer: { + padding: theme.spacing(3), + }, + labelWrapper: { + display: 'flex', + flex: 1, + flexDirection: 'row', + justifyContent: 'space-between', + }, + stepWrapper: { + width: '100%', + }, + }), +); + +type TaskStep = { + id: string; + name: string; + status: Status; + startedAt?: string; + endedAt?: string; +}; + +const StepTimeTicker = ({ step }: { step: TaskStep }) => { + const [time, setTime] = useState(''); + + useInterval(() => { + if (!step.startedAt) { + setTime(''); + return; + } + + const end = step.endedAt + ? DateTime.fromISO(step.endedAt) + : DateTime.local(); + + const startedAt = DateTime.fromISO(step.startedAt); + const formatted = Interval.fromDateTimes(startedAt, end) + .toDuration() + .valueOf(); + + setTime(humanizeDuration(formatted, { round: true })); + }, 1000); + + return {time}; +}; + +const useStepIconStyles = makeStyles((theme: BackstageTheme) => + createStyles({ + root: { + color: theme.palette.text.disabled, + display: 'flex', + height: 22, + alignItems: 'center', + }, + completed: { + color: theme.palette.status.ok, + }, + error: { + color: theme.palette.status.error, + }, + }), +); + +function TaskStepIconComponent(props: StepIconProps) { + const classes = useStepIconStyles(); + const { active, completed, error } = props; + + const getMiddle = () => { + if (active) { + return ; + } + if (completed) { + return ; + } + if (error) { + return ; + } + return ; + }; + + return ( +
+ {getMiddle()} +
+ ); +} + +export const TaskStatusStepper = memo( + ({ + steps, + currentStepId, + onUserStepChange, + }: { + steps: TaskStep[]; + currentStepId: string | undefined; + onUserStepChange: (id: string) => void; + }) => { + const classes = useStyles(); + + return ( +
+ s.id === currentStepId)} + orientation="vertical" + nonLinear + > + {steps.map((step, index) => { + const isCompleted = step.status === 'completed'; + const isFailed = step.status === 'failed'; + const isActive = step.status === 'processing'; + return ( + + onUserStepChange(step.id)}> + +
+ {step.name} + +
+
+
+
+ ); + })} +
+
+ ); + }, +); + +const TaskLogger = memo(({ log }: { log: string }) => { + return ( +
+ +
+ ); +}); + +export const TaskPage = () => { + const [userSelectedStepId, setUserSelectedStepId] = useState< + string | undefined + >(undefined); + const [lastActiveStepId, setLastActiveStepId] = useState( + undefined, + ); + const { taskId } = useParams(); + const taskStream = useTaskEventStream(taskId); + const completed = taskStream.completed; + const steps = useMemo( + () => + taskStream.task?.spec.steps.map(step => ({ + ...step, + ...taskStream?.steps?.[step.id], + })) ?? [], + [taskStream], + ); + + useEffect(() => { + const mostRecentFailedOrActiveStep = steps.find(step => + ['failed', 'processing'].includes(step.status), + ); + if (completed && !mostRecentFailedOrActiveStep) { + setLastActiveStepId(steps[steps.length - 1]?.id); + return; + } + + setLastActiveStepId(mostRecentFailedOrActiveStep?.id); + }, [steps, completed]); + + const currentStepId = userSelectedStepId ?? lastActiveStepId; + + const logAsString = useMemo(() => { + if (!currentStepId) { + return 'Loading...'; + } + const log = taskStream.stepLogs[currentStepId]; + + if (!log?.length) { + return 'Waiting for logs...'; + } + return log.join('\n'); + }, [taskStream.stepLogs, currentStepId]); + + const taskNotFound = + taskStream.completed === true && + taskStream.loading === false && + !taskStream.task; + + const entityRef = taskStream.output?.entityRef; + const remoteUrl = taskStream.output?.remoteUrl; + return ( + +
+ Task Activity + + } + subtitle={`Activity for task: ${taskId}`} + /> + + {taskNotFound ? ( + + ) : ( +
+ + + + + {(entityRef || remoteUrl) && ( + + {entityRef && ( + + )} + {remoteUrl && ( + + )} + + )} + + + + + + +
+ )} +
+ + ); +}; diff --git a/plugins/scaffolder/src/components/JobStage/index.ts b/plugins/scaffolder/src/components/TaskPage/index.ts similarity index 89% rename from plugins/scaffolder/src/components/JobStage/index.ts rename to plugins/scaffolder/src/components/TaskPage/index.ts index d6d3534a88..3695c2792e 100644 --- a/plugins/scaffolder/src/components/JobStage/index.ts +++ b/plugins/scaffolder/src/components/TaskPage/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { JobStage } from './JobStage'; +export { TaskPage } from './TaskPage'; diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index ef17335b26..c38f4e09a5 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button } from '@backstage/core'; +import { Button, useRouteRef } from '@backstage/core'; import { BackstageTheme, pageTheme } from '@backstage/theme'; import { Card, @@ -25,8 +25,8 @@ import { useTheme, } from '@material-ui/core'; import React from 'react'; -import { generatePath } from 'react-router-dom'; -import { templateRoute } from '../../routes'; +import { generatePath } from 'react-router'; +import { rootRouteRef } from '../../routes'; const useStyles = makeStyles(theme => ({ header: { @@ -59,11 +59,14 @@ export const TemplateCard = ({ name, }: TemplateCardProps) => { const backstageTheme = useTheme(); + const rootLink = useRouteRef(rootRouteRef); const themeId = pageTheme[type] ? type : 'other'; const theme = backstageTheme.getPageTheme({ themeId }); const classes = useStyles({ backgroundImage: theme.backgroundImage }); - const href = generatePath(templateRoute.path, { templateName: name }); + const href = generatePath(`${rootLink()}/templates/:templateName`, { + templateName: name, + }); return ( diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx index 09df6dba1a..f0c77f2f03 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.test.tsx @@ -22,7 +22,7 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { MemoryRouter, Route } from 'react-router'; import { ScaffolderApi, scaffolderApiRef } from '../../api'; -import { rootRoute } from '../../routes'; +import { rootRouteRef } from '../../routes'; import { TemplatePage } from './TemplatePage'; const templateMock = { @@ -97,11 +97,15 @@ describe('TemplatePage', () => { , + { + mountedRoutes: { + '/create': rootRouteRef, + }, + }, ); expect(rendered.queryByText('Create a New Component')).toBeInTheDocument(); expect(rendered.queryByText('React SSR Template')).toBeInTheDocument(); - // await act(async () => await mutate('templates/test')); }); it('renders spinner while loading', async () => { @@ -114,13 +118,18 @@ describe('TemplatePage', () => { , + { + mountedRoutes: { + '/create': rootRouteRef, + }, + }, ); expect(rendered.queryByText('Create a New Component')).toBeInTheDocument(); expect(rendered.queryByTestId('loading-progress')).toBeInTheDocument(); - // Need to cleanup the promise or will timeout - act(() => { - resolve!({ items: [] }); + + await act(async () => { + resolve!({ items: [templateMock] }); }); }); @@ -134,7 +143,7 @@ describe('TemplatePage', () => { - This is root} /> + This is root} /> , diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index d779f8b42c..0fc1a3cf5d 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -22,23 +22,18 @@ import { Lifecycle, Page, useApi, + useRouteRef, } from '@backstage/core'; -import { - catalogApiRef, - entityRoute, - entityRouteParams, -} from '@backstage/plugin-catalog-react'; +import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { LinearProgress } from '@material-ui/core'; import { IChangeEvent } from '@rjsf/core'; import parseGitUrl from 'git-url-parse'; import React, { useCallback, useState } from 'react'; -import { generatePath, Navigate } from 'react-router'; +import { generatePath, useNavigate, Navigate } from 'react-router'; import { useParams } from 'react-router-dom'; import { useAsync } from 'react-use'; import { scaffolderApiRef } from '../../api'; -import { rootRoute } from '../../routes'; -import { useJobPolling } from '../hooks/useJobPolling'; -import { JobStatusModal } from '../JobStatusModal'; +import { rootRouteRef } from '../../routes'; import { MultistepJsonForm } from '../MultistepJsonForm'; const useTemplate = ( @@ -50,7 +45,7 @@ const useTemplate = ( filter: { kind: 'Template', 'metadata.name': templateName }, }); return response.items as TemplateEntityV1alpha1[]; - }); + }, [catalogApi, templateName]); return { template: value?.[0], loading, error }; }; @@ -76,57 +71,28 @@ const OWNER_REPO_SCHEMA = { }, }, }; + export const TemplatePage = () => { const errorApi = useApi(errorApiRef); const catalogApi = useApi(catalogApiRef); const scaffolderApi = useApi(scaffolderApiRef); const { templateName } = useParams(); - const [catalogLink, setCatalogLink] = useState(); + const navigate = useNavigate(); + const rootLink = useRouteRef(rootRouteRef); const { template, loading } = useTemplate(templateName, catalogApi); const [formState, setFormState] = useState({}); - const [modalOpen, setModalOpen] = useState(false); const handleFormReset = () => setFormState({}); + const handleChange = useCallback( (e: IChangeEvent) => setFormState({ ...formState, ...e.formData }), [setFormState, formState], ); - const [jobId, setJobId] = useState(null); - const job = useJobPolling(jobId, async jobItem => { - if (!jobItem.metadata.catalogInfoUrl) { - errorApi.post( - new Error(`No catalogInfoUrl returned from the scaffolder`), - ); - return; - } - - try { - const { - entities: [createdEntity], - } = await catalogApi.addLocation({ - target: jobItem.metadata.catalogInfoUrl, - }); - - const resolvedPath = generatePath( - `/catalog/${entityRoute.path}`, - entityRouteParams(createdEntity), - ); - - setCatalogLink(resolvedPath); - } catch (ex) { - errorApi.post( - new Error( - `Something went wrong trying to add the new 'catalog-info.yaml' to the catalog`, - ), - ); - } - }); - const handleCreate = async () => { try { const id = await scaffolderApi.scaffold(templateName, formState); - setJobId(id); - setModalOpen(true); + + navigate(generatePath(`${rootLink()}/tasks/:taskId`, { taskId: id })); } catch (e) { errorApi.post(e); } @@ -134,7 +100,7 @@ export const TemplatePage = () => { if (!loading && !template) { errorApi.post(new Error('Template was not found.')); - return ; + return ; } if (template && !template?.spec?.schema) { @@ -143,7 +109,7 @@ export const TemplatePage = () => { 'Template schema is corrupted, please check the template.yaml file.', ), ); - return ; + return ; } return ( @@ -159,12 +125,6 @@ export const TemplatePage = () => { /> {loading && } - setModalOpen(false)} - /> {template && ( { + current[next.id] = { status: 'open', id: next.id }; + return current; + }, {} as { [stepId in string]: Step }); + draft.stepLogs = action.data.spec.steps.reduce((current, next) => { + current[next.id] = []; + return current; + }, {} as { [stepId in string]: string[] }); + draft.loading = false; + draft.error = undefined; + draft.completed = false; + draft.task = action.data; + return; + } + + case 'LOGS': { + const entries = action.data; + const logLines = []; + + for (const entry of entries) { + const logLine = `${entry.createdAt} ${entry.body.message}`; + logLines.push(logLine); + + if (!entry.body.stepId || !draft.steps?.[entry.body.stepId]) { + continue; + } + + const currentStepLog = draft.stepLogs?.[entry.body.stepId]; + const currentStep = draft.steps?.[entry.body.stepId]; + + if (entry.body.status && entry.body.status !== currentStep.status) { + currentStep.status = entry.body.status; + + if (currentStep.status === 'processing') { + currentStep.startedAt = entry.createdAt; + } + + if ( + ['cancelled', 'failed', 'completed'].includes(currentStep.status) + ) { + currentStep.endedAt = entry.createdAt; + } + } + + currentStepLog?.push(logLine); + } + + return; + } + + case 'COMPLETED': { + draft.completed = true; + draft.output = action.data.body.output; + return; + } + + case 'ERROR': { + draft.error = action.data; + draft.loading = false; + draft.completed = true; + return; + } + + default: + return; + } +} + +export const useTaskEventStream = (taskId: string): TaskStream => { + const scaffolderApi = useApi(scaffolderApiRef); + const [state, dispatch] = useImmerReducer(reducer, { + loading: true, + completed: false, + stepLogs: {} as { [stepId in string]: string[] }, + steps: {} as { [stepId in string]: Step }, + }); + + useEffect(() => { + let didCancel = false; + let subscription: Subscription | undefined; + let logPusher: NodeJS.Timeout | undefined; + + scaffolderApi.getTask(taskId).then( + task => { + if (didCancel) { + return; + } + dispatch({ type: 'INIT', data: task }); + + // TODO(blam): Use a normal fetch to fetch the current log for the event stream + // and use that for an INIT_EVENTs dispatch event, and then + // use the last event ID to subscribe using after option to + // stream logs. Without this, if you have a lot of logs, it can look like the + // task is being rebuilt on load as it progresses through the steps at a slower + // rate whilst it builds the status from the event logs + const observable = scaffolderApi.streamLogs({ taskId }); + + const collectedLogEvents = new Array(); + + function emitLogs() { + if (collectedLogEvents.length) { + const logs = collectedLogEvents.splice( + 0, + collectedLogEvents.length, + ); + dispatch({ type: 'LOGS', data: logs }); + } + } + + logPusher = setInterval(emitLogs, 500); + + subscription = observable.subscribe({ + next: event => { + switch (event.type) { + case 'log': + return collectedLogEvents.push(event); + case 'completion': + emitLogs(); + dispatch({ type: 'COMPLETED', data: event }); + return undefined; + default: + throw new Error( + `Unhandled event type ${event.type} in observer`, + ); + } + }, + error: error => { + emitLogs(); + dispatch({ type: 'ERROR', data: error }); + }, + }); + }, + error => { + if (!didCancel) { + dispatch({ type: 'ERROR', data: error }); + } + }, + ); + + return () => { + didCancel = true; + if (subscription) { + subscription.unsubscribe(); + } + if (logPusher) { + clearInterval(logPusher); + } + }; + }, [scaffolderApi, dispatch, taskId]); + + return state; +}; diff --git a/plugins/scaffolder/src/components/hooks/useJobPolling.ts b/plugins/scaffolder/src/components/hooks/useJobPolling.ts deleted file mode 100644 index 2cd4fbdc9f..0000000000 --- a/plugins/scaffolder/src/components/hooks/useJobPolling.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { useEffect, useState } from 'react'; -import { Job } from '../../types'; -import { useApi } from '@backstage/core'; -import { scaffolderApiRef } from '../../api'; -import { useInterval } from 'react-use'; - -const DEFAULT_POLLING_INTERVAL = 1000; - -export const useJobPolling = ( - jobId: string | null, - onFinish?: (j: Job) => void, - pollingInterval = DEFAULT_POLLING_INTERVAL, -) => { - const scaffolderApi = useApi(scaffolderApiRef); - const [currentJob, setCurrentJob] = useState(null); - - useEffect(() => { - const resetCurrentJob = async () => { - if (jobId) { - const job = await scaffolderApi.getJob(jobId); - setCurrentJob(job); - } - }; - - resetCurrentJob(); - }, [jobId, scaffolderApi]); - - const shouldBeRunningInterval = - jobId && - currentJob?.status !== 'COMPLETED' && - currentJob?.status !== 'FAILED'; - - useInterval( - async () => { - if (jobId) { - const job = await scaffolderApi.getJob(jobId); - if (job?.status === 'COMPLETED' || job?.status === 'FAILED') { - onFinish?.(job); - } - setCurrentJob(job); - } - }, - shouldBeRunningInterval ? pollingInterval : null, - ); - - return currentJob; -}; diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 5f102e7853..e0b574e6c6 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -17,8 +17,7 @@ export { scaffolderPlugin, scaffolderPlugin as plugin, - TemplateIndexPage, - TemplatePage, + ScaffolderPage, } from './plugin'; -export { ScaffolderApi, scaffolderApiRef } from './api'; -export { rootRoute, templateRoute } from './routes'; +export type { ScaffolderApi } from './api'; +export { ScaffolderClient, scaffolderApiRef } from './api'; diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index 3e884d7683..a6ba1a9899 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -21,10 +21,8 @@ import { identityApiRef, createRoutableExtension, } from '@backstage/core'; -import { ScaffolderPage as ScaffolderPageComponent } from './components/ScaffolderPage'; -import { TemplatePage as TemplatePageComponent } from './components/TemplatePage'; -import { rootRoute, templateRoute } from './routes'; -import { scaffolderApiRef, ScaffolderApi } from './api'; +import { rootRouteRef } from './routes'; +import { scaffolderApiRef, ScaffolderClient } from './api'; export const scaffolderPlugin = createPlugin({ id: 'scaffolder', @@ -33,31 +31,17 @@ export const scaffolderPlugin = createPlugin({ api: scaffolderApiRef, deps: { discoveryApi: discoveryApiRef, identityApi: identityApiRef }, factory: ({ discoveryApi, identityApi }) => - new ScaffolderApi({ discoveryApi, identityApi }), + new ScaffolderClient({ discoveryApi, identityApi }), }), ], - register({ router }) { - router.addRoute(rootRoute, ScaffolderPageComponent); - router.addRoute(templateRoute, TemplatePageComponent); - }, routes: { - templateIndex: rootRoute, - template: templateRoute, + root: rootRouteRef, }, }); -export const TemplateIndexPage = scaffolderPlugin.provide( +export const ScaffolderPage = scaffolderPlugin.provide( createRoutableExtension({ - component: () => - import('./components/ScaffolderPage').then(m => m.ScaffolderPage), - mountPoint: rootRoute, - }), -); - -export const TemplatePage = scaffolderPlugin.provide( - createRoutableExtension({ - component: () => - import('./components/TemplatePage').then(m => m.TemplatePage), - mountPoint: templateRoute, + component: () => import('./components/Router').then(m => m.Router), + mountPoint: rootRouteRef, }), ); diff --git a/plugins/scaffolder/src/routes.ts b/plugins/scaffolder/src/routes.ts index 8efd1d3aaf..413e8f4194 100644 --- a/plugins/scaffolder/src/routes.ts +++ b/plugins/scaffolder/src/routes.ts @@ -15,12 +15,6 @@ */ import { createRouteRef } from '@backstage/core'; -export const rootRoute = createRouteRef({ - path: '/create', +export const rootRouteRef = createRouteRef({ title: 'Create new entity', }); - -export const templateRoute = createRouteRef({ - path: '/create/:templateName', - title: 'Entity creation', -}); diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 45672c603d..0499f4aca2 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { JsonValue } from '@backstage/config'; +export type Status = 'open' | 'processing' | 'failed' | 'completed'; export type JobStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; export type Job = { id: string; @@ -35,3 +37,20 @@ export type Stage = { startedAt: string; endedAt?: string; }; + +export type ScaffolderStep = { + id: string; + name: string; + action: string; + parameters?: { [name: string]: JsonValue }; +}; + +export type ScaffolderTask = { + id: string; + spec: { + steps: ScaffolderStep[]; + }; + status: 'failed' | 'completed' | 'processing' | 'open' | 'cancelled'; + lastHeartbeatAt: string; + createdAt: string; +}; diff --git a/yarn.lock b/yarn.lock index b0a74498c5..eecc5051df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6117,6 +6117,11 @@ dependencies: "@types/node" "*" +"@types/humanize-duration@^3.18.1": + version "3.18.1" + resolved "https://registry.npmjs.org/@types/humanize-duration/-/humanize-duration-3.18.1.tgz#10090d596053703e7de0ac43a37b96cd9fc78309" + integrity sha512-MUgbY3CF7hg/a/jogixmAufLjJBQT7WEf8Q+kYJkOc47ytngg1IuZobCngdTjAgY83JWEogippge5O5fplaQlw== + "@types/inquirer@^7.3.1": version "7.3.1" resolved "https://registry.npmjs.org/@types/inquirer/-/inquirer-7.3.1.tgz#1f231224e7df11ccfaf4cf9acbcc3b935fea292d" @@ -14716,6 +14721,11 @@ human-signals@^2.1.0: resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +humanize-duration@^3.25.1: + version "3.25.1" + resolved "https://registry.npmjs.org/humanize-duration/-/humanize-duration-3.25.1.tgz#50e12bf4b3f515ec91106107ee981e8cfe955d6f" + integrity sha512-P+dRo48gpLgc2R9tMRgiDRNULPKCmqFYgguwqOO2C0fjO35TgdURDQDANSR1Nt92iHlbHGMxOTnsB8H8xnMa2Q== + humanize-ms@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" @@ -17516,6 +17526,11 @@ luxon@1.25.0, luxon@^1.25.0: resolved "https://registry.npmjs.org/luxon/-/luxon-1.25.0.tgz#d86219e90bc0102c0eb299d65b2f5e95efe1fe72" integrity sha512-hEgLurSH8kQRjY6i4YLey+mcKVAWXbDNlZRmM6AgWDJ1cY3atl8Ztf5wEY7VBReFbmGnwQPz7KYJblL8B2k0jQ== +luxon@^1.26.0: + version "1.26.0" + resolved "https://registry.npmjs.org/luxon/-/luxon-1.26.0.tgz#d3692361fda51473948252061d0f8561df02b578" + integrity sha512-+V5QIQ5f6CDXQpWNICELwjwuHdqeJM1UenlZWx5ujcRMc9venvluCjFb4t5NYLhb6IhkbMVOxzVuOqkgMxee2A== + macos-release@^2.2.0: version "2.3.0" resolved "https://registry.npmjs.org/macos-release/-/macos-release-2.3.0.tgz#eb1930b036c0800adebccd5f17bc4c12de8bb71f" @@ -25357,6 +25372,11 @@ use-composed-ref@^1.0.0: dependencies: ts-essentials "^2.0.3" +use-immer@^0.4.2: + version "0.4.2" + resolved "https://registry.npmjs.org/use-immer/-/use-immer-0.4.2.tgz#7d7e7d3386cc834d45b2279f37d8974023550f4f" + integrity sha512-ONfZHEv/gzt/jyYxrJD3ZFUllKJED8F1mds9Fr9CYj54LmsiuGDg3vkx+R7WHS72p7DbSS1QbM7xNYrVWX+KcA== + use-isomorphic-layout-effect@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.0.0.tgz#f56b4ed633e1c21cd9fc76fe249002a1c28989fb"