diff --git a/.changeset/dry-spies-cover.md b/.changeset/dry-spies-cover.md index 7006dee5c4..aae683faaf 100644 --- a/.changeset/dry-spies-cover.md +++ b/.changeset/dry-spies-cover.md @@ -3,4 +3,6 @@ '@backstage/theme': patch --- -Will Add warning variant to `DismissableBanner` component. +Added a warning variant to `DismissableBanner` component. If you are using a +custom theme, you will need to add the optional `palette.banner.warning` color, +otherwise this variant will fall back to the `palette.banner.error` color. diff --git a/.changeset/selfish-countries-prove.md b/.changeset/selfish-countries-prove.md new file mode 100644 index 0000000000..03e36f6d80 --- /dev/null +++ b/.changeset/selfish-countries-prove.md @@ -0,0 +1,42 @@ +--- +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Adds a new endpoint for consuming logs from the Scaffolder that uses long polling instead of Server Sent Events. + +This is useful if Backstage is accessed from an environment that doesn't support SSE correctly, which happens in combination with certain enterprise HTTP Proxy servers. + +It is intended to switch the endpoint globally for the whole instance. +If you want to use it, you can provide a reconfigured API to the `scaffolderApiRef`: + +```tsx +// packages/app/src/apis.ts + +// ... +import { + scaffolderApiRef, + ScaffolderClient, +} from '@backstage/plugin-scaffolder'; + +export const apis: AnyApiFactory[] = [ + // ... + + createApiFactory({ + api: scaffolderApiRef, + deps: { + discoveryApi: discoveryApiRef, + identityApi: identityApiRef, + scmIntegrationsApi: scmIntegrationsApiRef, + }, + factory: ({ discoveryApi, identityApi, scmIntegrationsApi }) => + new ScaffolderClient({ + discoveryApi, + identityApi, + scmIntegrationsApi, + // use long polling instead of an eventsource + useLongPollingLogs: true, + }), + }), +]; +``` diff --git a/.changeset/spicy-panthers-thank.md b/.changeset/spicy-panthers-thank.md new file mode 100644 index 0000000000..6c28b002f4 --- /dev/null +++ b/.changeset/spicy-panthers-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': patch +--- + +I have added default icons for the catalog, scaffolder, techdocs, and search. diff --git a/ADOPTERS.md b/ADOPTERS.md index 797d178270..06501a2a10 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -62,3 +62,4 @@ | [Adevinta](https://www.adevinta.com) | [Ray Sinnema](https://github.com/RemonSinnema) | Showcase shared services to our internal customers. | | [Splunk](https://www.splunk.com) | [@tonytamsf](https://github.com/tonytamsf) | Developer portal as a centralized place to find people, services, documentation, escalation policies and give bravos. This portal is also being used as a centralized search engine for engineering specific documentation. | | [SoundCloud](https://www.soundcloud.com) | [Julio Zynger](https://github.com/julioz) | Developer portal as a [humane registry](https://martinfowler.com/bliki/HumaneRegistry.html) for the organization: catalog of people, services, documentation, feature toggles, escalation policies, etc. | +| [Volvofinans Bank](https://www.volvofinans.se) | [Johan Hammar](https://github.com/johanhammar) | Developer portal enabling engineers to manage and explore software and documentation. | diff --git a/packages/core-app-api/src/app/icons.tsx b/packages/core-app-api/src/app/icons.tsx index a102026b29..b234a292b1 100644 --- a/packages/core-app-api/src/app/icons.tsx +++ b/packages/core-app-api/src/app/icons.tsx @@ -18,6 +18,9 @@ import { IconComponent } from '@backstage/core-plugin-api'; import MuiApartmentIcon from '@material-ui/icons/Apartment'; import MuiBrokenImageIcon from '@material-ui/icons/BrokenImage'; import MuiCategoryIcon from '@material-ui/icons/Category'; +import MuiCreateNewFolderIcon from '@material-ui/icons/CreateNewFolder'; +import MuiSubjectIcon from '@material-ui/icons/Subject'; +import MuiSearchIcon from '@material-ui/icons/Search'; import MuiChatIcon from '@material-ui/icons/Chat'; import MuiDashboardIcon from '@material-ui/icons/Dashboard'; import MuiDocsIcon from '@material-ui/icons/Description'; @@ -35,6 +38,9 @@ import MuiWarningIcon from '@material-ui/icons/Warning'; type AppIconsKey = | 'brokenImage' | 'catalog' + | 'scaffolder' + | 'techdocs' + | 'search' | 'chat' | 'dashboard' | 'docs' @@ -58,6 +64,9 @@ export const defaultAppIcons: AppIcons = { brokenImage: MuiBrokenImageIcon, // To be confirmed: see https://github.com/backstage/backstage/issues/4970 catalog: MuiMenuBookIcon, + scaffolder: MuiCreateNewFolderIcon, + techdocs: MuiSubjectIcon, + search: MuiSearchIcon, chat: MuiChatIcon, dashboard: MuiDashboardIcon, docs: MuiDocsIcon, diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 40082925c6..92b449f705 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -80,7 +80,8 @@ const useStyles = makeStyles( backgroundColor: theme.palette.banner.error, }, warning: { - backgroundColor: theme.palette.banner.warning, + backgroundColor: + theme.palette.banner.warning ?? theme.palette.banner.error, }, }), { name: 'BackstageDismissableBanner' }, diff --git a/packages/theme/api-report.md b/packages/theme/api-report.md index 30dcb7b040..2dc6ab7b55 100644 --- a/packages/theme/api-report.md +++ b/packages/theme/api-report.md @@ -64,7 +64,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; - warning: string; + warning?: string; }; }; diff --git a/packages/theme/src/types.ts b/packages/theme/src/types.ts index 40f45ee739..e1acba3a01 100644 --- a/packages/theme/src/types.ts +++ b/packages/theme/src/types.ts @@ -76,7 +76,7 @@ export type BackstagePaletteAdditions = { error: string; text: string; link: string; - warning: string; + warning?: string; }; }; diff --git a/plugins/jenkins-backend/README.md b/plugins/jenkins-backend/README.md index bac56b6eaf..1413a45136 100644 --- a/plugins/jenkins-backend/README.md +++ b/plugins/jenkins-backend/README.md @@ -12,6 +12,12 @@ This is the backend half of the 2 Jenkins plugins and is responsible for: This plugin needs to be added to an existing backstage instance. +```bash +# From your Backstage root directory +cd packages/backend +yarn add @backstage/plugin-jenkins-backend +``` + Typically, this means creating a `src/plugins/jenkins.ts` file and adding a reference to it to `src/index.ts` ### jenkins.ts @@ -42,6 +48,29 @@ export default async function createPlugin({ } ``` +### src/index.ts + +```diff +diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts +index f2b14b2..2c64f47 100644 +--- a/packages/backend/src/index.ts ++++ b/packages/backend/src/index.ts +@@ -22,6 +22,7 @@ import { Config } from '@backstage/config'; + import app from './plugins/app'; ++import jenkins from './plugins/jenkins'; + import scaffolder from './plugins/scaffolder'; +@@ -56,6 +57,7 @@ async function main() { + const authEnv = useHotMemoize(module, () => createEnv('auth')); ++ const jenkinsEnv = useHotMemoize(module, () => createEnv('jenkins')); + const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); +@@ -63,6 +65,7 @@ async function main() { + + const apiRouter = Router(); + apiRouter.use('/catalog', await catalog(catalogEnv)); ++ apiRouter.use('/jenkins', await jenkins(jenkinsEnv)); + apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); +``` + This plugin must be provided with a JenkinsInfoProvider, this is a strategy object for finding the Jenkins instance and job for an entity. There is a standard one provided, but the Integrator is free to build their own. diff --git a/plugins/jenkins/README.md b/plugins/jenkins/README.md index 1af511b16f..2012555d93 100644 --- a/plugins/jenkins/README.md +++ b/plugins/jenkins/README.md @@ -16,7 +16,7 @@ cd packages/app yarn add @backstage/plugin-jenkins ``` -2. Add and configure the backend plugin according to it's instructions +2. Add and configure the [jenkins-backend](../jenkins-backend) plugin according to it's instructions 3. Add the `EntityJenkinsContent` extension to the `CI/CD` page and `EntityLatestJenkinsRunCard` to the `overview` page in the app (or wherever you'd prefer): diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index 1c5343d458..d8ba15279f 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -29,24 +29,25 @@ jest.doMock('fs-extra', () => ({ })); import { + DatabaseManager, + DockerContainerRunner, getVoidLogger, PluginDatabaseManager, - DatabaseManager, UrlReaders, - DockerContainerRunner, } from '@backstage/backend-common'; import { CatalogApi } from '@backstage/catalog-client'; +import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; -import { TemplateEntityV1beta2 } from '@backstage/catalog-model'; /** * TODO: The following should import directly from the router file. * Due to a circular dependency between this plugin and the * plugin-scaffolder-backend-module-cookiecutter plugin, it results in an error: * TypeError: _pluginscaffolderbackend.createTemplateAction is not a function */ -import { createRouter } from '../index'; +import { createRouter, DatabaseTaskStore, TaskBroker } from '../index'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; const createCatalogClient = (templates: any[] = []) => ({ @@ -73,6 +74,7 @@ const mockUrlReader = UrlReaders.default({ describe('createRouter', () => { let app: express.Express; + let taskBroker: TaskBroker; const template: TemplateEntityV1beta2 = { apiVersion: 'backstage.io/v1beta2', kind: 'Template', @@ -102,7 +104,17 @@ describe('createRouter', () => { }, }; - beforeAll(async () => { + beforeEach(async () => { + const logger = getVoidLogger(); + const databaseTaskStore = await DatabaseTaskStore.create({ + database: await createDatabase().getClient(), + }); + taskBroker = new StorageTaskBroker(databaseTaskStore, logger); + + jest.spyOn(taskBroker, 'dispatch'); + jest.spyOn(taskBroker, 'get'); + jest.spyOn(taskBroker, 'observe'); + const router = await createRouter({ logger: getVoidLogger(), config: new ConfigReader({}), @@ -110,11 +122,12 @@ describe('createRouter', () => { catalogClient: createCatalogClient([template]), containerRunner: new DockerContainerRunner({} as any), reader: mockUrlReader, + taskBroker, }); app = express().use(router); }); - beforeEach(() => { + afterEach(() => { jest.resetAllMocks(); }); @@ -142,6 +155,12 @@ describe('createRouter', () => { }); it('return the template id', async () => { + ( + taskBroker.dispatch as jest.Mocked['dispatch'] + ).mockResolvedValue({ + taskId: 'a-random-id', + }); + const response = await request(app) .post('/v2/tasks') .send({ @@ -151,28 +170,241 @@ describe('createRouter', () => { }, }); - expect(response.body.id).toBeDefined(); + expect(response.body.id).toBe('a-random-id'); expect(response.status).toEqual(201); }); }); describe('GET /v2/tasks/:taskId', () => { it('does not divulge secrets', async () => { - const postResponse = await request(app) - .post('/v2/tasks') - .set('Authorization', 'Bearer secret') - .send({ - templateName: 'create-react-app-template', - values: { - required: 'required-value', - }, - }); + (taskBroker.get as jest.Mocked['get']).mockResolvedValue({ + id: 'a-random-id', + spec: {} as any, + status: 'completed', + createdAt: '', + secrets: { token: 'secret' }, + }); - const response = await request(app) - .get(`/v2/tasks/${postResponse.body.id}`) - .send(); + const response = await request(app).get(`/v2/tasks/a-random-id`); expect(response.status).toEqual(200); + expect(response.body.status).toBe('completed'); expect(response.body.secrets).toBeUndefined(); }); }); + + describe('GET /v2/tasks/:taskId/eventstream', () => { + it('should return log messages', async () => { + const unsubscribe = jest.fn(); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + // emit after this function returned + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ], + }); + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + }); + + return { unsubscribe }; + }); + + let statusCode: any = undefined; + let headers: any = {}; + const responseDataFn = jest.fn(); + + const req = request(app) + .get('/v2/tasks/a-random-id/eventstream') + .set('accept', 'text/event-stream') + .parse((res, _) => { + ({ statusCode, headers } = res as any); + + res.on('data', chunk => { + responseDataFn(chunk.toString()); + + // the server expects the client to abort the request + if (chunk.includes('completion')) { + req.abort(); + } + }); + }); + + // wait for the request to finish + await req.catch(() => { + // ignore 'aborted' error + }); + + expect(statusCode).toBe(200); + expect(headers['content-type']).toBe('text/event-stream'); + expect(responseDataFn).toBeCalledTimes(2); + expect(responseDataFn).toBeCalledWith(`event: log +data: {"id":0,"taskId":"a-random-id","type":"log","createdAt":"","body":{"message":"My log message"}} + +`); + expect(responseDataFn).toBeCalledWith(`event: completion +data: {"id":1,"taskId":"a-random-id","type":"completion","createdAt":"","body":{"message":"Finished!"}} + +`); + + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( + { taskId: 'a-random-id' }, + expect.any(Function), + ); + + expect(unsubscribe).toBeCalledTimes(1); + }); + + it('should return log messages with after query', async () => { + const unsubscribe = jest.fn(); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + setImmediate(() => { + callback(undefined, { + events: [ + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + }); + return { unsubscribe }; + }); + + let statusCode: any = undefined; + let headers: any = {}; + + const req = request(app) + .get('/v2/tasks/a-random-id/eventstream') + .query({ after: 10 }) + .set('accept', 'text/event-stream') + .parse((res, _) => { + ({ statusCode, headers } = res as any); + + res.on('data', () => { + // close immediately + req.abort(); + }); + }); + + // wait for the request to finish + await req.catch(() => { + // ignore 'aborted' error + }); + + expect(statusCode).toBe(200); + expect(headers['content-type']).toBe('text/event-stream'); + + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( + { taskId: 'a-random-id', after: 10 }, + expect.any(Function), + ); + + expect(unsubscribe).toBeCalledTimes(1); + }); + }); + + describe('GET /v2/tasks/:taskId/events', () => { + it('should return log messages', async () => { + const unsubscribe = jest.fn(); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation(({ taskId }, callback) => { + callback(undefined, { + events: [ + { + id: 0, + taskId, + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + { + id: 1, + taskId, + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ], + }); + return { unsubscribe }; + }); + + const response = await request(app).get('/v2/tasks/a-random-id/events'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([ + { + id: 0, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + { + id: 1, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]); + + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( + { taskId: 'a-random-id' }, + expect.any(Function), + ); + expect(unsubscribe).toBeCalledTimes(1); + }); + + it('should return log messages with after query', async () => { + const unsubscribe = jest.fn(); + ( + taskBroker.observe as jest.Mocked['observe'] + ).mockImplementation((_, callback) => { + callback(undefined, { events: [] }); + return { unsubscribe }; + }); + + const response = await request(app) + .get('/v2/tasks/a-random-id/events') + .query({ after: 10 }); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([]); + + expect(taskBroker.observe).toBeCalledTimes(1); + expect(taskBroker.observe).toBeCalledWith( + { taskId: 'a-random-id', after: 10 }, + expect.any(Function), + ); + expect(unsubscribe).toBeCalledTimes(1); + }); + }); }); diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index a78b04da78..2ae30593f2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -14,33 +14,33 @@ * limitations under the License. */ -import { Config } from '@backstage/config'; -import express from 'express'; -import Router from 'express-promise-router'; -import { Logger } from 'winston'; -import { CatalogEntityClient } from '../lib/catalog'; -import { validate } from 'jsonschema'; -import { - DatabaseTaskStore, - TemplateActionRegistry, - TaskWorker, - TemplateAction, - createBuiltinActions, -} from '../scaffolder'; -import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; -import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; import { ContainerRunner, PluginDatabaseManager, UrlReader, } from '@backstage/backend-common'; -import { InputError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; -import { TemplateEntityV1beta2, Entity } from '@backstage/catalog-model'; -import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; - +import { Entity, TemplateEntityV1beta2 } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { InputError, NotFoundError } from '@backstage/errors'; import { ScmIntegrations } from '@backstage/integration'; -import { TaskBroker, TaskSpec } from '../scaffolder/tasks/types'; +import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; +import express from 'express'; +import Router from 'express-promise-router'; +import { validate } from 'jsonschema'; +import { Logger } from 'winston'; +import { CatalogEntityClient } from '../lib/catalog'; +import { + createBuiltinActions, + DatabaseTaskStore, + TaskBroker, + TaskSpec, + TaskWorker, + TemplateAction, + TemplateActionRegistry, +} from '../scaffolder'; +import { StorageTaskBroker } from '../scaffolder/tasks/StorageTaskBroker'; +import { getEntityBaseUrl, getWorkingDirectory } from './helpers'; /** * RouterOptions @@ -247,7 +247,9 @@ export async function createRouter( }) .get('/v2/tasks/:taskId/eventstream', async (req, res) => { const { taskId } = req.params; - const after = Number(req.query.after) || undefined; + const after = + req.query.after !== undefined ? Number(req.query.after) : undefined; + logger.debug(`Event stream observing taskId '${taskId}' opened`); // Mandatory headers and http status to keep connection open @@ -278,7 +280,8 @@ export async function createRouter( // to automatically reconnect because it lost connection. } } - res.flush(); + // res.flush() is only available with the compression middleware + res.flush?.(); if (shouldUnsubscribe) unsubscribe(); }, ); @@ -288,6 +291,43 @@ export async function createRouter( unsubscribe(); logger.debug(`Event stream observing taskId '${taskId}' closed`); }); + }) + .get('/v2/tasks/:taskId/events', async (req, res) => { + const { taskId } = req.params; + const after = Number(req.query.after) || undefined; + + let unsubscribe = () => {}; + + // cancel the request after 30 seconds. this aligns with the recommendations of RFC 6202. + const timeout = setTimeout(() => { + unsubscribe(); + res.json([]); + }, 30_000); + + // Get all known events after an id (always includes the completion event) and return the first callback + ({ unsubscribe } = taskBroker.observe( + { taskId, after }, + (error, { events }) => { + // stop the timeout + clearTimeout(timeout); + unsubscribe(); + + if (error) { + logger.error( + `Received error from log when observing taskId '${taskId}', ${error}`, + ); + } + + res.json(events); + }, + )); + + // When client closes connection we update the clients list + // avoiding the disconnected one + req.on('close', () => { + unsubscribe(); + clearTimeout(timeout); + }); }); const app = express(); diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 554fccabb7..0ac6910264 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -178,6 +178,7 @@ export class ScaffolderClient implements ScaffolderApi { discoveryApi: DiscoveryApi; identityApi: IdentityApi; scmIntegrationsApi: ScmIntegrationRegistry; + useLongPollingLogs?: boolean; }); // (undocumented) getIntegrationsList(options: { allowedHosts: string[] }): Promise< @@ -199,13 +200,7 @@ export class ScaffolderClient implements ScaffolderApi { // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen scaffold(templateName: string, values: Record): Promise; // (undocumented) - streamLogs({ - taskId, - after, - }: { - taskId: string; - after?: number; - }): Observable; + streamLogs(opts: { taskId: string; after?: number }): Observable; } // Warning: (ae-missing-release-tag) "ScaffolderFieldExtensions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 0b517671ba..58bee32201 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -55,6 +55,7 @@ "json-schema": "^0.3.0", "lodash": "^4.17.21", "luxon": "^2.0.2", + "qs": "^6.9.4", "react": "^16.13.1", "react-dom": "^16.13.1", "react-lazylog": "^4.5.2", @@ -77,6 +78,7 @@ "@types/jest": "^26.0.7", "@types/node": "^14.14.32", "cross-fetch": "^3.0.6", + "event-source-polyfill": "^1.0.25", "msw": "^0.35.0" }, "files": [ diff --git a/plugins/scaffolder/src/api.test.ts b/plugins/scaffolder/src/api.test.ts index cc4b70283c..01463a6e1c 100644 --- a/plugins/scaffolder/src/api.test.ts +++ b/plugins/scaffolder/src/api.test.ts @@ -14,12 +14,24 @@ * limitations under the License. */ -import { ScmIntegrations } from '@backstage/integration'; -import { ScaffolderClient } from './api'; import { ConfigReader } from '@backstage/core-app-api'; +import { ScmIntegrations } from '@backstage/integration'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { ScaffolderClient } from './api'; + +const MockedEventSource = global.EventSource as jest.MockedClass< + typeof EventSource +>; + +const server = setupServer(); describe('api', () => { - const discoveryApi = {} as any; + setupRequestMockHandlers(server); + const mockBaseUrl = 'http://backstage/api'; + + const discoveryApi = { getBaseUrl: async () => mockBaseUrl }; const identityApi = {} as any; const scmIntegrationsApi = ScmIntegrations.fromConfig( new ConfigReader({ @@ -32,10 +44,14 @@ describe('api', () => { }, }), ); - const apiClient = new ScaffolderClient({ - scmIntegrationsApi, - discoveryApi, - identityApi, + + let apiClient: ScaffolderClient; + beforeEach(() => { + apiClient = new ScaffolderClient({ + scmIntegrationsApi, + discoveryApi, + identityApi, + }); }); it('should return default and custom integrations', async () => { @@ -51,4 +67,240 @@ describe('api', () => { expect(allowedHosts).toContain(integration.host), ); }); + + describe('streamEvents', () => { + describe('eventsource', () => { + it('should work', async () => { + MockedEventSource.prototype.addEventListener.mockImplementation( + (type, fn) => { + if (typeof fn !== 'function') { + return; + } + + if (type === 'log') { + fn({ + data: '{"id":1,"taskId":"a-random-id","type":"log","createdAt":"","body":{"message":"My log message"}}', + } as any); + } else if (type === 'completion') { + fn({ + data: '{"id":2,"taskId":"a-random-id","type":"completion","createdAt":"","body":{"message":"Finished!"}}', + } as any); + } + }, + ); + + const next = jest.fn(); + + await new Promise(complete => { + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }); + }); + + expect(MockedEventSource).toBeCalledWith( + 'http://backstage/api/v2/tasks/a-random-task-id/eventstream', + { withCredentials: true }, + ); + expect(MockedEventSource.prototype.close).toBeCalled(); + + expect(next).toBeCalledTimes(2); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + }); + + describe('longPolling', () => { + beforeEach(() => { + apiClient = new ScaffolderClient({ + scmIntegrationsApi, + discoveryApi, + identityApi, + useLongPollingLogs: true, + }); + }); + + it('should work', async () => { + server.use( + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (req, res, ctx) => { + const { taskId } = req.params; + const after = req.url.searchParams.get('after'); + + if (taskId === 'a-random-task-id') { + if (!after) { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } else if (after === '1') { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } + } + + return res(ctx.status(500)); + }, + ), + ); + + const next = jest.fn(); + + await new Promise(complete => + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }), + ); + + expect(next).toBeCalledTimes(2); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + + it('should unsubscribe', async () => { + expect.assertions(3); + + server.use( + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (req, res, ctx) => { + const { taskId } = req.params; + + const after = req.url.searchParams.get('after'); + + // use assertion to make sure it is not called after unsubscribing + expect(after).toBe(null); + + if (taskId === 'a-random-task-id') { + return res( + ctx.json([ + { + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }, + ]), + ); + } + + return res(ctx.status(500)); + }, + ), + ); + + const next = jest.fn(); + + await new Promise(complete => { + const subscription = apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ + next: (...args) => { + next(...args); + subscription.unsubscribe(); + complete(); + }, + }); + }); + + expect(next).toBeCalledTimes(1); + expect(next).toBeCalledWith({ + id: 1, + taskId: 'a-random-id', + type: 'log', + createdAt: '', + body: { message: 'My log message' }, + }); + }); + + it('should continue after error', async () => { + const called = jest.fn(); + + server.use( + rest.get( + `${mockBaseUrl}/v2/tasks/:taskId/events`, + (_req, res, ctx) => { + called(); + + if (called.mock.calls.length > 1) { + return res( + ctx.json([ + { + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }, + ]), + ); + } + + return res(ctx.status(500)); + }, + ), + ); + + const next = jest.fn(); + + await new Promise(complete => + apiClient + .streamLogs({ taskId: 'a-random-task-id' }) + .subscribe({ next, complete }), + ); + + expect(called).toBeCalledTimes(2); + + expect(next).toBeCalledTimes(1); + expect(next).toBeCalledWith({ + id: 2, + taskId: 'a-random-id', + type: 'completion', + createdAt: '', + body: { message: 'Finished!' }, + }); + }); + }); + }); }); diff --git a/plugins/scaffolder/src/api.ts b/plugins/scaffolder/src/api.ts index 1b531b652d..a16ab59053 100644 --- a/plugins/scaffolder/src/api.ts +++ b/plugins/scaffolder/src/api.ts @@ -15,17 +15,18 @@ */ import { EntityName } from '@backstage/catalog-model'; -import { JsonObject, JsonValue, Observable } from '@backstage/types'; -import { ResponseError } from '@backstage/errors'; -import { ScmIntegrationRegistry } from '@backstage/integration'; -import { Field, FieldValidation } from '@rjsf/core'; -import ObservableImpl from 'zen-observable'; -import { ListActionsResponse, ScaffolderTask, Status } from './types'; import { createApiRef, DiscoveryApi, IdentityApi, } from '@backstage/core-plugin-api'; +import { ResponseError } from '@backstage/errors'; +import { ScmIntegrationRegistry } from '@backstage/integration'; +import { JsonObject, JsonValue, Observable } from '@backstage/types'; +import { Field, FieldValidation } from '@rjsf/core'; +import qs from 'qs'; +import ObservableImpl from 'zen-observable'; +import { ListActionsResponse, ScaffolderTask, Status } from './types'; export const scaffolderApiRef = createApiRef({ id: 'plugin.scaffolder.service', @@ -94,15 +95,18 @@ export class ScaffolderClient implements ScaffolderApi { private readonly discoveryApi: DiscoveryApi; private readonly identityApi: IdentityApi; private readonly scmIntegrationsApi: ScmIntegrationRegistry; + private readonly useLongPollingLogs: boolean; constructor(options: { discoveryApi: DiscoveryApi; identityApi: IdentityApi; scmIntegrationsApi: ScmIntegrationRegistry; + useLongPollingLogs?: boolean; }) { this.discoveryApi = options.discoveryApi; this.identityApi = options.identityApi; this.scmIntegrationsApi = options.scmIntegrationsApi; + this.useLongPollingLogs = options.useLongPollingLogs ?? false; } async getIntegrationsList(options: { allowedHosts: string[] }) { @@ -189,7 +193,15 @@ export class ScaffolderClient implements ScaffolderApi { return await response.json(); } - streamLogs({ + streamLogs(opts: { taskId: string; after?: number }): Observable { + if (this.useLongPollingLogs) { + return this.streamLogsPolling(opts); + } + + return this.streamLogsEventStream(opts); + } + + private streamLogsEventStream({ taskId, after, }: { @@ -239,6 +251,46 @@ export class ScaffolderClient implements ScaffolderApi { }); } + private streamLogsPolling({ + taskId, + after: inputAfter, + }: { + taskId: string; + after?: number; + }): Observable { + let after = inputAfter; + + return new ObservableImpl(subscriber => { + this.discoveryApi.getBaseUrl('scaffolder').then(async baseUrl => { + while (!subscriber.closed) { + const url = `${baseUrl}/v2/tasks/${encodeURIComponent( + taskId, + )}/events?${qs.stringify({ after })}`; + const response = await fetch(url); + + if (!response.ok) { + // wait for one second to not run into an + await new Promise(resolve => setTimeout(resolve, 1000)); + continue; + } + + const logs = (await response.json()) as LogEvent[]; + + for (const event of logs) { + after = Number(event.id); + + subscriber.next(event); + + if (event.type === 'completion') { + subscriber.complete(); + return; + } + } + } + }); + }); + } + /** * @returns ListActionsResponse containing all registered actions. */ diff --git a/plugins/scaffolder/src/setupTests.ts b/plugins/scaffolder/src/setupTests.ts index 963c0f188b..10252413a7 100644 --- a/plugins/scaffolder/src/setupTests.ts +++ b/plugins/scaffolder/src/setupTests.ts @@ -15,3 +15,7 @@ */ import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; + +const { EventSourcePolyfill } = jest.requireMock('event-source-polyfill'); +global.EventSource = EventSourcePolyfill;