From 85abf247eff54b2939d1d700a6070c694f75bb7c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 20 May 2024 13:52:26 +0200 Subject: [PATCH 01/83] events-backend: add dev setup Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 75 +++++++++++++++++++++++++++++ plugins/events-backend/package.json | 1 + yarn.lock | 1 + 3 files changed, 77 insertions(+) create mode 100644 plugins/events-backend/dev/index.ts diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts new file mode 100644 index 0000000000..c0626543c8 --- /dev/null +++ b/plugins/events-backend/dev/index.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createBackend } from '@backstage/backend-defaults'; +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import { eventsServiceRef } from '@backstage/plugin-events-node'; + +const backend = createBackend(); + +backend.add(import('../src/alpha')); + +backend.add( + createBackendPlugin({ + pluginId: 'producer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + }, + async init({ events, logger }) { + setInterval(() => { + logger.info(`Publishing event to topic 'test'`); + events.publish({ + eventPayload: { foo: 'bar' }, + topic: 'test', + metadata: { meta: 'baz' }, + }); + }, 5000); + }, + }); + }, + }), +); + +backend.add( + createBackendPlugin({ + pluginId: 'consumer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + }, + async init({ events, logger }) { + events.subscribe({ + id: 'test-1', + topics: ['test'], + async onEvent(event) { + logger.info(`Received event: ${JSON.stringify(event, null, 2)}`); + }, + }); + }, + }); + }, + }), +); + +backend.start(); diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index 34f64d05e0..c25109cf82 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -61,6 +61,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-defaults": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@backstage/plugin-events-backend-test-utils": "workspace:^", diff --git a/yarn.lock b/yarn.lock index 8a3b7b8023..cee8a0c9ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6138,6 +6138,7 @@ __metadata: resolution: "@backstage/plugin-events-backend@workspace:plugins/events-backend" dependencies: "@backstage/backend-common": ^0.25.0 + "@backstage/backend-defaults": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" From 335443229f9ce802a3b1dbd2e9a73a718a1069e0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 20 May 2024 18:49:44 +0200 Subject: [PATCH 02/83] events-backend: initial WebSocket server Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 38 ++++++-- plugins/events-backend/package.json | 3 +- .../src/service/EventsPlugin.ts | 5 ++ .../src/service/hub/EventHub.ts | 86 +++++++++++++++++++ .../events-backend/src/service/hub/index.ts | 17 ++++ yarn.lock | 1 + 6 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 plugins/events-backend/src/service/hub/EventHub.ts create mode 100644 plugins/events-backend/src/service/hub/index.ts diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index c0626543c8..95ed3e0be8 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -19,6 +19,7 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; +import { WebSocket } from 'ws'; import { eventsServiceRef } from '@backstage/plugin-events-node'; const backend = createBackend(); @@ -35,14 +36,14 @@ backend.add( logger: coreServices.logger, }, async init({ events, logger }) { - setInterval(() => { - logger.info(`Publishing event to topic 'test'`); - events.publish({ - eventPayload: { foo: 'bar' }, - topic: 'test', - metadata: { meta: 'baz' }, - }); - }, 5000); + // setInterval(() => { + // logger.info(`Publishing event to topic 'test'`); + // events.publish({ + // eventPayload: { foo: 'bar' }, + // topic: 'test', + // metadata: { meta: 'baz' }, + // }); + // }, 5000); }, }); }, @@ -57,8 +58,10 @@ backend.add( deps: { events: eventsServiceRef, logger: coreServices.logger, + discovery: coreServices.discovery, + rootLifecycle: coreServices.rootLifecycle, }, - async init({ events, logger }) { + async init({ events, logger, discovery, rootLifecycle }) { events.subscribe({ id: 'test-1', topics: ['test'], @@ -66,6 +69,23 @@ backend.add( logger.info(`Received event: ${JSON.stringify(event, null, 2)}`); }, }); + + rootLifecycle.addStartupHook(async () => { + logger.info('Started!'); + const baseUrl = await discovery.getBaseUrl('events'); + console.log(`DEBUG: baseUrl=`, baseUrl); + const ws = new WebSocket(`${baseUrl}/hub/connect`); + ws.onopen = () => { + console.log('DEBUG: ws.onopen'); + ws.send('derp!'); + }; + ws.onmessage = event => { + console.log(`DEBUG: event=`, event.data); + }; + ws.onerror = error => { + console.log(`Client error`, String(error)); + }; + }); }, }); }, diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index c25109cf82..6665ff1e52 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -58,7 +58,8 @@ "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "winston": "^3.2.1" + "winston": "^3.2.1", + "ws": "^8.17.0" }, "devDependencies": { "@backstage/backend-defaults": "workspace:^", diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index d95dbf3efc..0e115c9772 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -28,6 +28,7 @@ import { } from '@backstage/plugin-events-node'; import Router from 'express-promise-router'; import { HttpPostIngressEventPublisher } from './http'; +import { EventHub } from './hub'; class EventsExtensionPointImpl implements EventsExtensionPoint { #httpPostIngresses: HttpPostIngressOptions[] = []; @@ -93,6 +94,10 @@ export const eventsPlugin = createBackendPlugin({ }); const eventsRouter = Router(); http.bind(eventsRouter); + + const hub = await EventHub.create({ logger }); + eventsRouter.use('/hub', hub.handler()); + router.use(eventsRouter); router.addAuthPolicy({ allow: 'unauthenticated', diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts new file mode 100644 index 0000000000..b1ef75d9e6 --- /dev/null +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { Handler } from 'express'; +import Router from 'express-promise-router'; +import { WebSocketServer, type WebSocket } from 'ws'; + +export class EventHub { + static async create(options: { logger: LoggerService }) { + const logger = options.logger.child({ type: 'EventHub' }); + const router = Router(); + + const server = new WebSocketServer({ + noServer: true, + clientTracking: false, + }); + server.on('error', error => { + logger.error(`WebSocket server error`, error); + }); + + const hub = new EventHub(server, router, logger); + + router.get('/connect', hub.#handleGetConnect); + + return hub; + } + + readonly #server: WebSocketServer; + readonly #handler: Handler; + readonly #logger: LoggerService; + + #connections = new Set(); + + private constructor( + server: WebSocketServer, + handler: Handler, + logger: LoggerService, + ) { + this.#server = server; + this.#handler = handler; + this.#logger = logger; + } + + handler(): Handler { + return this.#handler; + } + + #handleGetConnect: Handler = (req, _res) => { + this.#server.handleUpgrade(req, req.socket, Buffer.alloc(0), conn => { + const id = Math.random().toString(36).slice(2, 10); + const logger = this.#logger.child({ connection: id }); + + logger.info(`New connection from '${req.socket.remoteAddress}'`); + this.#connections.add(conn); + + conn.onmessage = event => { + logger.debug(`Message from client: ${JSON.stringify(event.data)}`); + }; + conn.send('hello there!'); + + conn.addListener('ping', () => { + conn.pong(); + }); + }); + }; + + close() { + this.#connections.forEach(conn => conn.close()); + this.#connections.clear(); + this.#server.close(); + } +} diff --git a/plugins/events-backend/src/service/hub/index.ts b/plugins/events-backend/src/service/hub/index.ts new file mode 100644 index 0000000000..4acc848298 --- /dev/null +++ b/plugins/events-backend/src/service/hub/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { EventHub } from './EventHub'; diff --git a/yarn.lock b/yarn.lock index cee8a0c9ac..18e926a14e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6150,6 +6150,7 @@ __metadata: express-promise-router: ^4.1.0 supertest: ^7.0.0 winston: ^3.2.1 + ws: ^8.17.0 languageName: unknown linkType: soft From 9fff7a4bb85cce9e899a999f31609e3865f083a6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 20 May 2024 19:16:08 +0200 Subject: [PATCH 03/83] events-backend: abstraction for individual connections Signed-off-by: Patrik Oldsberg --- .../src/service/hub/EventHub.ts | 97 +++++++++++++++---- 1 file changed, 80 insertions(+), 17 deletions(-) diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index b1ef75d9e6..d98c9e4d3d 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -17,8 +17,74 @@ import { LoggerService } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; +import { Socket } from 'net'; import { WebSocketServer, type WebSocket } from 'ws'; +/** + * Manages a single WebSocket connection. + * + * @internal + */ +class EventClientConnection { + static create(options: { + ws: WebSocket; + socket: Socket; + logger: LoggerService; + }) { + const { ws } = options; + + const id = Math.random().toString(36).slice(2, 10); + const logger = options.logger.child({ connection: id }); + + ws.addListener('ping', () => { + ws.pong(); + }); + + ws.onmessage = event => { + logger.debug(`Message from client: ${JSON.stringify(event.data)}`); + }; + ws.send('hello there!'); + + const conn = new EventClientConnection(id, ws, options.socket, logger); + + logger.info(`New ${conn}`); + + return conn; + } + + readonly #id: string; + readonly #ws: WebSocket; + readonly #socket: Socket; + readonly #logger: LoggerService; + + constructor( + id: string, + ws: WebSocket, + socket: Socket, + logger: LoggerService, + ) { + this.#id = id; + this.#ws = ws; + this.#socket = socket; + this.#logger = logger; + } + + get id() { + return this.#id; + } + + close() { + this.#ws.close(); + this.#logger.info(`Closed ${this}`); + } + + toString() { + return `EventClientConnection{id=${this.#id},addr=${ + this.#socket.remoteAddress + }}`; + } +} + export class EventHub { static async create(options: { logger: LoggerService }) { const logger = options.logger.child({ type: 'EventHub' }); @@ -43,7 +109,7 @@ export class EventHub { readonly #handler: Handler; readonly #logger: LoggerService; - #connections = new Set(); + #connections = new Map(); private constructor( server: WebSocketServer, @@ -60,22 +126,19 @@ export class EventHub { } #handleGetConnect: Handler = (req, _res) => { - this.#server.handleUpgrade(req, req.socket, Buffer.alloc(0), conn => { - const id = Math.random().toString(36).slice(2, 10); - const logger = this.#logger.child({ connection: id }); - - logger.info(`New connection from '${req.socket.remoteAddress}'`); - this.#connections.add(conn); - - conn.onmessage = event => { - logger.debug(`Message from client: ${JSON.stringify(event.data)}`); - }; - conn.send('hello there!'); - - conn.addListener('ping', () => { - conn.pong(); - }); - }); + this.#server.handleUpgrade( + req, + req.socket, + Buffer.alloc(0), + (ws, { socket }) => { + const conn = EventClientConnection.create({ + ws, + socket, + logger: this.#logger, + }); + this.#connections.set(conn.id, conn); + }, + ); }; close() { From 68e05f6cb1f2754cc152638f22f7c4fdb44f70bc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 21 May 2024 11:51:02 +0200 Subject: [PATCH 04/83] events-backend: connection event handlers + auth Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 17 ++- .../src/service/EventsPlugin.ts | 5 +- .../src/service/hub/EventHub.ts | 129 +++++++++++++----- 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 95ed3e0be8..f5eb40ad50 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -59,9 +59,10 @@ backend.add( events: eventsServiceRef, logger: coreServices.logger, discovery: coreServices.discovery, + auth: coreServices.auth, rootLifecycle: coreServices.rootLifecycle, }, - async init({ events, logger, discovery, rootLifecycle }) { + async init({ events, logger, discovery, rootLifecycle, auth }) { events.subscribe({ id: 'test-1', topics: ['test'], @@ -74,7 +75,15 @@ backend.add( logger.info('Started!'); const baseUrl = await discovery.getBaseUrl('events'); console.log(`DEBUG: baseUrl=`, baseUrl); - const ws = new WebSocket(`${baseUrl}/hub/connect`); + const { token } = await auth.getPluginRequestToken({ + onBehalfOf: await auth.getOwnServiceCredentials(), + targetPluginId: 'events', + }); + const ws = new WebSocket(`${baseUrl}/hub/connect`, { + headers: { + Authorization: `Bearer ${token}`, + }, + }); ws.onopen = () => { console.log('DEBUG: ws.onopen'); ws.send('derp!'); @@ -82,8 +91,8 @@ backend.add( ws.onmessage = event => { console.log(`DEBUG: event=`, event.data); }; - ws.onerror = error => { - console.log(`Client error`, String(error)); + ws.onerror = event => { + console.log(`Client error`, event.error); }; }); }, diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index 0e115c9772..ecb234066d 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -76,9 +76,10 @@ export const eventsPlugin = createBackendPlugin({ config: coreServices.rootConfig, events: eventsServiceRef, logger: coreServices.logger, + httpAuth: coreServices.httpAuth, router: coreServices.httpRouter, }, - async init({ config, events, logger, router }) { + async init({ config, events, logger, httpAuth, router }) { const ingresses = Object.fromEntries( extensionPoint.httpPostIngresses.map(ingress => [ ingress.topic, @@ -95,7 +96,7 @@ export const eventsPlugin = createBackendPlugin({ const eventsRouter = Router(); http.bind(eventsRouter); - const hub = await EventHub.create({ logger }); + const hub = await EventHub.create({ logger, httpAuth }); eventsRouter.use('/hub', hub.handler()); router.use(eventsRouter); diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index d98c9e4d3d..cc99a2b2dd 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -14,11 +14,17 @@ * limitations under the License. */ -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + BackstageCredentials, + BackstageServicePrincipal, + HttpAuthService, + LoggerService, +} from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; import { Socket } from 'net'; -import { WebSocketServer, type WebSocket } from 'ws'; +import { STATUS_CODES } from 'http'; +import { WebSocketServer, type WebSocket, RawData } from 'ws'; /** * Manages a single WebSocket connection. @@ -30,63 +36,90 @@ class EventClientConnection { ws: WebSocket; socket: Socket; logger: LoggerService; + credentials: BackstageCredentials; }) { - const { ws } = options; + const { ws, credentials } = options; const id = Math.random().toString(36).slice(2, 10); - const logger = options.logger.child({ connection: id }); - - ws.addListener('ping', () => { - ws.pong(); + const logger = options.logger.child({ + connection: id, + subject: credentials.principal.subject, }); - ws.onmessage = event => { - logger.debug(`Message from client: ${JSON.stringify(event.data)}`); - }; - ws.send('hello there!'); + const conn = new EventClientConnection(id, ws, logger, credentials); - const conn = new EventClientConnection(id, ws, options.socket, logger); + ws.addListener('close', conn.#handleClose); + ws.addListener('error', conn.#handleError); + ws.addListener('message', conn.#handleMessage); + ws.addListener('ping', () => ws.pong()); - logger.info(`New ${conn}`); + logger.info( + `New event client connection from '${options.socket.remoteAddress}'`, + ); return conn; } readonly #id: string; readonly #ws: WebSocket; - readonly #socket: Socket; readonly #logger: LoggerService; + readonly #credentials: BackstageCredentials; constructor( id: string, ws: WebSocket, - socket: Socket, logger: LoggerService, + credentials: BackstageCredentials, ) { this.#id = id; this.#ws = ws; - this.#socket = socket; this.#logger = logger; + this.#credentials = credentials; } get id() { return this.#id; } + #handleClose = (code: number, reason: Buffer) => { + this.#removeListeners(); + this.#logger.info(`Remote closed code=${code} reason=${reason}`); + }; + + #handleError = (error: Error) => { + this.#removeListeners(); + this.#logger.error(`WebSocket error`, error); + }; + + #handleMessage = (data: RawData, isBinary: boolean) => { + console.log(`DEBUG: isBinary=${isBinary} data=${data}`); + }; + close() { + this.#removeListeners(); this.#ws.close(); - this.#logger.info(`Closed ${this}`); + this.#logger.info(`Closed connection`); + } + + #removeListeners() { + this.#ws.removeListener('close', this.#handleClose); + this.#ws.removeListener('error', this.#handleError); + this.#ws.removeListener('message', this.#handleMessage); } toString() { - return `EventClientConnection{id=${this.#id},addr=${ - this.#socket.remoteAddress + return `eventClientConnection{id=${this.#id},subject=${ + this.#credentials.principal.subject }}`; } } export class EventHub { - static async create(options: { logger: LoggerService }) { + static async create(options: { + logger: LoggerService; + httpAuth: HttpAuthService; + }) { + const { httpAuth } = options; const logger = options.logger.child({ type: 'EventHub' }); const router = Router(); @@ -94,11 +127,12 @@ export class EventHub { noServer: true, clientTracking: false, }); + server.on('error', error => { logger.error(`WebSocket server error`, error); }); - const hub = new EventHub(server, router, logger); + const hub = new EventHub(server, router, logger, httpAuth); router.get('/connect', hub.#handleGetConnect); @@ -108,6 +142,7 @@ export class EventHub { readonly #server: WebSocketServer; readonly #handler: Handler; readonly #logger: LoggerService; + readonly #httpAuth: HttpAuthService; #connections = new Map(); @@ -115,30 +150,54 @@ export class EventHub { server: WebSocketServer, handler: Handler, logger: LoggerService, + httpAuth: HttpAuthService, ) { this.#server = server; this.#handler = handler; this.#logger = logger; + this.#httpAuth = httpAuth; } handler(): Handler { return this.#handler; } - #handleGetConnect: Handler = (req, _res) => { - this.#server.handleUpgrade( - req, - req.socket, - Buffer.alloc(0), - (ws, { socket }) => { - const conn = EventClientConnection.create({ - ws, - socket, - logger: this.#logger, - }); - this.#connections.set(conn.id, conn); - }, - ); + #handleGetConnect: Handler = async (req, _res) => { + try { + const credentials = await this.#httpAuth.credentials(req, { + allow: ['service'], + }); + + this.#server.handleUpgrade( + req, + req.socket, + Buffer.alloc(0), + (ws, { socket }) => { + const conn = EventClientConnection.create({ + ws, + socket, + logger: this.#logger, + credentials, + }); + this.#connections.set(conn.id, conn); + }, + ); + } catch (error) { + let status = 500; + if (error.name === 'AuthenticationError') { + status = 401; + } else if (error.name === 'NotAllowedError') { + status = 403; + } + req.socket.write( + `HTTP/1.1 ${status} ${STATUS_CODES[status]}\r\n` + + 'Upgrade: WebSocket\r\n' + + 'Connection: Upgrade\r\n' + + '\r\n', + ); + req.socket.destroy(); + this.#logger.info('WebSocket upgrade failed', error); + } }; close() { From 5032c181e7387943e7693b3305a9e3cc79dc6cc6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 21 May 2024 17:57:04 +0200 Subject: [PATCH 05/83] events-backend: initial req/res protocol Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 9 +- plugins/events-backend/package.json | 6 +- .../src/service/hub/EventHub.ts | 172 +++++++++++++++++- yarn.lock | 12 +- 4 files changed, 189 insertions(+), 10 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index f5eb40ad50..445fa4b4b9 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -86,7 +86,14 @@ backend.add( }); ws.onopen = () => { console.log('DEBUG: ws.onopen'); - ws.send('derp!'); + ws.send( + JSON.stringify([ + 'req', + 1, + 'subscribe', + { id: 'derp', topics: ['test'] }, + ]), + ); }; ws.onmessage = event => { console.log(`DEBUG: event=`, event.data); diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index 6665ff1e52..dc7c2b0110 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -54,12 +54,16 @@ "@backstage/backend-common": "^0.25.0", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/plugin-events-node": "workspace:^", + "@backstage/types": "workspace:^", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", "winston": "^3.2.1", - "ws": "^8.17.0" + "ws": "^8.17.0", + "zod": "^3.22.4", + "zod-validation-error": "^3.3.0" }, "devDependencies": { "@backstage/backend-defaults": "workspace:^", diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index cc99a2b2dd..21692ca10f 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -25,6 +25,76 @@ import Router from 'express-promise-router'; import { Socket } from 'net'; import { STATUS_CODES } from 'http'; import { WebSocketServer, type WebSocket, RawData } from 'ws'; +import { z, ZodError } from 'zod'; +import { fromZodError } from 'zod-validation-error'; +import { JsonObject, JsonValue } from '@backstage/types'; +import { serializeError } from '@backstage/errors'; + +/* + +# Protocol + +## Request/Response + +General request/response format used for all communication: + +-> [type: 'req', id: number, method: string, params: JsonObject] +<- [type: 'res', id: number, status: 'resolved' | 'rejected', result: JsonObject] + +## Client -> Server + +### Subscribe + +-> method: 'subscribe', params: { id: string, topics: string[] } +<- result: void + +### Publish + +-> method: 'publish', params: { topic: string, payload: JsonObject } +<- result: void + +## Server -> Client + +### Event + +-> method: 'event', params: { topic: string, payload: JsonObject } +<- result: void + +*/ + +const messageSchema = z.union([ + z.tuple([ + z.literal('req'), + z.number().int().gt(0), + z.string().min(1), + z.any(), + ]), + z.tuple([ + z.literal('res'), + z.number().int().gt(0), + z.enum(['resolved', 'rejected']), + z.any(), + ]), +]); +const subscribeParamsSchema = z.object({ + id: z.string().min(1), + topics: z.array(z.string().min(1)), +}); +const publishParamsSchema = z.object({ + topic: z.string().min(1), + payload: z.any(), +}); +const eventParamsSchema = z.object({ + topic: z.string().min(1), + payload: z.any(), +}); + +function errorToJson(error: Error): JsonObject { + if (error.name === 'ZodError') { + return serializeError(fromZodError(error as ZodError)); + } + return serializeError(error); +} /** * Manages a single WebSocket connection. @@ -51,7 +121,6 @@ class EventClientConnection { ws.addListener('close', conn.#handleClose); ws.addListener('error', conn.#handleError); ws.addListener('message', conn.#handleMessage); - ws.addListener('ping', () => ws.pong()); logger.info( `New event client connection from '${options.socket.remoteAddress}'`, @@ -65,6 +134,19 @@ class EventClientConnection { readonly #logger: LoggerService; readonly #credentials: BackstageCredentials; + #seq = 1; + readonly #pendingRequests = new Map< + number, + { resolve(result: unknown): void; reject(error: unknown): void } + >(); + readonly #requestHandlers = new Map< + string, + { + schema: z.ZodType; + handler: (params: any) => unknown; + } + >(); + constructor( id: string, ws: WebSocket, @@ -91,10 +173,78 @@ class EventClientConnection { this.#logger.error(`WebSocket error`, error); }; - #handleMessage = (data: RawData, isBinary: boolean) => { - console.log(`DEBUG: isBinary=${isBinary} data=${data}`); + #handleMessage = (rawData: RawData, isBinary: boolean) => { + if (isBinary) { + return; + } + try { + const data = Array.isArray(rawData) + ? Buffer.concat(rawData) + : Buffer.from(rawData); + const message = messageSchema.parse(JSON.parse(data.toString('utf8'))); + + if (message[0] === 'req') { + const [, seq, method, params] = message; + const handler = this.#requestHandlers.get(method); + if (!handler) { + throw new Error(`Unknown method '${method}'`); + } + + try { + const parsedParams = handler.schema.parse(params); + + Promise.resolve(handler.handler(parsedParams)).then( + result => { + this.#sendMessage('res', seq, 'resolved', result ?? null); + }, + error => { + this.#sendMessage('res', seq, 'rejected', errorToJson(error)); + }, + ); + } catch (error) { + this.#sendMessage('res', seq, false, errorToJson(error)); + } + } else if (message[0] === 'res') { + const [, seq, success, result] = message; + const pendingRequest = this.#pendingRequests.get(seq); + if (!pendingRequest) { + throw new Error(`Received response for unknown request seq=${seq}`); + } + this.#pendingRequests.delete(seq); + if (success) { + pendingRequest.resolve(result); + } else { + pendingRequest.reject(result); + } + } + } catch (error) { + this.#logger.error('Invalid message received', error); + } }; + addRequestHandler( + method: string, + schema: z.ZodType, + handler: (params: TParams) => unknown, + ) { + this.#requestHandlers.set(method, { schema, handler }); + } + + async request( + method: string, + params: TReq, + ): Promise { + return new Promise((resolve, reject) => { + const seq = this.#seq++; + this.#pendingRequests.set(seq, { resolve, reject }); + this.#sendMessage('req', seq, method, params); + }); + } + + #sendMessage(...message: JsonValue[]) { + this.#ws.send(JSON.stringify(message)); + } + close() { this.#removeListeners(); this.#ws.close(); @@ -162,7 +312,7 @@ export class EventHub { return this.#handler; } - #handleGetConnect: Handler = async (req, _res) => { + #handleGetConnect: Handler = async (req, _res, next) => { try { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], @@ -179,6 +329,20 @@ export class EventHub { logger: this.#logger, credentials, }); + conn.addRequestHandler( + 'subscribe', + subscribeParamsSchema, + async params => { + console.log(`DEBUG: subscribe req`, params); + }, + ); + conn.addRequestHandler( + 'publish', + publishParamsSchema, + async params => { + console.log(`DEBUG: publish req`, params); + }, + ); this.#connections.set(conn.id, conn); }, ); diff --git a/yarn.lock b/yarn.lock index 18e926a14e..8f93a53290 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6143,14 +6143,18 @@ __metadata: "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/plugin-events-backend-test-utils": "workspace:^" "@backstage/plugin-events-node": "workspace:^" + "@backstage/types": "workspace:^" "@types/express": ^4.17.6 express: ^4.17.1 express-promise-router: ^4.1.0 supertest: ^7.0.0 winston: ^3.2.1 ws: ^8.17.0 + zod: ^3.22.4 + zod-validation-error: ^3.3.0 languageName: unknown linkType: soft @@ -44662,12 +44666,12 @@ __metadata: languageName: node linkType: hard -"zod-validation-error@npm:^3.0.3": - version: 3.1.0 - resolution: "zod-validation-error@npm:3.1.0" +"zod-validation-error@npm:^3.0.3, zod-validation-error@npm:^3.3.0": + version: 3.3.0 + resolution: "zod-validation-error@npm:3.3.0" peerDependencies: zod: ^3.18.0 - checksum: 84df01c91d594701eaf7f5f007be881e47f7adef2e3f3765f7be031cb78033f9be0924273106cb81b586d8020da9885dbb81b3da363f00a51df00f26274f2b23 + checksum: cbf81ecd27df675d72883b69833565af787302e70ad970ae4a5dab84e1cb8739cedf094b35f7f4b78307adaadb7cab0c0a8f7debeb6516e3fee998a3d4e13422 languageName: node linkType: hard From 6b148372c1f883a7c3713e8a73d1f6adeaeb3cdd Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 22 May 2024 14:23:10 +0200 Subject: [PATCH 06/83] events-backend: initial events store Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 13 +- .../src/service/hub/EventHub.ts | 166 +++++++++++++++++- 2 files changed, 172 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 445fa4b4b9..33bb01a985 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -94,9 +94,20 @@ backend.add( { id: 'derp', topics: ['test'] }, ]), ); + setTimeout(() => { + console.log(`DEBUG: publish!`); + ws.send( + JSON.stringify([ + 'req', + 1, + 'publish', + { topic: 'test', payload: { foo: 'bar' } }, + ]), + ); + }, 1000); }; ws.onmessage = event => { - console.log(`DEBUG: event=`, event.data); + console.log(`DEBUG: client event=`, event.data); }; ws.onerror = event => { console.log(`Client error`, event.error); diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 21692ca10f..7cb68ce808 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -29,6 +29,7 @@ import { z, ZodError } from 'zod'; import { fromZodError } from 'zod-validation-error'; import { JsonObject, JsonValue } from '@backstage/types'; import { serializeError } from '@backstage/errors'; +import { EventParams } from '@backstage/plugin-events-node'; /* @@ -85,8 +86,13 @@ const publishParamsSchema = z.object({ payload: z.any(), }); const eventParamsSchema = z.object({ - topic: z.string().min(1), - payload: z.any(), + events: z.array( + z.object({ + topic: z.string().min(1), + payload: z.any(), + metadata: z.any().optional(), + }), + ), }); function errorToJson(error: Error): JsonObject { @@ -96,6 +102,106 @@ function errorToJson(error: Error): JsonObject { return serializeError(error); } +type EventHubStore = { + publish(options: { + params: EventParams; + subscriberIds: string[]; + }): Promise; + + upsertSubscription(id: string, topics: string[]): Promise; + + readSubscription(id: string): Promise<{ events: EventParams[] }>; + + listen( + topicIds: string[], + onNotify: (topicId: string) => void, + ): Promise<() => void>; +}; + +const MAX_BATCH_SIZE = 5; + +class MemoryEventHubStore implements EventHubStore { + #events = new Array(); + #subscribers = new Map< + string, + { id: string; seq: number; topics: Set } + >(); + #listeners = new Set<{ + topics: Set; + notify(topicId: string): void; + }>(); + + async publish(options: { + params: EventParams; + subscriberIds: string[]; + }): Promise { + const topicId = options.params.topic; + + let hasOtherSubscribers = false; + for (const sub of this.#subscribers.values()) { + if (sub.topics.has(topicId) && !options.subscriberIds.includes(sub.id)) { + hasOtherSubscribers = true; + break; + } + } + if (!hasOtherSubscribers) { + return; + } + + const nextSeq = this.#getMaxSeq() + 1; + this.#events.push({ ...options.params, seq: nextSeq }); + + for (const listener of this.#listeners) { + if (listener.topics.has(topicId)) { + listener.notify(topicId); + } + } + } + + #getMaxSeq() { + return this.#events[this.#events.length - 1]?.seq ?? 0; + } + + async upsertSubscription(id: string, topics: string[]): Promise { + const existing = this.#subscribers.get(id); + if (existing) { + existing.topics = new Set(topics); + return; + } + const sub = { + id: id, + seq: this.#getMaxSeq(), + topics: new Set(topics), + }; + this.#subscribers.set(id, sub); + } + + async readSubscription(id: string): Promise<{ events: EventParams[] }> { + const sub = this.#subscribers.get(id); + if (!sub) { + throw new Error(`Subscription not found`); + } + const events = this.#events + .filter(event => event.seq > sub.seq && sub.topics.has(event.topic)) + .slice(0, MAX_BATCH_SIZE); + + sub.seq = events[events.length - 1]?.seq ?? sub.seq; + + return { events: events.map(event => ({ ...event, req: undefined })) }; + } + + async listen( + topicIds: string[], + onNotify: (topicId: string) => void, + ): Promise<() => void> { + const listener = { topics: new Set(topicIds), notify: onNotify }; + this.#listeners.add(listener); + return () => { + this.#listeners.delete(listener); + }; + } +} + /** * Manages a single WebSocket connection. * @@ -230,7 +336,7 @@ class EventClientConnection { this.#requestHandlers.set(method, { schema, handler }); } - async request( + async request( method: string, params: TReq, ): Promise { @@ -293,6 +399,7 @@ export class EventHub { readonly #handler: Handler; readonly #logger: LoggerService; readonly #httpAuth: HttpAuthService; + readonly #store: EventHubStore; #connections = new Map(); @@ -306,13 +413,14 @@ export class EventHub { this.#handler = handler; this.#logger = logger; this.#httpAuth = httpAuth; + this.#store = new MemoryEventHubStore(); } handler(): Handler { return this.#handler; } - #handleGetConnect: Handler = async (req, _res, next) => { + #handleGetConnect: Handler = async (req, _res) => { try { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], @@ -333,14 +441,60 @@ export class EventHub { 'subscribe', subscribeParamsSchema, async params => { - console.log(`DEBUG: subscribe req`, params); + await this.#store.upsertSubscription(params.id, params.topics); + + this.#logger.info( + `New subscription '${params.id}' topics='${params.topics.join( + "', '", + )}'`, + ); + + const read = () => + this.#store.readSubscription(params.id).then( + ({ events }) => { + if (events.length > 0) { + conn + .request, void>( + 'events', + { events }, + ) + .catch(error => { + this.#logger.error( + `Failed to send events to subscription ${params.id}`, + error, + ); + }); + } + }, + error => { + this.#logger.error( + `Failed to read subscription ${params.id}`, + error, + ); + }, + ); + + const removeListener = await this.#store.listen( + params.topics, + read, + ); + ws.addListener('close', removeListener); + + await read(); }, ); conn.addRequestHandler( 'publish', publishParamsSchema, async params => { - console.log(`DEBUG: publish req`, params); + await this.#store.publish({ + params: { + topic: params.topic, + eventPayload: params.payload, + }, + subscriberIds: [], + }); + this.#logger.info(`Published event to '${params.topic}'`); }, ); this.#connections.set(conn.id, conn); From a90ce4aa39037cb84e58bc6ce4151b20bdeb398a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 22 May 2024 14:25:31 +0200 Subject: [PATCH 07/83] events-node: update EventParams type Signed-off-by: Patrik Oldsberg --- .changeset/stale-roses-serve.md | 5 +++++ plugins/events-node/package.json | 3 ++- plugins/events-node/src/api/EventParams.ts | 6 ++++-- yarn.lock | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 .changeset/stale-roses-serve.md diff --git a/.changeset/stale-roses-serve.md b/.changeset/stale-roses-serve.md new file mode 100644 index 0000000000..dc1ab50f88 --- /dev/null +++ b/.changeset/stale-roses-serve.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-events-node': patch +--- + +The `EventParams` payload must now be a `JsonValue`, and the `metadata` type has been tweaked. diff --git a/plugins/events-node/package.json b/plugins/events-node/package.json index 6795c13441..9f822c7a94 100644 --- a/plugins/events-node/package.json +++ b/plugins/events-node/package.json @@ -51,7 +51,8 @@ "test": "backstage-cli package test" }, "dependencies": { - "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/types": "workspace:^" }, "devDependencies": { "@backstage/backend-common": "^0.25.0", diff --git a/plugins/events-node/src/api/EventParams.ts b/plugins/events-node/src/api/EventParams.ts index 19a562aee0..9cd88b160f 100644 --- a/plugins/events-node/src/api/EventParams.ts +++ b/plugins/events-node/src/api/EventParams.ts @@ -14,10 +14,12 @@ * limitations under the License. */ +import { JsonValue } from '@backstage/types'; + /** * @public */ -export interface EventParams { +export interface EventParams { /** * Topic for which this event should be published. */ @@ -29,5 +31,5 @@ export interface EventParams { /** * Metadata (e.g., HTTP headers and similar for events received from external). */ - metadata?: Record; + metadata?: { [name in string]?: string | string[] }; } diff --git a/yarn.lock b/yarn.lock index 8f93a53290..f6a430f9af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6166,6 +6166,7 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" + "@backstage/types": "workspace:^" languageName: unknown linkType: soft From ee52e38a04aca350b2301a6da6a77babce728238 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 22 May 2024 18:34:56 +0200 Subject: [PATCH 08/83] events-backend: add hub http subscription API + listen by subscription ID Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 49 ++++++++++++--- .../src/service/hub/EventHub.ts | 63 ++++++++++++++++--- 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 33bb01a985..77aa379744 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -79,6 +79,39 @@ backend.add( onBehalfOf: await auth.getOwnServiceCredentials(), targetPluginId: 'events', }); + + const subRes = await fetch(`${baseUrl}/hub/subscriptions/123`, { + method: 'PUT', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + topics: ['test'], + }), + }); + console.log( + `DEBUG: sub create req = ${subRes.status} ${subRes.statusText}`, + ); + + const poll = async () => { + const res = await fetch(`${baseUrl}/hub/subscriptions/123`, { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + }); + + const data = res.status === 200 && (await res.json()); + console.log( + `DEBUG: sub poll req = ${res.status} ${res.statusText}`, + data, + ); + poll(); + }; + + poll(); + const ws = new WebSocket(`${baseUrl}/hub/connect`, { headers: { Authorization: `Bearer ${token}`, @@ -86,14 +119,14 @@ backend.add( }); ws.onopen = () => { console.log('DEBUG: ws.onopen'); - ws.send( - JSON.stringify([ - 'req', - 1, - 'subscribe', - { id: 'derp', topics: ['test'] }, - ]), - ); + // ws.send( + // JSON.stringify([ + // 'req', + // 1, + // 'subscribe', + // { id: 'derp', topics: ['test'] }, + // ]), + // ); setTimeout(() => { console.log(`DEBUG: publish!`); ws.send( diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 7cb68ce808..c4d0a785d3 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -20,7 +20,7 @@ import { HttpAuthService, LoggerService, } from '@backstage/backend-plugin-api'; -import { Handler } from 'express'; +import express, { Handler } from 'express'; import Router from 'express-promise-router'; import { Socket } from 'net'; import { STATUS_CODES } from 'http'; @@ -113,7 +113,7 @@ type EventHubStore = { readSubscription(id: string): Promise<{ events: EventParams[] }>; listen( - topicIds: string[], + subscriptionId: string, onNotify: (topicId: string) => void, ): Promise<() => void>; }; @@ -191,10 +191,14 @@ class MemoryEventHubStore implements EventHubStore { } async listen( - topicIds: string[], + subscriptionId: string, onNotify: (topicId: string) => void, ): Promise<() => void> { - const listener = { topics: new Set(topicIds), notify: onNotify }; + const sub = this.#subscribers.get(subscriptionId); + if (!sub) { + throw new Error(`Subscription not found`); + } + const listener = { topics: sub.topics, notify: onNotify }; this.#listeners.add(listener); return () => { this.#listeners.delete(listener); @@ -390,8 +394,15 @@ export class EventHub { const hub = new EventHub(server, router, logger, httpAuth); + // WS router.get('/connect', hub.#handleGetConnect); + router.use(express.json()); + + // Long-polling + router.get('/subscriptions/:id', hub.#handleGetSubscription); + router.put('/subscriptions/:id', hub.#handlePutSubscription); + return hub; } @@ -474,10 +485,7 @@ export class EventHub { }, ); - const removeListener = await this.#store.listen( - params.topics, - read, - ); + const removeListener = await this.#store.listen(params.id, read); ws.addListener('close', removeListener); await read(); @@ -518,6 +526,45 @@ export class EventHub { } }; + #handleGetSubscription: Handler = async (req, res) => { + const credentials = await this.#httpAuth.credentials(req, { + allow: ['service'], + }); + const id = req.params.id; + + const { events } = await this.#store.readSubscription(id); + + this.#logger.info( + `Reading subscription '${id}' resulted in ${events.length} events`, + { subject: credentials.principal.subject }, + ); + + if (events.length > 0) { + res.json({ events }); + return; + } + + this.#store.listen(id, () => { + res.status(204).end(); + }); + }; + + #handlePutSubscription: Handler = async (req, res) => { + const credentials = await this.#httpAuth.credentials(req, { + allow: ['service'], + }); + const id = req.params.id; + + await this.#store.upsertSubscription(id, req.body.topics); + + this.#logger.info( + `New subscription '${id}' topics='${req.body.topics.join("', '")}'`, + { subject: credentials.principal.subject }, + ); + + res.status(201).end(); + }; + close() { this.#connections.forEach(conn => conn.close()); this.#connections.clear(); From ea4b912f93847fc38b0fa98313ac58e476eba57b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 22 May 2024 18:42:39 +0200 Subject: [PATCH 09/83] events-backend: add hub HTTP publish Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 29 +++++++++++++++---- .../src/service/hub/EventHub.ts | 21 +++++++++++++- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 77aa379744..752b88fb1f 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -95,12 +95,15 @@ backend.add( ); const poll = async () => { - const res = await fetch(`${baseUrl}/hub/subscriptions/123`, { - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', + const res = await fetch( + `${baseUrl}/hub/subscriptions/123/events`, + { + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, }, - }); + ); const data = res.status === 200 && (await res.json()); console.log( @@ -109,9 +112,23 @@ backend.add( ); poll(); }; - poll(); + setTimeout(() => { + console.log(`DEBUG: publishing!`); + fetch(`${baseUrl}/hub/events`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + topic: 'test', + payload: { herp: 'derp' }, + }), + }); + }, 500); + const ws = new WebSocket(`${baseUrl}/hub/connect`, { headers: { Authorization: `Bearer ${token}`, diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index c4d0a785d3..584abffc9b 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -399,8 +399,10 @@ export class EventHub { router.use(express.json()); + router.post('/events', hub.#handlePostEvents); + // Long-polling - router.get('/subscriptions/:id', hub.#handleGetSubscription); + router.get('/subscriptions/:id/events', hub.#handleGetSubscription); router.put('/subscriptions/:id', hub.#handlePutSubscription); return hub; @@ -526,6 +528,23 @@ export class EventHub { } }; + #handlePostEvents: Handler = async (req, res) => { + const credentials = await this.#httpAuth.credentials(req, { + allow: ['service'], + }); + await this.#store.publish({ + params: { + topic: req.body.topic, + eventPayload: req.body.payload, + }, + subscriberIds: [], + }); + this.#logger.info(`Published event to '${req.body.topic}'`, { + subject: credentials.principal.subject, + }); + res.status(201).end(); + }; + #handleGetSubscription: Handler = async (req, res) => { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], From acdefea2a1e0f4d9aeaaa8b26a78cce397dc93e2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 22 May 2024 20:19:00 +0200 Subject: [PATCH 10/83] events-backend: add OpenAPI schema Signed-off-by: Patrik Oldsberg --- plugins/events-backend/package.json | 3 + .../src/schema/openapi.generated.ts | 289 ++++++++++++++++++ .../events-backend/src/schema/openapi.yaml | 185 +++++++++++ .../src/service/EventsPlugin.ts | 2 +- .../src/service/hub/EventHub.ts | 48 ++- yarn.lock | 2 + 6 files changed, 515 insertions(+), 14 deletions(-) create mode 100644 plugins/events-backend/src/schema/openapi.generated.ts create mode 100644 plugins/events-backend/src/schema/openapi.yaml diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index dc7c2b0110..3160e6d494 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -44,6 +44,7 @@ "scripts": { "build": "backstage-cli package build", "clean": "backstage-cli package clean", + "generate": "backstage-repo-tools package schema openapi generate --server", "lint": "backstage-cli package lint", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", @@ -52,6 +53,7 @@ }, "dependencies": { "@backstage/backend-common": "^0.25.0", + "@backstage/backend-openapi-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", @@ -70,6 +72,7 @@ "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", "@backstage/plugin-events-backend-test-utils": "workspace:^", + "@backstage/repo-tools": "workspace:^", "supertest": "^7.0.0" }, "configSchema": "config.d.ts" diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts new file mode 100644 index 0000000000..7d17ad2bf6 --- /dev/null +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -0,0 +1,289 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { createValidatedOpenApiRouter } from '@backstage/backend-openapi-utils'; + +export const spec = { + openapi: '3.0.3', + info: { + title: 'events', + version: '1', + description: 'The Backstage backend plugin that powers the events system.', + license: { + name: 'Apache-2.0', + url: 'http://www.apache.org/licenses/LICENSE-2.0.html', + }, + contact: {}, + }, + servers: [ + { + url: '/', + }, + ], + components: { + examples: {}, + headers: {}, + parameters: { + subscriptionId: { + name: 'subscriptionId', + in: 'path', + required: true, + allowReserved: true, + schema: { + type: 'string', + }, + }, + }, + requestBodies: {}, + responses: { + ErrorResponse: { + description: 'An error response from the backend.', + content: { + 'application/json; charset=utf-8': { + schema: { + $ref: '#/components/schemas/Error', + }, + }, + }, + }, + }, + schemas: { + Event: { + type: 'object', + required: ['topic', 'payload'], + properties: { + topic: { + type: 'string', + description: 'The topic that the event is published on', + }, + payload: { + $ref: '#/components/schemas/JsonObject', + description: 'The event payload', + }, + }, + }, + Error: { + type: 'object', + properties: { + error: { + type: 'object', + properties: { + name: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + required: ['name', 'message'], + }, + request: { + type: 'object', + properties: { + method: { + type: 'string', + }, + url: { + type: 'string', + }, + }, + required: ['method', 'url'], + }, + response: { + type: 'object', + properties: { + statusCode: { + type: 'number', + }, + }, + required: ['statusCode'], + }, + }, + required: ['error', 'request', 'response'], + }, + JsonObject: { + type: 'object', + properties: {}, + additionalProperties: {}, + }, + }, + securitySchemes: { + JWT: { + type: 'http', + scheme: 'bearer', + bearerFormat: 'JWT', + }, + }, + }, + paths: { + '/hub/events': { + post: { + operationId: 'PostEvent', + description: 'Publish a new event', + responses: { + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['event'], + properties: { + event: { + $ref: '#/components/schemas/Event', + }, + subscriptionIds: { + type: 'array', + description: + 'The IDs of subscriptions that have already received this event', + items: { + type: 'string', + }, + }, + }, + }, + examples: { + 'Publishing a simple Event': { + value: { + event: { + topic: 'test-topic', + payload: { + myData: 'foo', + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + '/hub/subscriptions/{subscriptionId}': { + put: { + operationId: 'PutSubscription', + description: + 'Ensures that the subscription exists with the provided configuration', + responses: { + '201': { + description: 'The subscription exists or was created successfully', + }, + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + parameters: [ + { + $ref: '#/components/parameters/subscriptionId', + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['topics'], + properties: { + topics: { + type: 'array', + description: 'The topics to subscribe to', + items: { + type: 'string', + }, + }, + }, + }, + examples: { + 'Subscribing to a single topic': { + value: { + topics: ['test-topic'], + }, + }, + }, + }, + }, + }, + }, + }, + '/hub/subscriptions/{subscriptionId}/events': { + get: { + operationId: 'GetSubscriptionEvents', + description: 'Get new events for the provided subscription', + responses: { + '200': { + description: 'New events', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + events: { + type: 'array', + items: { + $ref: '#/components/schemas/Event', + }, + }, + }, + required: ['results'], + }, + }, + }, + }, + '201': { + description: 'Block poll response, new events are available', + }, + default: { + $ref: '#/components/responses/ErrorResponse', + }, + }, + security: [ + {}, + { + JWT: [], + }, + ], + parameters: [ + { + $ref: '#/components/parameters/subscriptionId', + }, + ], + }, + }, + }, +} as const; +export const createOpenApiRouter = async ( + options?: Parameters['1'], +) => createValidatedOpenApiRouter(spec, options); diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml new file mode 100644 index 0000000000..1adee01ab5 --- /dev/null +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -0,0 +1,185 @@ +openapi: 3.0.3 +info: + title: events + version: '1' + description: The Backstage backend plugin that powers the events system. + license: + name: Apache-2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + contact: {} +servers: + - url: / +components: + examples: {} + headers: {} + parameters: + subscriptionId: + name: subscriptionId + in: path + required: true + allowReserved: true + schema: + type: string + requestBodies: {} + responses: + ErrorResponse: + description: An error response from the backend. + content: + application/json; charset=utf-8: + schema: + $ref: '#/components/schemas/Error' + schemas: + Event: + type: object + required: + - topic + - payload + properties: + topic: + type: string + description: The topic that the event is published on + payload: + $ref: '#/components/schemas/JsonObject' + description: The event payload + + Error: + type: object + properties: + error: + type: object + properties: + name: + type: string + message: + type: string + required: + - name + - message + request: + type: object + properties: + method: + type: string + url: + type: string + required: + - method + - url + response: + type: object + properties: + statusCode: + type: number + required: + - statusCode + required: + - error + - request + - response + + JsonObject: + type: object + properties: {} + # Free form object. + additionalProperties: {} + securitySchemes: + JWT: + type: http + scheme: bearer + bearerFormat: JWT +paths: + /hub/events: + post: + operationId: PostEvent + description: Publish a new event + responses: + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - event + properties: + event: + $ref: '#/components/schemas/Event' + subscriptionIds: + type: array + description: The IDs of subscriptions that have already received this event + items: + type: string + examples: + Publishing a simple Event: + value: + event: + topic: test-topic + payload: + myData: foo + + /hub/subscriptions/{subscriptionId}: + put: + operationId: PutSubscription + description: Ensures that the subscription exists with the provided configuration + responses: + '201': + description: The subscription exists or was created successfully + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + parameters: + - $ref: '#/components/parameters/subscriptionId' + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - topics + properties: + topics: + type: array + description: The topics to subscribe to + items: + type: string + examples: + Subscribing to a single topic: + value: + topics: + - test-topic + + /hub/subscriptions/{subscriptionId}/events: + get: + operationId: GetSubscriptionEvents + description: Get new events for the provided subscription + responses: + '200': + description: New events + content: + application/json: + schema: + type: object + properties: + events: + type: array + items: + $ref: '#/components/schemas/Event' + required: + - results + '201': + description: Block poll response, new events are available + default: + $ref: '#/components/responses/ErrorResponse' + security: + - {} + - JWT: [] + parameters: + - $ref: '#/components/parameters/subscriptionId' diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index ecb234066d..b7892a5079 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -97,7 +97,7 @@ export const eventsPlugin = createBackendPlugin({ http.bind(eventsRouter); const hub = await EventHub.create({ logger, httpAuth }); - eventsRouter.use('/hub', hub.handler()); + router.use(hub.handler()); router.use(eventsRouter); router.addAuthPolicy({ diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 584abffc9b..d20401c98f 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -20,7 +20,7 @@ import { HttpAuthService, LoggerService, } from '@backstage/backend-plugin-api'; -import express, { Handler } from 'express'; +import { Handler } from 'express'; import Router from 'express-promise-router'; import { Socket } from 'net'; import { STATUS_CODES } from 'http'; @@ -30,6 +30,8 @@ import { fromZodError } from 'zod-validation-error'; import { JsonObject, JsonValue } from '@backstage/types'; import { serializeError } from '@backstage/errors'; import { EventParams } from '@backstage/plugin-events-node'; +import { spec, createOpenApiRouter } from '../../schema/openapi.generated'; +import { internal } from '@backstage/backend-openapi-utils'; /* @@ -395,15 +397,23 @@ export class EventHub { const hub = new EventHub(server, router, logger, httpAuth); // WS - router.get('/connect', hub.#handleGetConnect); + router.get('/hub/connect', hub.#handleGetConnect); - router.use(express.json()); + const apiRouter = await createOpenApiRouter(); - router.post('/events', hub.#handlePostEvents); + router.use(apiRouter); + + apiRouter.post('/hub/events', hub.#handlePostEvents); // Long-polling - router.get('/subscriptions/:id/events', hub.#handleGetSubscription); - router.put('/subscriptions/:id', hub.#handlePutSubscription); + apiRouter.get( + '/hub/subscriptions/:subscriptionId/events', + hub.#handleGetSubscription, + ); + apiRouter.put( + '/hub/subscriptions/:subscriptionId', + hub.#handlePutSubscription, + ); return hub; } @@ -528,14 +538,18 @@ export class EventHub { } }; - #handlePostEvents: Handler = async (req, res) => { + #handlePostEvents: internal.DocRequestHandler< + typeof spec, + '/hub/events', + 'post' + > = async (req, res) => { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); await this.#store.publish({ params: { - topic: req.body.topic, - eventPayload: req.body.payload, + topic: req.body.event.topic, + eventPayload: req.body.event.payload, }, subscriberIds: [], }); @@ -545,11 +559,15 @@ export class EventHub { res.status(201).end(); }; - #handleGetSubscription: Handler = async (req, res) => { + #handleGetSubscription: internal.DocRequestHandler< + typeof spec, + '/hub/subscriptions/{subscriptionId}/events', + 'get' + > = async (req, res) => { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); - const id = req.params.id; + const id = req.params.subscriptionId; const { events } = await this.#store.readSubscription(id); @@ -568,11 +586,15 @@ export class EventHub { }); }; - #handlePutSubscription: Handler = async (req, res) => { + #handlePutSubscription: internal.DocRequestHandler< + typeof spec, + '/hub/subscriptions/{subscriptionId}', + 'put' + > = async (req, res) => { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); - const id = req.params.id; + const id = req.params.subscriptionId; await this.#store.upsertSubscription(id, req.body.topics); diff --git a/yarn.lock b/yarn.lock index f6a430f9af..d1024c0d96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6139,6 +6139,7 @@ __metadata: dependencies: "@backstage/backend-common": ^0.25.0 "@backstage/backend-defaults": "workspace:^" + "@backstage/backend-openapi-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" @@ -6146,6 +6147,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/plugin-events-backend-test-utils": "workspace:^" "@backstage/plugin-events-node": "workspace:^" + "@backstage/repo-tools": "workspace:^" "@backstage/types": "workspace:^" "@types/express": ^4.17.6 express: ^4.17.1 From b8be639b9a9800217bee0fad99ab23cd46b8b1a1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 23 May 2024 01:31:10 +0200 Subject: [PATCH 11/83] events-node: generate client Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 73 ++------- plugins/events-backend/package.json | 2 +- plugins/events-node/package.json | 4 +- .../src/generated/apis/DefaultApi.client.ts | 152 ++++++++++++++++++ .../events-node/src/generated/apis/index.ts | 17 ++ plugins/events-node/src/generated/index.ts | 18 +++ .../src/generated/models/ErrorError.model.ts | 24 +++ .../generated/models/ErrorRequest.model.ts | 24 +++ .../generated/models/ErrorResponse.model.ts | 23 +++ .../src/generated/models/Event.model.ts | 27 ++++ .../GetSubscriptionEvents200Response.model.ts | 24 +++ .../src/generated/models/ModelError.model.ts | 28 ++++ .../models/PostEventRequest.model.ts | 28 ++++ .../models/PutSubscriptionRequest.model.ts | 26 +++ .../events-node/src/generated/models/index.ts | 24 +++ plugins/events-node/src/generated/pluginId.ts | 17 ++ .../src/generated/types/discovery.ts | 22 +++ .../events-node/src/generated/types/fetch.ts | 22 +++ yarn.lock | 2 + 19 files changed, 496 insertions(+), 61 deletions(-) create mode 100644 plugins/events-node/src/generated/apis/DefaultApi.client.ts create mode 100644 plugins/events-node/src/generated/apis/index.ts create mode 100644 plugins/events-node/src/generated/index.ts create mode 100644 plugins/events-node/src/generated/models/ErrorError.model.ts create mode 100644 plugins/events-node/src/generated/models/ErrorRequest.model.ts create mode 100644 plugins/events-node/src/generated/models/ErrorResponse.model.ts create mode 100644 plugins/events-node/src/generated/models/Event.model.ts create mode 100644 plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts create mode 100644 plugins/events-node/src/generated/models/ModelError.model.ts create mode 100644 plugins/events-node/src/generated/models/PostEventRequest.model.ts create mode 100644 plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts create mode 100644 plugins/events-node/src/generated/models/index.ts create mode 100644 plugins/events-node/src/generated/pluginId.ts create mode 100644 plugins/events-node/src/generated/types/discovery.ts create mode 100644 plugins/events-node/src/generated/types/fetch.ts diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 752b88fb1f..9476da1d0d 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -21,6 +21,7 @@ import { } from '@backstage/backend-plugin-api'; import { WebSocket } from 'ws'; import { eventsServiceRef } from '@backstage/plugin-events-node'; +import { DefaultApiClient } from '../../events-node/src/generated'; const backend = createBackend(); @@ -73,6 +74,8 @@ backend.add( rootLifecycle.addStartupHook(async () => { logger.info('Started!'); + + const client = new DefaultApiClient({ discoveryApi: discovery }); const baseUrl = await discovery.getBaseUrl('events'); console.log(`DEBUG: baseUrl=`, baseUrl); const { token } = await auth.getPluginRequestToken({ @@ -80,29 +83,23 @@ backend.add( targetPluginId: 'events', }); - const subRes = await fetch(`${baseUrl}/hub/subscriptions/123`, { - method: 'PUT', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', + const subRes = await client.putSubscription( + { + path: { subscriptionId: '123' }, + body: { topics: ['test'] }, }, - body: JSON.stringify({ - topics: ['test'], - }), - }); + { token }, + ); console.log( `DEBUG: sub create req = ${subRes.status} ${subRes.statusText}`, ); const poll = async () => { - const res = await fetch( - `${baseUrl}/hub/subscriptions/123/events`, + const res = await client.getSubscriptionEvents( { - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, + path: { subscriptionId: '123' }, }, + { token }, ); const data = res.status === 200 && (await res.json()); @@ -116,52 +113,10 @@ backend.add( setTimeout(() => { console.log(`DEBUG: publishing!`); - fetch(`${baseUrl}/hub/events`, { - method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - topic: 'test', - payload: { herp: 'derp' }, - }), + client.postEvent({ + body: { event: { topic: 'test', payload: { foo: 'bar' } } }, }); }, 500); - - const ws = new WebSocket(`${baseUrl}/hub/connect`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - ws.onopen = () => { - console.log('DEBUG: ws.onopen'); - // ws.send( - // JSON.stringify([ - // 'req', - // 1, - // 'subscribe', - // { id: 'derp', topics: ['test'] }, - // ]), - // ); - setTimeout(() => { - console.log(`DEBUG: publish!`); - ws.send( - JSON.stringify([ - 'req', - 1, - 'publish', - { topic: 'test', payload: { foo: 'bar' } }, - ]), - ); - }, 1000); - }; - ws.onmessage = event => { - console.log(`DEBUG: client event=`, event.data); - }; - ws.onerror = event => { - console.log(`Client error`, event.error); - }; }); }, }); diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index 3160e6d494..aa788330ae 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -44,7 +44,7 @@ "scripts": { "build": "backstage-cli package build", "clean": "backstage-cli package clean", - "generate": "backstage-repo-tools package schema openapi generate --server", + "generate": "backstage-repo-tools package schema openapi generate --server --client-package plugins/events-node", "lint": "backstage-cli package lint", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack", diff --git a/plugins/events-node/package.json b/plugins/events-node/package.json index 9f822c7a94..de880dc399 100644 --- a/plugins/events-node/package.json +++ b/plugins/events-node/package.json @@ -52,7 +52,9 @@ }, "dependencies": { "@backstage/backend-plugin-api": "workspace:^", - "@backstage/types": "workspace:^" + "@backstage/types": "workspace:^", + "cross-fetch": "^4.0.0", + "uri-template": "^2.0.0" }, "devDependencies": { "@backstage/backend-common": "^0.25.0", diff --git a/plugins/events-node/src/generated/apis/DefaultApi.client.ts b/plugins/events-node/src/generated/apis/DefaultApi.client.ts new file mode 100644 index 0000000000..7a2099e79a --- /dev/null +++ b/plugins/events-node/src/generated/apis/DefaultApi.client.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { DiscoveryApi } from '../types/discovery'; +import { FetchApi } from '../types/fetch'; +import crossFetch from 'cross-fetch'; +import { pluginId } from '../pluginId'; +import * as parser from 'uri-template'; + +import { GetSubscriptionEvents200Response } from '../models/GetSubscriptionEvents200Response.model'; +import { PostEventRequest } from '../models/PostEventRequest.model'; +import { PutSubscriptionRequest } from '../models/PutSubscriptionRequest.model'; + +/** + * Wraps the Response type to convey a type on the json call. + * + * @public + */ +export type TypedResponse = Omit & { + json: () => Promise; +}; + +/** + * Options you can pass into a request for additional information. + * + * @public + */ +export interface RequestOptions { + token?: string; +} + +/** + * no description + */ +export class DefaultApiClient { + private readonly discoveryApi: DiscoveryApi; + private readonly fetchApi: FetchApi; + + constructor(options: { + discoveryApi: { getBaseUrl(pluginId: string): Promise }; + fetchApi?: { fetch: typeof fetch }; + }) { + this.discoveryApi = options.discoveryApi; + this.fetchApi = options.fetchApi || { fetch: crossFetch }; + } + + /** + * Get new events for the provided subscription + * @param subscriptionId + */ + public async getSubscriptionEvents( + // @ts-ignore + request: { + path: { + subscriptionId: string; + }; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/hub/subscriptions/{subscriptionId}/events`; + + const uri = parser.parse(uriTemplate).expand({ + subscriptionId: request.path.subscriptionId, + }); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'GET', + }); + } + + /** + * Publish a new event + * @param postEventRequest + */ + public async postEvent( + // @ts-ignore + request: { + body: PostEventRequest; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/hub/events`; + + const uri = parser.parse(uriTemplate).expand({}); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'POST', + body: JSON.stringify(request.body), + }); + } + + /** + * Ensures that the subscription exists with the provided configuration + * @param subscriptionId + * @param putSubscriptionRequest + */ + public async putSubscription( + // @ts-ignore + request: { + path: { + subscriptionId: string; + }; + body: PutSubscriptionRequest; + }, + options?: RequestOptions, + ): Promise> { + const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); + + const uriTemplate = `/hub/subscriptions/{subscriptionId}`; + + const uri = parser.parse(uriTemplate).expand({ + subscriptionId: request.path.subscriptionId, + }); + + return await this.fetchApi.fetch(`${baseUrl}${uri}`, { + headers: { + 'Content-Type': 'application/json', + ...(options?.token && { Authorization: `Bearer ${options?.token}` }), + }, + method: 'PUT', + body: JSON.stringify(request.body), + }); + } +} diff --git a/plugins/events-node/src/generated/apis/index.ts b/plugins/events-node/src/generated/apis/index.ts new file mode 100644 index 0000000000..51dcca33fe --- /dev/null +++ b/plugins/events-node/src/generated/apis/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './DefaultApi.client'; diff --git a/plugins/events-node/src/generated/index.ts b/plugins/events-node/src/generated/index.ts new file mode 100644 index 0000000000..bb399e97a0 --- /dev/null +++ b/plugins/events-node/src/generated/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './apis'; +export * from './models'; diff --git a/plugins/events-node/src/generated/models/ErrorError.model.ts b/plugins/events-node/src/generated/models/ErrorError.model.ts new file mode 100644 index 0000000000..4bcc60b660 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorError.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorError { + name: string; + message: string; +} diff --git a/plugins/events-node/src/generated/models/ErrorRequest.model.ts b/plugins/events-node/src/generated/models/ErrorRequest.model.ts new file mode 100644 index 0000000000..be62627814 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorRequest.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorRequest { + method: string; + url: string; +} diff --git a/plugins/events-node/src/generated/models/ErrorResponse.model.ts b/plugins/events-node/src/generated/models/ErrorResponse.model.ts new file mode 100644 index 0000000000..008f8b7348 --- /dev/null +++ b/plugins/events-node/src/generated/models/ErrorResponse.model.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface ErrorResponse { + statusCode: number; +} diff --git a/plugins/events-node/src/generated/models/Event.model.ts b/plugins/events-node/src/generated/models/Event.model.ts new file mode 100644 index 0000000000..b5a13b4d8d --- /dev/null +++ b/plugins/events-node/src/generated/models/Event.model.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface Event { + /** + * The topic that the event is published on + */ + topic: string; + payload: { [key: string]: any }; +} diff --git a/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts new file mode 100644 index 0000000000..653bfaf3fc --- /dev/null +++ b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { Event } from '../models/Event.model'; + +export interface GetSubscriptionEvents200Response { + events?: Array; +} diff --git a/plugins/events-node/src/generated/models/ModelError.model.ts b/plugins/events-node/src/generated/models/ModelError.model.ts new file mode 100644 index 0000000000..88b8e76762 --- /dev/null +++ b/plugins/events-node/src/generated/models/ModelError.model.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { ErrorError } from '../models/ErrorError.model'; +import { ErrorRequest } from '../models/ErrorRequest.model'; +import { ErrorResponse } from '../models/ErrorResponse.model'; + +export interface ModelError { + error: ErrorError; + request: ErrorRequest; + response: ErrorResponse; +} diff --git a/plugins/events-node/src/generated/models/PostEventRequest.model.ts b/plugins/events-node/src/generated/models/PostEventRequest.model.ts new file mode 100644 index 0000000000..527064bc13 --- /dev/null +++ b/plugins/events-node/src/generated/models/PostEventRequest.model.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** +import { Event } from '../models/Event.model'; + +export interface PostEventRequest { + event: Event; + /** + * The IDs of subscriptions that have already received this event + */ + subscriptionIds?: Array; +} diff --git a/plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts b/plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts new file mode 100644 index 0000000000..3de35e0f81 --- /dev/null +++ b/plugins/events-node/src/generated/models/PutSubscriptionRequest.model.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// ****************************************************************** +// * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. * +// ****************************************************************** + +export interface PutSubscriptionRequest { + /** + * The topics to subscribe to + */ + topics: Array; +} diff --git a/plugins/events-node/src/generated/models/index.ts b/plugins/events-node/src/generated/models/index.ts new file mode 100644 index 0000000000..5044f65fff --- /dev/null +++ b/plugins/events-node/src/generated/models/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../models/ErrorError.model'; +export * from '../models/ErrorRequest.model'; +export * from '../models/ErrorResponse.model'; +export * from '../models/Event.model'; +export * from '../models/GetSubscriptionEvents200Response.model'; +export * from '../models/ModelError.model'; +export * from '../models/PostEventRequest.model'; +export * from '../models/PutSubscriptionRequest.model'; diff --git a/plugins/events-node/src/generated/pluginId.ts b/plugins/events-node/src/generated/pluginId.ts new file mode 100644 index 0000000000..36b5433cc3 --- /dev/null +++ b/plugins/events-node/src/generated/pluginId.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const pluginId = 'events'; diff --git a/plugins/events-node/src/generated/types/discovery.ts b/plugins/events-node/src/generated/types/discovery.ts new file mode 100644 index 0000000000..a7f87d3780 --- /dev/null +++ b/plugins/events-node/src/generated/types/discovery.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is a copy of the DiscoveryApi, to avoid importing core-plugin-api. + */ +export type DiscoveryApi = { + getBaseUrl(pluginId: string): Promise; +}; diff --git a/plugins/events-node/src/generated/types/fetch.ts b/plugins/events-node/src/generated/types/fetch.ts new file mode 100644 index 0000000000..3de56c028e --- /dev/null +++ b/plugins/events-node/src/generated/types/fetch.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This is a copy of FetchApi, to avoid importing core-plugin-api. + */ +export type FetchApi = { + fetch: typeof fetch; +}; diff --git a/yarn.lock b/yarn.lock index d1024c0d96..6636108958 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6169,6 +6169,8 @@ __metadata: "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/types": "workspace:^" + cross-fetch: ^4.0.0 + uri-template: ^2.0.0 languageName: unknown linkType: soft From 3a966aff5f3d7377139ba0731009506be2a5f7bb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 23 May 2024 09:51:15 +0200 Subject: [PATCH 12/83] events-backend: server-side implementation of local subscriber optimization Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 39 +++++++++++++++++-- .../src/service/hub/EventHub.ts | 22 +++++++---- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 9476da1d0d..47c2763a01 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -93,6 +93,16 @@ backend.add( console.log( `DEBUG: sub create req = ${subRes.status} ${subRes.statusText}`, ); + const subRes2 = await client.putSubscription( + { + path: { subscriptionId: 'abc' }, + body: { topics: ['test'] }, + }, + { token }, + ); + console.log( + `DEBUG: sub create req = ${subRes2.status} ${subRes2.statusText}`, + ); const poll = async () => { const res = await client.getSubscriptionEvents( @@ -111,11 +121,34 @@ backend.add( }; poll(); + const poll2 = async () => { + const res = await client.getSubscriptionEvents( + { + path: { subscriptionId: 'abc' }, + }, + { token }, + ); + + const data = res.status === 200 && (await res.json()); + console.log( + `DEBUG: sub poll2 req = ${res.status} ${res.statusText}`, + data, + ); + poll2(); + }; + poll2(); + setTimeout(() => { console.log(`DEBUG: publishing!`); - client.postEvent({ - body: { event: { topic: 'test', payload: { foo: 'bar' } } }, - }); + client.postEvent( + { + body: { + event: { topic: 'test', payload: { foo: 'bar' } }, + subscriptionIds: ['123'], + }, + }, + { token }, + ); }, 500); }); }, diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index d20401c98f..e8080ecb30 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -123,7 +123,9 @@ type EventHubStore = { const MAX_BATCH_SIZE = 5; class MemoryEventHubStore implements EventHubStore { - #events = new Array(); + #events = new Array< + EventParams & { seq: number; subscriberIds: Set } + >(); #subscribers = new Map< string, { id: string; seq: number; topics: Set } @@ -138,10 +140,11 @@ class MemoryEventHubStore implements EventHubStore { subscriberIds: string[]; }): Promise { const topicId = options.params.topic; + const subscriberIds = new Set(options.subscriberIds); let hasOtherSubscribers = false; for (const sub of this.#subscribers.values()) { - if (sub.topics.has(topicId) && !options.subscriberIds.includes(sub.id)) { + if (sub.topics.has(topicId) && !subscriberIds.has(sub.id)) { hasOtherSubscribers = true; break; } @@ -151,7 +154,7 @@ class MemoryEventHubStore implements EventHubStore { } const nextSeq = this.#getMaxSeq() + 1; - this.#events.push({ ...options.params, seq: nextSeq }); + this.#events.push({ ...options.params, subscriberIds, seq: nextSeq }); for (const listener of this.#listeners) { if (listener.topics.has(topicId)) { @@ -184,12 +187,17 @@ class MemoryEventHubStore implements EventHubStore { throw new Error(`Subscription not found`); } const events = this.#events - .filter(event => event.seq > sub.seq && sub.topics.has(event.topic)) + .filter( + event => + event.seq > sub.seq && + sub.topics.has(event.topic) && + !event.subscriberIds.has(id), + ) .slice(0, MAX_BATCH_SIZE); sub.seq = events[events.length - 1]?.seq ?? sub.seq; - return { events: events.map(event => ({ ...event, req: undefined })) }; + return { events: events.map(event => ({ ...event, seq: undefined })) }; } async listen( @@ -551,9 +559,9 @@ export class EventHub { topic: req.body.event.topic, eventPayload: req.body.event.payload, }, - subscriberIds: [], + subscriberIds: req.body.subscriptionIds ?? [], }); - this.#logger.info(`Published event to '${req.body.topic}'`, { + this.#logger.info(`Published event to '${req.body.event.topic}'`, { subject: credentials.principal.subject, }); res.status(201).end(); From aa0c6a389db08f0b2ae2e49aeb2ce3dbf8f2e1ad Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 23 May 2024 09:57:37 +0200 Subject: [PATCH 13/83] events-backend: remove hub WebSocket implementation Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 1 - plugins/events-backend/package.json | 5 +- .../src/service/hub/EventHub.ts | 373 +----------------- yarn.lock | 11 +- 4 files changed, 7 insertions(+), 383 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 47c2763a01..1c0d38215b 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -19,7 +19,6 @@ import { coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; -import { WebSocket } from 'ws'; import { eventsServiceRef } from '@backstage/plugin-events-node'; import { DefaultApiClient } from '../../events-node/src/generated'; diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index aa788330ae..2f9db1b081 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -62,10 +62,7 @@ "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "winston": "^3.2.1", - "ws": "^8.17.0", - "zod": "^3.22.4", - "zod-validation-error": "^3.3.0" + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-defaults": "workspace:^", diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index e8080ecb30..a014bb1add 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -14,96 +14,13 @@ * limitations under the License. */ -import { - BackstageCredentials, - BackstageServicePrincipal, - HttpAuthService, - LoggerService, -} from '@backstage/backend-plugin-api'; +import { HttpAuthService, LoggerService } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; -import { Socket } from 'net'; -import { STATUS_CODES } from 'http'; -import { WebSocketServer, type WebSocket, RawData } from 'ws'; -import { z, ZodError } from 'zod'; -import { fromZodError } from 'zod-validation-error'; -import { JsonObject, JsonValue } from '@backstage/types'; -import { serializeError } from '@backstage/errors'; import { EventParams } from '@backstage/plugin-events-node'; import { spec, createOpenApiRouter } from '../../schema/openapi.generated'; import { internal } from '@backstage/backend-openapi-utils'; -/* - -# Protocol - -## Request/Response - -General request/response format used for all communication: - --> [type: 'req', id: number, method: string, params: JsonObject] -<- [type: 'res', id: number, status: 'resolved' | 'rejected', result: JsonObject] - -## Client -> Server - -### Subscribe - --> method: 'subscribe', params: { id: string, topics: string[] } -<- result: void - -### Publish - --> method: 'publish', params: { topic: string, payload: JsonObject } -<- result: void - -## Server -> Client - -### Event - --> method: 'event', params: { topic: string, payload: JsonObject } -<- result: void - -*/ - -const messageSchema = z.union([ - z.tuple([ - z.literal('req'), - z.number().int().gt(0), - z.string().min(1), - z.any(), - ]), - z.tuple([ - z.literal('res'), - z.number().int().gt(0), - z.enum(['resolved', 'rejected']), - z.any(), - ]), -]); -const subscribeParamsSchema = z.object({ - id: z.string().min(1), - topics: z.array(z.string().min(1)), -}); -const publishParamsSchema = z.object({ - topic: z.string().min(1), - payload: z.any(), -}); -const eventParamsSchema = z.object({ - events: z.array( - z.object({ - topic: z.string().min(1), - payload: z.any(), - metadata: z.any().optional(), - }), - ), -}); - -function errorToJson(error: Error): JsonObject { - if (error.name === 'ZodError') { - return serializeError(fromZodError(error as ZodError)); - } - return serializeError(error); -} - type EventHubStore = { publish(options: { params: EventParams; @@ -216,174 +133,6 @@ class MemoryEventHubStore implements EventHubStore { } } -/** - * Manages a single WebSocket connection. - * - * @internal - */ -class EventClientConnection { - static create(options: { - ws: WebSocket; - socket: Socket; - logger: LoggerService; - credentials: BackstageCredentials; - }) { - const { ws, credentials } = options; - - const id = Math.random().toString(36).slice(2, 10); - const logger = options.logger.child({ - connection: id, - subject: credentials.principal.subject, - }); - - const conn = new EventClientConnection(id, ws, logger, credentials); - - ws.addListener('close', conn.#handleClose); - ws.addListener('error', conn.#handleError); - ws.addListener('message', conn.#handleMessage); - - logger.info( - `New event client connection from '${options.socket.remoteAddress}'`, - ); - - return conn; - } - - readonly #id: string; - readonly #ws: WebSocket; - readonly #logger: LoggerService; - readonly #credentials: BackstageCredentials; - - #seq = 1; - readonly #pendingRequests = new Map< - number, - { resolve(result: unknown): void; reject(error: unknown): void } - >(); - readonly #requestHandlers = new Map< - string, - { - schema: z.ZodType; - handler: (params: any) => unknown; - } - >(); - - constructor( - id: string, - ws: WebSocket, - logger: LoggerService, - credentials: BackstageCredentials, - ) { - this.#id = id; - this.#ws = ws; - this.#logger = logger; - this.#credentials = credentials; - } - - get id() { - return this.#id; - } - - #handleClose = (code: number, reason: Buffer) => { - this.#removeListeners(); - this.#logger.info(`Remote closed code=${code} reason=${reason}`); - }; - - #handleError = (error: Error) => { - this.#removeListeners(); - this.#logger.error(`WebSocket error`, error); - }; - - #handleMessage = (rawData: RawData, isBinary: boolean) => { - if (isBinary) { - return; - } - try { - const data = Array.isArray(rawData) - ? Buffer.concat(rawData) - : Buffer.from(rawData); - const message = messageSchema.parse(JSON.parse(data.toString('utf8'))); - - if (message[0] === 'req') { - const [, seq, method, params] = message; - const handler = this.#requestHandlers.get(method); - if (!handler) { - throw new Error(`Unknown method '${method}'`); - } - - try { - const parsedParams = handler.schema.parse(params); - - Promise.resolve(handler.handler(parsedParams)).then( - result => { - this.#sendMessage('res', seq, 'resolved', result ?? null); - }, - error => { - this.#sendMessage('res', seq, 'rejected', errorToJson(error)); - }, - ); - } catch (error) { - this.#sendMessage('res', seq, false, errorToJson(error)); - } - } else if (message[0] === 'res') { - const [, seq, success, result] = message; - const pendingRequest = this.#pendingRequests.get(seq); - if (!pendingRequest) { - throw new Error(`Received response for unknown request seq=${seq}`); - } - this.#pendingRequests.delete(seq); - if (success) { - pendingRequest.resolve(result); - } else { - pendingRequest.reject(result); - } - } - } catch (error) { - this.#logger.error('Invalid message received', error); - } - }; - - addRequestHandler( - method: string, - schema: z.ZodType, - handler: (params: TParams) => unknown, - ) { - this.#requestHandlers.set(method, { schema, handler }); - } - - async request( - method: string, - params: TReq, - ): Promise { - return new Promise((resolve, reject) => { - const seq = this.#seq++; - this.#pendingRequests.set(seq, { resolve, reject }); - this.#sendMessage('req', seq, method, params); - }); - } - - #sendMessage(...message: JsonValue[]) { - this.#ws.send(JSON.stringify(message)); - } - - close() { - this.#removeListeners(); - this.#ws.close(); - this.#logger.info(`Closed connection`); - } - - #removeListeners() { - this.#ws.removeListener('close', this.#handleClose); - this.#ws.removeListener('error', this.#handleError); - this.#ws.removeListener('message', this.#handleMessage); - } - - toString() { - return `eventClientConnection{id=${this.#id},subject=${ - this.#credentials.principal.subject - }}`; - } -} - export class EventHub { static async create(options: { logger: LoggerService; @@ -393,19 +142,7 @@ export class EventHub { const logger = options.logger.child({ type: 'EventHub' }); const router = Router(); - const server = new WebSocketServer({ - noServer: true, - clientTracking: false, - }); - - server.on('error', error => { - logger.error(`WebSocket server error`, error); - }); - - const hub = new EventHub(server, router, logger, httpAuth); - - // WS - router.get('/hub/connect', hub.#handleGetConnect); + const hub = new EventHub(router, logger, httpAuth); const apiRouter = await createOpenApiRouter(); @@ -426,21 +163,16 @@ export class EventHub { return hub; } - readonly #server: WebSocketServer; readonly #handler: Handler; readonly #logger: LoggerService; readonly #httpAuth: HttpAuthService; readonly #store: EventHubStore; - #connections = new Map(); - private constructor( - server: WebSocketServer, handler: Handler, logger: LoggerService, httpAuth: HttpAuthService, ) { - this.#server = server; this.#handler = handler; this.#logger = logger; this.#httpAuth = httpAuth; @@ -451,101 +183,6 @@ export class EventHub { return this.#handler; } - #handleGetConnect: Handler = async (req, _res) => { - try { - const credentials = await this.#httpAuth.credentials(req, { - allow: ['service'], - }); - - this.#server.handleUpgrade( - req, - req.socket, - Buffer.alloc(0), - (ws, { socket }) => { - const conn = EventClientConnection.create({ - ws, - socket, - logger: this.#logger, - credentials, - }); - conn.addRequestHandler( - 'subscribe', - subscribeParamsSchema, - async params => { - await this.#store.upsertSubscription(params.id, params.topics); - - this.#logger.info( - `New subscription '${params.id}' topics='${params.topics.join( - "', '", - )}'`, - ); - - const read = () => - this.#store.readSubscription(params.id).then( - ({ events }) => { - if (events.length > 0) { - conn - .request, void>( - 'events', - { events }, - ) - .catch(error => { - this.#logger.error( - `Failed to send events to subscription ${params.id}`, - error, - ); - }); - } - }, - error => { - this.#logger.error( - `Failed to read subscription ${params.id}`, - error, - ); - }, - ); - - const removeListener = await this.#store.listen(params.id, read); - ws.addListener('close', removeListener); - - await read(); - }, - ); - conn.addRequestHandler( - 'publish', - publishParamsSchema, - async params => { - await this.#store.publish({ - params: { - topic: params.topic, - eventPayload: params.payload, - }, - subscriberIds: [], - }); - this.#logger.info(`Published event to '${params.topic}'`); - }, - ); - this.#connections.set(conn.id, conn); - }, - ); - } catch (error) { - let status = 500; - if (error.name === 'AuthenticationError') { - status = 401; - } else if (error.name === 'NotAllowedError') { - status = 403; - } - req.socket.write( - `HTTP/1.1 ${status} ${STATUS_CODES[status]}\r\n` + - 'Upgrade: WebSocket\r\n' + - 'Connection: Upgrade\r\n' + - '\r\n', - ); - req.socket.destroy(); - this.#logger.info('WebSocket upgrade failed', error); - } - }; - #handlePostEvents: internal.DocRequestHandler< typeof spec, '/hub/events', @@ -613,10 +250,4 @@ export class EventHub { res.status(201).end(); }; - - close() { - this.#connections.forEach(conn => conn.close()); - this.#connections.clear(); - this.#server.close(); - } } diff --git a/yarn.lock b/yarn.lock index 6636108958..2aeb54be7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6154,9 +6154,6 @@ __metadata: express-promise-router: ^4.1.0 supertest: ^7.0.0 winston: ^3.2.1 - ws: ^8.17.0 - zod: ^3.22.4 - zod-validation-error: ^3.3.0 languageName: unknown linkType: soft @@ -44671,12 +44668,12 @@ __metadata: languageName: node linkType: hard -"zod-validation-error@npm:^3.0.3, zod-validation-error@npm:^3.3.0": - version: 3.3.0 - resolution: "zod-validation-error@npm:3.3.0" +"zod-validation-error@npm:^3.0.3": + version: 3.1.0 + resolution: "zod-validation-error@npm:3.1.0" peerDependencies: zod: ^3.18.0 - checksum: cbf81ecd27df675d72883b69833565af787302e70ad970ae4a5dab84e1cb8739cedf094b35f7f4b78307adaadb7cab0c0a8f7debeb6516e3fee998a3d4e13422 + checksum: 84df01c91d594701eaf7f5f007be881e47f7adef2e3f3765f7be031cb78033f9be0924273106cb81b586d8020da9885dbb81b3da363f00a51df00f26274f2b23 languageName: node linkType: hard From d32c8ff2e3e31ee592d9559034ca236f105d0926 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 23 May 2024 10:14:14 +0200 Subject: [PATCH 14/83] events-backend: split EventHub module Signed-off-by: Patrik Oldsberg --- .../src/service/hub/EventHub.ts | 124 ++---------------- .../src/service/hub/MemoryEventHubStore.ts | 113 ++++++++++++++++ .../events-backend/src/service/hub/types.ts | 33 +++++ 3 files changed, 154 insertions(+), 116 deletions(-) create mode 100644 plugins/events-backend/src/service/hub/MemoryEventHubStore.ts create mode 100644 plugins/events-backend/src/service/hub/types.ts diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index a014bb1add..594713da17 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -17,121 +17,11 @@ import { HttpAuthService, LoggerService } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; -import { EventParams } from '@backstage/plugin-events-node'; import { spec, createOpenApiRouter } from '../../schema/openapi.generated'; import { internal } from '@backstage/backend-openapi-utils'; - -type EventHubStore = { - publish(options: { - params: EventParams; - subscriberIds: string[]; - }): Promise; - - upsertSubscription(id: string, topics: string[]): Promise; - - readSubscription(id: string): Promise<{ events: EventParams[] }>; - - listen( - subscriptionId: string, - onNotify: (topicId: string) => void, - ): Promise<() => void>; -}; - -const MAX_BATCH_SIZE = 5; - -class MemoryEventHubStore implements EventHubStore { - #events = new Array< - EventParams & { seq: number; subscriberIds: Set } - >(); - #subscribers = new Map< - string, - { id: string; seq: number; topics: Set } - >(); - #listeners = new Set<{ - topics: Set; - notify(topicId: string): void; - }>(); - - async publish(options: { - params: EventParams; - subscriberIds: string[]; - }): Promise { - const topicId = options.params.topic; - const subscriberIds = new Set(options.subscriberIds); - - let hasOtherSubscribers = false; - for (const sub of this.#subscribers.values()) { - if (sub.topics.has(topicId) && !subscriberIds.has(sub.id)) { - hasOtherSubscribers = true; - break; - } - } - if (!hasOtherSubscribers) { - return; - } - - const nextSeq = this.#getMaxSeq() + 1; - this.#events.push({ ...options.params, subscriberIds, seq: nextSeq }); - - for (const listener of this.#listeners) { - if (listener.topics.has(topicId)) { - listener.notify(topicId); - } - } - } - - #getMaxSeq() { - return this.#events[this.#events.length - 1]?.seq ?? 0; - } - - async upsertSubscription(id: string, topics: string[]): Promise { - const existing = this.#subscribers.get(id); - if (existing) { - existing.topics = new Set(topics); - return; - } - const sub = { - id: id, - seq: this.#getMaxSeq(), - topics: new Set(topics), - }; - this.#subscribers.set(id, sub); - } - - async readSubscription(id: string): Promise<{ events: EventParams[] }> { - const sub = this.#subscribers.get(id); - if (!sub) { - throw new Error(`Subscription not found`); - } - const events = this.#events - .filter( - event => - event.seq > sub.seq && - sub.topics.has(event.topic) && - !event.subscriberIds.has(id), - ) - .slice(0, MAX_BATCH_SIZE); - - sub.seq = events[events.length - 1]?.seq ?? sub.seq; - - return { events: events.map(event => ({ ...event, seq: undefined })) }; - } - - async listen( - subscriptionId: string, - onNotify: (topicId: string) => void, - ): Promise<() => void> { - const sub = this.#subscribers.get(subscriptionId); - if (!sub) { - throw new Error(`Subscription not found`); - } - const listener = { topics: sub.topics, notify: onNotify }; - this.#listeners.add(listener); - return () => { - this.#listeners.delete(listener); - }; - } -} +import { MemoryEventHubStore } from './MemoryEventHubStore'; +import { EventHubStore } from './types'; +import { EventParams } from '@backstage/plugin-events-node'; export class EventHub { static async create(options: { @@ -141,8 +31,9 @@ export class EventHub { const { httpAuth } = options; const logger = options.logger.child({ type: 'EventHub' }); const router = Router(); + const store = new MemoryEventHubStore(); - const hub = new EventHub(router, logger, httpAuth); + const hub = new EventHub(router, logger, httpAuth, store); const apiRouter = await createOpenApiRouter(); @@ -172,11 +63,12 @@ export class EventHub { handler: Handler, logger: LoggerService, httpAuth: HttpAuthService, + store: EventHubStore, ) { this.#handler = handler; this.#logger = logger; this.#httpAuth = httpAuth; - this.#store = new MemoryEventHubStore(); + this.#store = store; } handler(): Handler { @@ -195,7 +87,7 @@ export class EventHub { params: { topic: req.body.event.topic, eventPayload: req.body.event.payload, - }, + } as EventParams, subscriberIds: req.body.subscriptionIds ?? [], }); this.#logger.info(`Published event to '${req.body.event.topic}'`, { diff --git a/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts b/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts new file mode 100644 index 0000000000..7ab5b98f82 --- /dev/null +++ b/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { EventHubStore } from './types'; + +const MAX_BATCH_SIZE = 5; + +export class MemoryEventHubStore implements EventHubStore { + #events = new Array< + EventParams & { seq: number; subscriberIds: Set } + >(); + #subscribers = new Map< + string, + { id: string; seq: number; topics: Set } + >(); + #listeners = new Set<{ + topics: Set; + notify(topicId: string): void; + }>(); + + async publish(options: { + params: EventParams; + subscriberIds: string[]; + }): Promise { + const topicId = options.params.topic; + const subscriberIds = new Set(options.subscriberIds); + + let hasOtherSubscribers = false; + for (const sub of this.#subscribers.values()) { + if (sub.topics.has(topicId) && !subscriberIds.has(sub.id)) { + hasOtherSubscribers = true; + break; + } + } + if (!hasOtherSubscribers) { + return; + } + + const nextSeq = this.#getMaxSeq() + 1; + this.#events.push({ ...options.params, subscriberIds, seq: nextSeq }); + + for (const listener of this.#listeners) { + if (listener.topics.has(topicId)) { + listener.notify(topicId); + } + } + } + + #getMaxSeq() { + return this.#events[this.#events.length - 1]?.seq ?? 0; + } + + async upsertSubscription(id: string, topics: string[]): Promise { + const existing = this.#subscribers.get(id); + if (existing) { + existing.topics = new Set(topics); + return; + } + const sub = { + id: id, + seq: this.#getMaxSeq(), + topics: new Set(topics), + }; + this.#subscribers.set(id, sub); + } + + async readSubscription(id: string): Promise<{ events: EventParams[] }> { + const sub = this.#subscribers.get(id); + if (!sub) { + throw new Error(`Subscription not found`); + } + const events = this.#events + .filter( + event => + event.seq > sub.seq && + sub.topics.has(event.topic) && + !event.subscriberIds.has(id), + ) + .slice(0, MAX_BATCH_SIZE); + + sub.seq = events[events.length - 1]?.seq ?? sub.seq; + + return { events: events.map(event => ({ ...event, seq: undefined })) }; + } + + async listen( + subscriptionId: string, + onNotify: (topicId: string) => void, + ): Promise<() => void> { + const sub = this.#subscribers.get(subscriptionId); + if (!sub) { + throw new Error(`Subscription not found`); + } + const listener = { topics: sub.topics, notify: onNotify }; + this.#listeners.add(listener); + return () => { + this.#listeners.delete(listener); + }; + } +} diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts new file mode 100644 index 0000000000..8e1124252a --- /dev/null +++ b/plugins/events-backend/src/service/hub/types.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EventParams } from '@backstage/plugin-events-node'; + +export type EventHubStore = { + publish(options: { + params: EventParams; + subscriberIds: string[]; + }): Promise; + + upsertSubscription(id: string, topics: string[]): Promise; + + readSubscription(id: string): Promise<{ events: EventParams[] }>; + + listen( + subscriptionId: string, + onNotify: (topicId: string) => void, + ): Promise<() => void>; +}; From 93cb484bcd81ebd207063420af2305942fc1235e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 00:48:53 +0200 Subject: [PATCH 15/83] events-backend: added initial DB events store Signed-off-by: Patrik Oldsberg --- .../migrations/20240523100528_init.js | 87 ++++++++ plugins/events-backend/package.json | 4 +- .../src/service/EventsPlugin.ts | 5 +- .../src/service/hub/DatabaseEventHubStore.ts | 202 ++++++++++++++++++ .../src/service/hub/EventHub.ts | 26 ++- .../events-backend/src/service/hub/types.ts | 2 +- yarn.lock | 1 + 7 files changed, 316 insertions(+), 11 deletions(-) create mode 100644 plugins/events-backend/migrations/20240523100528_init.js create mode 100644 plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js new file mode 100644 index 0000000000..90cdb770a5 --- /dev/null +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -0,0 +1,87 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function up(knex) { + // The event bus only supports PostgresSQL + if (knex.client.config.client === 'pg') { + await knex.schema.createTable('event_bus_events', table => { + table.comment('Events published to the events bus'); + table + .bigIncrements('id') + .primary() + .comment('The unique ID of this event'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the event was created'); + table.text('topic').notNullable().comment('The topic of the event'); + table + .text('data_json') + .notNullable() + .comment('The payload data of this event'); + table + .specificType('consumed_by', 'text ARRAY') + .comment( + 'The IDs of the subscribers that have already consumed this event', + ); + }); + + await knex.schema.createTable('event_bus_subscriptions', table => { + table.comment('Subscriptions to the event bus'); + table + .string('id') + .primary() + .notNullable() + .comment('The unique ID of this particular subscription'); + table + .dateTime('created_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the subscription was created'); + table + .dateTime('updated_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment('The time that the subscription was last updated'); + table + .bigInteger('read_until') + .notNullable() + .comment( + 'The sequence counter until which the subscription has read events', + ); + table + .specificType('topics', 'text ARRAY') + .comment('The topics that this subscriber is interested in'); + }); + } +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + if (knex.client.config.client === 'pg') { + await knex.schema.dropTable('events'); + } +}; diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index 2f9db1b081..f7a5ecc0c2 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -39,7 +39,8 @@ }, "files": [ "config.d.ts", - "dist" + "dist", + "migrations" ], "scripts": { "build": "backstage-cli package build", @@ -62,6 +63,7 @@ "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "knex": "^3.0.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index b7892a5079..089ea928bc 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -75,11 +75,12 @@ export const eventsPlugin = createBackendPlugin({ deps: { config: coreServices.rootConfig, events: eventsServiceRef, + database: coreServices.database, logger: coreServices.logger, httpAuth: coreServices.httpAuth, router: coreServices.httpRouter, }, - async init({ config, events, logger, httpAuth, router }) { + async init({ config, events, database, logger, httpAuth, router }) { const ingresses = Object.fromEntries( extensionPoint.httpPostIngresses.map(ingress => [ ingress.topic, @@ -96,7 +97,7 @@ export const eventsPlugin = createBackendPlugin({ const eventsRouter = Router(); http.bind(eventsRouter); - const hub = await EventHub.create({ logger, httpAuth }); + const hub = await EventHub.create({ database, logger, httpAuth }); router.use(hub.handler()); router.use(eventsRouter); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts new file mode 100644 index 0000000000..646812482d --- /dev/null +++ b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts @@ -0,0 +1,202 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { EventHubStore } from './types'; +import { Knex } from 'knex'; +import { + DatabaseService, + LoggerService, + resolvePackagePath, +} from '@backstage/backend-plugin-api'; + +const MAX_BATCH_SIZE = 10; + +const TABLE_EVENTS = 'event_bus_events'; +const TABLE_SUBSCRIPTIONS = 'event_bus_subscriptions'; + +type EventsRow = { + id: string; + created_at: Date; + topic: string; + data_json: string; + consumed_by: string[]; +}; + +type SubscriptionsRow = { + id: string; + created_at: Date; + updated_at: Date; + read_until: string; + topics: string[]; +}; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-events-backend', + 'migrations', +); + +export class DatabaseEventHubStore implements EventHubStore { + static async create(options: { + database: DatabaseService; + logger: LoggerService; + }): Promise { + const db = await options.database.getClient(); + + if (db.client.config.client !== 'pg') { + throw new Error( + `DatabaseEventHubStore only supports PostgreSQL, got '${db.client.config.client}'`, + ); + } + + if (!options.database.migrations?.skip) { + await db.migrate.latest({ + directory: migrationsDir, + }); + options.logger.info('DatabaseEventHubStore migrations ran successfully'); + } + + return new DatabaseEventHubStore(db); + } + + readonly #db: Knex; + + private constructor(db: Knex) { + this.#db = db; + } + + async publish(options: { + params: EventParams; + subscriberIds: string[]; + }): Promise<{ id: string }> { + const result = await this.#db(TABLE_EVENTS) + .insert({ + topic: options.params.topic, + data_json: JSON.stringify({ + payload: options.params.eventPayload, + metadata: options.params.metadata, + }), + consumed_by: options.subscriberIds, + }) + .returning('id'); + + if (result.length !== 1) { + throw new Error(`Failed to insert event, updated ${result.length} rows`); + } + + const { id } = result[0]; + + // TODO: notify + + return { id }; + } + + async upsertSubscription(id: string, topics: string[]): Promise { + const result = await this.#db(TABLE_SUBSCRIPTIONS) + .insert({ + id, + updated_at: this.#db.fn.now(), + topics, + read_until: this.#db(TABLE_EVENTS).max('id') as any, // TODO: figure out TS, + }) + .onConflict('id') + .merge(['topics', 'updated_at']) + .returning('*'); + + if (result.length !== 1) { + throw new Error( + `Failed to upsert subscription, updated ${result.length} rows`, + ); + } + } + + async readSubscription(id: string): Promise<{ events: EventParams[] }> { + const result = await this.#db(TABLE_SUBSCRIPTIONS) + // Read the target subscription so that we can use the read marker and topics + .with('sub', q => q.select().from(TABLE_SUBSCRIPTIONS).where({ id })) + // Read the next batch of events for the subscription from its read marker + .with('events', q => + q + .select('event.*') + .from({ event: 'event_bus_events', sub: 'sub' }) + // For each event, check if it matches any of the topics that we're subscribed to + .where( + 'event.topic', + '=', + this.#db.ref('topics').withSchema('sub').wrap('ANY(', ')'), + ) + // Skip events that have already been consumed by this subscription + .where( + this.#db.raw('?', id), + '<>', + this.#db.ref('consumed_by').withSchema('event').wrap('ANY(', ')'), + ) + .where('event.id', '>', this.#db.ref('read_until').withSchema('sub')) + .orderBy('event.id', 'asc') + .limit(MAX_BATCH_SIZE), + ) + // Find the ID of the last event in the batch, for use as the new read_until marker + .with('last_event_id', q => q.max({ last_event_id: 'id' }).from('events')) + // Aggregate the events into a JSON array so that we can return all of them with the UPDATE + .with('events_array', q => + q + .select({ events: this.#db.raw('json_agg(row_to_json(events))') }) + .from('events'), + ) + // Update the read_until marker to the ID of the last event, or if no + // events where read, the last ID out of all events + .update({ + read_until: this.#db.raw( + 'COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events))', + ), + }) + .updateFrom({ + events_array: 'events_array', + last_event_id: 'last_event_id', + }) + .where(`${TABLE_SUBSCRIPTIONS}.id`, id) + .returning<[{ events: EventsRow[] }]>('events_array.events'); + + if (result.length !== 1) { + throw new Error( + `Failed to upsert subscription, updated ${result.length} rows`, + ); + } + + const rows = result[0].events; + if (!rows || rows.length === 0) { + return { events: [] }; + } + + return { + events: result[0].events.map(row => { + const { payload, metadata } = JSON.parse(row.data_json); + return { + topic: row.topic, + eventPayload: payload, + metadata, + }; + }), + }; + } + + async listen( + _subscriptionId: string, + _onNotify: (topicId: string) => void, + ): Promise<() => void> { + // TODO + return () => {}; + } +} diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 594713da17..c78a8359ef 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -14,24 +14,33 @@ * limitations under the License. */ -import { HttpAuthService, LoggerService } from '@backstage/backend-plugin-api'; +import { + DatabaseService, + HttpAuthService, + LoggerService, +} from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; import { spec, createOpenApiRouter } from '../../schema/openapi.generated'; import { internal } from '@backstage/backend-openapi-utils'; import { MemoryEventHubStore } from './MemoryEventHubStore'; +import { DatabaseEventHubStore } from './DatabaseEventHubStore'; import { EventHubStore } from './types'; import { EventParams } from '@backstage/plugin-events-node'; export class EventHub { static async create(options: { logger: LoggerService; + database: DatabaseService; httpAuth: HttpAuthService; }) { - const { httpAuth } = options; + const { database, httpAuth } = options; const logger = options.logger.child({ type: 'EventHub' }); const router = Router(); - const store = new MemoryEventHubStore(); + const store = await DatabaseEventHubStore.create({ + database, + logger, + }); const hub = new EventHub(router, logger, httpAuth, store); @@ -83,16 +92,19 @@ export class EventHub { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); - await this.#store.publish({ + const { id } = await this.#store.publish({ params: { topic: req.body.event.topic, eventPayload: req.body.event.payload, } as EventParams, subscriberIds: req.body.subscriptionIds ?? [], }); - this.#logger.info(`Published event to '${req.body.event.topic}'`, { - subject: credentials.principal.subject, - }); + this.#logger.info( + `Published event to '${req.body.event.topic}' with ID '${id}'`, + { + subject: credentials.principal.subject, + }, + ); res.status(201).end(); }; diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 8e1124252a..8adedc69c7 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -20,7 +20,7 @@ export type EventHubStore = { publish(options: { params: EventParams; subscriberIds: string[]; - }): Promise; + }): Promise<{ id: string }>; upsertSubscription(id: string, topics: string[]): Promise; diff --git a/yarn.lock b/yarn.lock index 2aeb54be7f..306e96a2c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6152,6 +6152,7 @@ __metadata: "@types/express": ^4.17.6 express: ^4.17.1 express-promise-router: ^4.1.0 + knex: ^3.0.0 supertest: ^7.0.0 winston: ^3.2.1 languageName: unknown From e00c64fcb9a218c12a63a8c4888ccf3585f83261 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 11:48:46 +0200 Subject: [PATCH 16/83] events-backend: added listen and notify for db events store Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventHubStore.ts | 184 +++++++++++++++++- 1 file changed, 174 insertions(+), 10 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts index 646812482d..e5e1334191 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts @@ -21,11 +21,14 @@ import { LoggerService, resolvePackagePath, } from '@backstage/backend-plugin-api'; +import { NotFoundError } from '@backstage/errors'; const MAX_BATCH_SIZE = 10; +const LISTENER_CONNECTION_TIMEOUT_MS = 10_000; const TABLE_EVENTS = 'event_bus_events'; const TABLE_SUBSCRIPTIONS = 'event_bus_subscriptions'; +const TOPIC_PUBLISH = 'event_bus_publish'; type EventsRow = { id: string; @@ -48,6 +51,142 @@ const migrationsDir = resolvePackagePath( 'migrations', ); +interface InternalDbClient { + acquireRawConnection(): Promise; + destroyRawConnection(conn: InternalDbConnection): Promise; +} + +interface InternalDbConnection { + query(sql: string): Promise; + end(): Promise; + on( + event: 'notification', + listener: (event: { channel: string; payload: string }) => void, + ): void; + on(event: 'error', listener: (error: Error) => void): void; + on(event: 'end', listener: (error?: Error) => void): void; + removeAllListeners(): void; +} + +// This internal class manages a single connection to the database that all listeners share +class DatabaseEventHubListener { + readonly #client: InternalDbClient; + readonly #logger: LoggerService; + + readonly #listeners = new Set<{ + topics: Set; + onNotify: (topicId: string) => void; + onError: () => void; + }>(); + + #connPromise?: Promise; + #connTimeout?: NodeJS.Timeout; + + constructor(client: InternalDbClient, logger: LoggerService) { + this.#client = client; + this.#logger = logger.child({ type: 'DatabaseEventHubListener' }); + } + + async listen( + topics: Set, + onNotify: (topicId: string) => void, + onError: () => void, + ): Promise<() => void> { + if (this.#connTimeout) { + clearTimeout(this.#connTimeout); + this.#connTimeout = undefined; + } + await this.#ensureConnection(); + + const listener = { topics, onNotify, onError }; + this.#listeners.add(listener); + + return () => { + this.#listeners.delete(listener); + + // We don't use any heartbeats for the connection, as clients will be + // driving that for us. We do however need to make sure we don't sit + // idle for too long without any listeners + if (this.#listeners.size === 0) { + this.#connTimeout = setTimeout(() => { + this.#connPromise?.then(conn => { + this.#logger.info('Listener connection timed out'); + this.#connPromise = undefined; + this.#destroyConnection(conn); + }); + }, LISTENER_CONNECTION_TIMEOUT_MS); + } + }; + } + + #handleNotify(topic: string) { + this.#logger.info(`Listener received notification for topic '${topic}'`); + for (const l of this.#listeners) { + if (l.topics.has(topic)) { + l.onNotify(topic); + } + } + } + + // We don't try to reconnect on error, instead we notify all listeners and let them try to establish a new connection + #handleError(error: Error) { + this.#logger.error( + `Listener connection failed, notifying all listeners`, + error, + ); + for (const l of this.#listeners) { + l.onError(); + } + } + + #destroyConnection(conn: InternalDbConnection) { + this.#client.destroyRawConnection(conn).catch(error => { + this.#logger.error(`Listener failed to destroy connection`, error); + }); + conn.removeAllListeners(); + } + + async #ensureConnection() { + if (this.#connPromise) { + await this.#connPromise; + return; + } + this.#connPromise = Promise.resolve().then(async () => { + const conn = await this.#client.acquireRawConnection(); + + try { + await conn.query(`LISTEN ${TOPIC_PUBLISH}`); + + conn.on('notification', event => { + this.#handleNotify(event.payload); + }); + conn.on('error', error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError(error); + }); + conn.on('end', error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError( + error ?? new Error('Connection ended unexpectedly'), + ); + }); + return conn; + } catch (error) { + this.#destroyConnection(conn); + throw error; + } + }); + try { + await this.#connPromise; + } catch (error) { + this.#connPromise = undefined; + throw error; + } + } +} + export class DatabaseEventHubStore implements EventHubStore { static async create(options: { database: DatabaseService; @@ -68,22 +207,27 @@ export class DatabaseEventHubStore implements EventHubStore { options.logger.info('DatabaseEventHubStore migrations ran successfully'); } - return new DatabaseEventHubStore(db); + return new DatabaseEventHubStore(db, options.logger); } readonly #db: Knex; + readonly #logger: LoggerService; + readonly #listener: DatabaseEventHubListener; - private constructor(db: Knex) { + private constructor(db: Knex, logger: LoggerService) { this.#db = db; + this.#logger = logger; + this.#listener = new DatabaseEventHubListener(db.client, logger); } async publish(options: { params: EventParams; subscriberIds: string[]; }): Promise<{ id: string }> { + const topic = options.params.topic; const result = await this.#db(TABLE_EVENTS) .insert({ - topic: options.params.topic, + topic, data_json: JSON.stringify({ payload: options.params.eventPayload, metadata: options.params.metadata, @@ -98,7 +242,15 @@ export class DatabaseEventHubStore implements EventHubStore { const { id } = result[0]; - // TODO: notify + // Notify other event bus instances that an event is available on the topic + const notifyResult = await this.#db.select( + this.#db.raw(`pg_notify(?, ?)`, [TOPIC_PUBLISH, topic]), + ); + if (notifyResult?.length !== 1) { + this.#logger.warn( + `Failed to notify subscribers of event with ID '${id}' on topic '${topic}'`, + ); + } return { id }; } @@ -169,9 +321,11 @@ export class DatabaseEventHubStore implements EventHubStore { .where(`${TABLE_SUBSCRIPTIONS}.id`, id) .returning<[{ events: EventsRow[] }]>('events_array.events'); - if (result.length !== 1) { + if (result.length === 0) { + throw new NotFoundError(`Subscription with ID '${id}' not found`); + } else if (result.length > 1) { throw new Error( - `Failed to upsert subscription, updated ${result.length} rows`, + `Failed to read subscription, unexpectedly updated ${result.length} rows`, ); } @@ -193,10 +347,20 @@ export class DatabaseEventHubStore implements EventHubStore { } async listen( - _subscriptionId: string, - _onNotify: (topicId: string) => void, + subscriptionId: string, + onNotify: (topicId: string) => void, ): Promise<() => void> { - // TODO - return () => {}; + const result = await this.#db(TABLE_SUBSCRIPTIONS) + .select('topics') + .where({ id: subscriptionId }) + .first(); + console.log(`DEBUG: result=`, result); + if (!result) { + throw new NotFoundError( + `Subscription with ID '${subscriptionId}' not found`, + ); + } + const topics = new Set(result.topics ?? []); + return this.#listener.listen(topics, onNotify, () => {}); } } From b5243a7c0da97bb7cc93d12d9fa8c2ff1e11077f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 12:08:19 +0200 Subject: [PATCH 17/83] events-backend: update listening logic to be more robust Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventHubStore.ts | 14 +++-- .../src/service/hub/EventHub.ts | 53 ++++++++++++++----- .../events-backend/src/service/hub/types.ts | 7 ++- 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts index e5e1334191..f310ae86a4 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts @@ -348,8 +348,11 @@ export class DatabaseEventHubStore implements EventHubStore { async listen( subscriptionId: string, - onNotify: (topicId: string) => void, - ): Promise<() => void> { + listeners: { + onNotify: (topicId: string) => void; + onError: () => void; + }, + ): Promise<{ cancel(): void }> { const result = await this.#db(TABLE_SUBSCRIPTIONS) .select('topics') .where({ id: subscriptionId }) @@ -361,6 +364,11 @@ export class DatabaseEventHubStore implements EventHubStore { ); } const topics = new Set(result.topics ?? []); - return this.#listener.listen(topics, onNotify, () => {}); + const cancel = await this.#listener.listen( + topics, + listeners.onNotify, + listeners.onError, + ); + return { cancel }; } } diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index c78a8359ef..97d641e107 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -118,21 +118,46 @@ export class EventHub { }); const id = req.params.subscriptionId; - const { events } = await this.#store.readSubscription(id); - - this.#logger.info( - `Reading subscription '${id}' resulted in ${events.length} events`, - { subject: credentials.principal.subject }, - ); - - if (events.length > 0) { - res.json({ events }); - return; - } - - this.#store.listen(id, () => { - res.status(204).end(); + let resolveShouldNotify: (shouldNotify: boolean) => void; + const shouldNotifyPromise = new Promise(resolve => { + resolveShouldNotify = resolve; }); + + const { cancel } = await this.#store.listen(id, { + onNotify() { + shouldNotifyPromise.then(shouldNotify => { + if (shouldNotify) { + res.status(204).end(); + } + }); + }, + onError() { + shouldNotifyPromise.then(shouldNotify => { + if (shouldNotify) { + res.status(500).end(); + } + }); + }, + }); + req.on('end', cancel); + + try { + const { events } = await this.#store.readSubscription(id); + + this.#logger.info( + `Reading subscription '${id}' resulted in ${events.length} events`, + { subject: credentials.principal.subject }, + ); + + if (events.length > 0) { + res.json({ events }); + resolveShouldNotify!(false); + } else { + resolveShouldNotify!(true); + } + } finally { + resolveShouldNotify!(false); + } }; #handlePutSubscription: internal.DocRequestHandler< diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 8adedc69c7..0a0a27b1c1 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -28,6 +28,9 @@ export type EventHubStore = { listen( subscriptionId: string, - onNotify: (topicId: string) => void, - ): Promise<() => void>; + listeners: { + onNotify(topicId: string): void; + onError(): void; + }, + ): Promise<{ cancel(): void }>; }; From ff60692772640bae271ad1dec0d445f55b38764a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 12:14:51 +0200 Subject: [PATCH 18/83] events-backend: use memory store if not using postgres + memory updates Signed-off-by: Patrik Oldsberg --- .../src/service/hub/EventHub.ts | 33 ++++++++++++------- .../src/service/hub/MemoryEventHubStore.ts | 20 +++++++---- .../events-backend/src/service/hub/types.ts | 2 +- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 97d641e107..b54e9fdf08 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -37,10 +37,19 @@ export class EventHub { const { database, httpAuth } = options; const logger = options.logger.child({ type: 'EventHub' }); const router = Router(); - const store = await DatabaseEventHubStore.create({ - database, - logger, - }); + + let store: EventHubStore; + const db = await database.getClient(); + if (db.client.config.client === 'pg') { + logger.info('Database is PostgreSQL, using database store'); + store = await DatabaseEventHubStore.create({ + database, + logger, + }); + } else { + logger.info('Database is not PostgreSQL, using memory store'); + store = new MemoryEventHubStore(); + } const hub = new EventHub(router, logger, httpAuth, store); @@ -92,19 +101,21 @@ export class EventHub { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); - const { id } = await this.#store.publish({ + const result = await this.#store.publish({ params: { topic: req.body.event.topic, eventPayload: req.body.event.payload, } as EventParams, subscriberIds: req.body.subscriptionIds ?? [], }); - this.#logger.info( - `Published event to '${req.body.event.topic}' with ID '${id}'`, - { - subject: credentials.principal.subject, - }, - ); + if (result) { + this.#logger.info( + `Published event to '${req.body.event.topic}' with ID '${result.id}'`, + { + subject: credentials.principal.subject, + }, + ); + } res.status(201).end(); }; diff --git a/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts b/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts index 7ab5b98f82..9b72c18770 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts @@ -34,7 +34,7 @@ export class MemoryEventHubStore implements EventHubStore { async publish(options: { params: EventParams; subscriberIds: string[]; - }): Promise { + }): Promise<{ id: string } | undefined> { const topicId = options.params.topic; const subscriberIds = new Set(options.subscriberIds); @@ -46,7 +46,7 @@ export class MemoryEventHubStore implements EventHubStore { } } if (!hasOtherSubscribers) { - return; + return undefined; } const nextSeq = this.#getMaxSeq() + 1; @@ -57,6 +57,7 @@ export class MemoryEventHubStore implements EventHubStore { listener.notify(topicId); } } + return { id: String(nextSeq) }; } #getMaxSeq() { @@ -98,16 +99,21 @@ export class MemoryEventHubStore implements EventHubStore { async listen( subscriptionId: string, - onNotify: (topicId: string) => void, - ): Promise<() => void> { + listeners: { + onNotify(topicId: string): void; + onError(): void; + }, + ): Promise<{ cancel(): void }> { const sub = this.#subscribers.get(subscriptionId); if (!sub) { throw new Error(`Subscription not found`); } - const listener = { topics: sub.topics, notify: onNotify }; + const listener = { topics: sub.topics, notify: listeners.onNotify }; this.#listeners.add(listener); - return () => { - this.#listeners.delete(listener); + return { + cancel: () => { + this.#listeners.delete(listener); + }, }; } } diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 0a0a27b1c1..6cc1cc58e5 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -20,7 +20,7 @@ export type EventHubStore = { publish(options: { params: EventParams; subscriberIds: string[]; - }): Promise<{ id: string }>; + }): Promise<{ id: string } | undefined>; upsertSubscription(id: string, topics: string[]): Promise; From 1eaa7199430aa7a2984ee424ee6c08d3e80a38db Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 13:48:38 +0200 Subject: [PATCH 19/83] events-backend: avoid storing events with no subscribers Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventHubStore.ts | 58 ++++++++++++++----- .../src/service/hub/EventHub.ts | 20 +++++-- 2 files changed, 60 insertions(+), 18 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts index f310ae86a4..f1c336e5a3 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts @@ -223,24 +223,54 @@ export class DatabaseEventHubStore implements EventHubStore { async publish(options: { params: EventParams; subscriberIds: string[]; - }): Promise<{ id: string }> { + }): Promise<{ id: string } | undefined> { const topic = options.params.topic; - const result = await this.#db(TABLE_EVENTS) - .insert({ - topic, - data_json: JSON.stringify({ - payload: options.params.eventPayload, - metadata: options.params.metadata, - }), - consumed_by: options.subscriberIds, - }) - .returning('id'); + // This query inserts a new event into the database, but only if there are + // subscribers to the topic that have not already been notified + const result = await this.#db + // There's no clean way to create a INSERT INTO .. SELECT with knex, so we end up with quite a lot of .raw(...) + .into( + this.#db.raw('?? (??, ??, ??)', [ + TABLE_EVENTS, + // These are the rows that we insert, and should match the SELECT below + 'topic', + 'data_json', + 'consumed_by', + ]), + ) + .insert( + (q: Knex.QueryBuilder) => + q + // We're not reading data to insert from anywhere else, just raw data + .select( + this.#db.raw('?', [topic]), + this.#db.raw('?', [ + JSON.stringify({ + payload: options.params.eventPayload, + metadata: options.params.metadata, + }), + ]), + this.#db.raw('?', [options.subscriberIds]), + ) + // The rest of this query is to check whether there are any + // subscribers that have not been notified yet + .from(TABLE_SUBSCRIPTIONS) + .whereNotIn('id', options.subscriberIds) // Skip notified subscribers + .andWhere(this.#db.raw('? = ANY(topics)', [topic])) // Match topic + .having(this.#db.raw('count(*)'), '>', 0), // Check if there are any results + ) + .returning<{ id: string }[]>('id'); - if (result.length !== 1) { - throw new Error(`Failed to insert event, updated ${result.length} rows`); + if (result.length === 0) { + return undefined; + } + if (result.length > 1) { + throw new Error( + `Failed to insert event, unexpectedly updated ${result.length} rows`, + ); } - const { id } = result[0]; + const [{ id }] = result; // Notify other event bus instances that an event is available on the topic const notifyResult = await this.#db.select( diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index b54e9fdf08..344f20970f 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -101,22 +101,34 @@ export class EventHub { const credentials = await this.#httpAuth.credentials(req, { allow: ['service'], }); + const topic = req.body.event.topic; + const subscriberIds = req.body.subscriptionIds ?? []; const result = await this.#store.publish({ params: { - topic: req.body.event.topic, + topic, eventPayload: req.body.event.payload, } as EventParams, - subscriberIds: req.body.subscriptionIds ?? [], + subscriberIds, }); if (result) { this.#logger.info( - `Published event to '${req.body.event.topic}' with ID '${result.id}'`, + `Published event to '${topic}' with ID '${result.id}'`, { subject: credentials.principal.subject, }, ); + res.status(201).end(); + } else { + this.#logger.info( + `Skipped publishing of event to '${topic}', subscribers have already been notified: '${subscriberIds.join( + "', '", + )}'`, + { + subject: credentials.principal.subject, + }, + ); + res.status(204).end(); } - res.status(201).end(); }; #handleGetSubscription: internal.DocRequestHandler< From 05dea0964bdf677cf55cf6c7976905686de0cead Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 13:50:13 +0200 Subject: [PATCH 20/83] events-backend: update schema to include publish responses Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/schema/openapi.generated.ts | 7 +++++++ plugins/events-backend/src/schema/openapi.yaml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index 7d17ad2bf6..90a9b10011 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -137,6 +137,13 @@ export const spec = { operationId: 'PostEvent', description: 'Publish a new event', responses: { + '201': { + description: 'The event was published successfully', + }, + '204': { + description: + 'The event did not need to be published as all subscribers have already been notified', + }, default: { $ref: '#/components/responses/ErrorResponse', }, diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index 1adee01ab5..bec0b2461b 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -93,6 +93,10 @@ paths: operationId: PostEvent description: Publish a new event responses: + '201': + description: The event was published successfully + '204': + description: The event did not need to be published as all subscribers have already been notified default: $ref: '#/components/responses/ErrorResponse' security: From c7855bab6d530b447c6e817c4d57bfddbfb8ec62 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 13:55:21 +0200 Subject: [PATCH 21/83] events-backend: refactor EventHub into a router creator Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.ts | 5 +- .../src/service/hub/EventHub.ts | 210 +++++++----------- .../events-backend/src/service/hub/index.ts | 2 +- 3 files changed, 83 insertions(+), 134 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index 089ea928bc..176d7e2f34 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -28,7 +28,7 @@ import { } from '@backstage/plugin-events-node'; import Router from 'express-promise-router'; import { HttpPostIngressEventPublisher } from './http'; -import { EventHub } from './hub'; +import { createEventBusRouter } from './hub'; class EventsExtensionPointImpl implements EventsExtensionPoint { #httpPostIngresses: HttpPostIngressOptions[] = []; @@ -97,8 +97,7 @@ export const eventsPlugin = createBackendPlugin({ const eventsRouter = Router(); http.bind(eventsRouter); - const hub = await EventHub.create({ database, logger, httpAuth }); - router.use(hub.handler()); + router.use(await createEventBusRouter({ database, logger, httpAuth })); router.use(eventsRouter); router.addAuthPolicy({ diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 344f20970f..59be37967e 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -21,89 +21,45 @@ import { } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; -import { spec, createOpenApiRouter } from '../../schema/openapi.generated'; -import { internal } from '@backstage/backend-openapi-utils'; +import { createOpenApiRouter } from '../../schema/openapi.generated'; import { MemoryEventHubStore } from './MemoryEventHubStore'; import { DatabaseEventHubStore } from './DatabaseEventHubStore'; import { EventHubStore } from './types'; import { EventParams } from '@backstage/plugin-events-node'; -export class EventHub { - static async create(options: { - logger: LoggerService; - database: DatabaseService; - httpAuth: HttpAuthService; - }) { - const { database, httpAuth } = options; - const logger = options.logger.child({ type: 'EventHub' }); - const router = Router(); +export async function createEventBusRouter(options: { + logger: LoggerService; + database: DatabaseService; + httpAuth: HttpAuthService; +}): Promise { + const { database, httpAuth } = options; + const logger = options.logger.child({ type: 'EventHub' }); + const router = Router(); - let store: EventHubStore; - const db = await database.getClient(); - if (db.client.config.client === 'pg') { - logger.info('Database is PostgreSQL, using database store'); - store = await DatabaseEventHubStore.create({ - database, - logger, - }); - } else { - logger.info('Database is not PostgreSQL, using memory store'); - store = new MemoryEventHubStore(); - } - - const hub = new EventHub(router, logger, httpAuth, store); - - const apiRouter = await createOpenApiRouter(); - - router.use(apiRouter); - - apiRouter.post('/hub/events', hub.#handlePostEvents); - - // Long-polling - apiRouter.get( - '/hub/subscriptions/:subscriptionId/events', - hub.#handleGetSubscription, - ); - apiRouter.put( - '/hub/subscriptions/:subscriptionId', - hub.#handlePutSubscription, - ); - - return hub; + let store: EventHubStore; + const db = await database.getClient(); + if (db.client.config.client === 'pg') { + logger.info('Database is PostgreSQL, using database store'); + store = await DatabaseEventHubStore.create({ + database, + logger, + }); + } else { + logger.info('Database is not PostgreSQL, using memory store'); + store = new MemoryEventHubStore(); } - readonly #handler: Handler; - readonly #logger: LoggerService; - readonly #httpAuth: HttpAuthService; - readonly #store: EventHubStore; + const apiRouter = await createOpenApiRouter(); - private constructor( - handler: Handler, - logger: LoggerService, - httpAuth: HttpAuthService, - store: EventHubStore, - ) { - this.#handler = handler; - this.#logger = logger; - this.#httpAuth = httpAuth; - this.#store = store; - } + router.use(apiRouter); - handler(): Handler { - return this.#handler; - } - - #handlePostEvents: internal.DocRequestHandler< - typeof spec, - '/hub/events', - 'post' - > = async (req, res) => { - const credentials = await this.#httpAuth.credentials(req, { + apiRouter.post('/hub/events', async (req, res) => { + const credentials = await httpAuth.credentials(req, { allow: ['service'], }); const topic = req.body.event.topic; const subscriberIds = req.body.subscriptionIds ?? []; - const result = await this.#store.publish({ + const result = await store.publish({ params: { topic, eventPayload: req.body.event.payload, @@ -111,15 +67,12 @@ export class EventHub { subscriberIds, }); if (result) { - this.#logger.info( - `Published event to '${topic}' with ID '${result.id}'`, - { - subject: credentials.principal.subject, - }, - ); + logger.info(`Published event to '${topic}' with ID '${result.id}'`, { + subject: credentials.principal.subject, + }); res.status(201).end(); } else { - this.#logger.info( + logger.info( `Skipped publishing of event to '${topic}', subscribers have already been notified: '${subscriberIds.join( "', '", )}'`, @@ -129,77 +82,74 @@ export class EventHub { ); res.status(204).end(); } - }; + }); - #handleGetSubscription: internal.DocRequestHandler< - typeof spec, - '/hub/subscriptions/{subscriptionId}/events', - 'get' - > = async (req, res) => { - const credentials = await this.#httpAuth.credentials(req, { - allow: ['service'], - }); - const id = req.params.subscriptionId; + apiRouter.get( + '/hub/subscriptions/:subscriptionId/events', + async (req, res) => { + const credentials = await httpAuth.credentials(req, { + allow: ['service'], + }); + const id = req.params.subscriptionId; - let resolveShouldNotify: (shouldNotify: boolean) => void; - const shouldNotifyPromise = new Promise(resolve => { - resolveShouldNotify = resolve; - }); + let resolveShouldNotify: (shouldNotify: boolean) => void; + const shouldNotifyPromise = new Promise(resolve => { + resolveShouldNotify = resolve; + }); - const { cancel } = await this.#store.listen(id, { - onNotify() { - shouldNotifyPromise.then(shouldNotify => { - if (shouldNotify) { - res.status(204).end(); - } - }); - }, - onError() { - shouldNotifyPromise.then(shouldNotify => { - if (shouldNotify) { - res.status(500).end(); - } - }); - }, - }); - req.on('end', cancel); + const { cancel } = await store.listen(id, { + onNotify() { + shouldNotifyPromise.then(shouldNotify => { + if (shouldNotify) { + res.status(204).end(); + } + }); + }, + onError() { + shouldNotifyPromise.then(shouldNotify => { + if (shouldNotify) { + res.status(500).end(); + } + }); + }, + }); + req.on('end', cancel); - try { - const { events } = await this.#store.readSubscription(id); + try { + const { events } = await store.readSubscription(id); - this.#logger.info( - `Reading subscription '${id}' resulted in ${events.length} events`, - { subject: credentials.principal.subject }, - ); + logger.info( + `Reading subscription '${id}' resulted in ${events.length} events`, + { subject: credentials.principal.subject }, + ); - if (events.length > 0) { - res.json({ events }); + if (events.length > 0) { + res.json({ events }); + resolveShouldNotify!(false); + } else { + resolveShouldNotify!(true); + } + } finally { resolveShouldNotify!(false); - } else { - resolveShouldNotify!(true); } - } finally { - resolveShouldNotify!(false); - } - }; + }, + ); - #handlePutSubscription: internal.DocRequestHandler< - typeof spec, - '/hub/subscriptions/{subscriptionId}', - 'put' - > = async (req, res) => { - const credentials = await this.#httpAuth.credentials(req, { + apiRouter.put('/hub/subscriptions/:subscriptionId', async (req, res) => { + const credentials = await httpAuth.credentials(req, { allow: ['service'], }); const id = req.params.subscriptionId; - await this.#store.upsertSubscription(id, req.body.topics); + await store.upsertSubscription(id, req.body.topics); - this.#logger.info( + logger.info( `New subscription '${id}' topics='${req.body.topics.join("', '")}'`, { subject: credentials.principal.subject }, ); res.status(201).end(); - }; + }); + + return apiRouter; } diff --git a/plugins/events-backend/src/service/hub/index.ts b/plugins/events-backend/src/service/hub/index.ts index 4acc848298..2e769f24d7 100644 --- a/plugins/events-backend/src/service/hub/index.ts +++ b/plugins/events-backend/src/service/hub/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { EventHub } from './EventHub'; +export { createEventBusRouter } from './EventHub'; From 8c7542f0d79a7cd702ae04f01a6c252ebd1afb96 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 13:58:35 +0200 Subject: [PATCH 22/83] events-backend: update even bus route paths Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/schema/openapi.generated.ts | 6 +++--- plugins/events-backend/src/schema/openapi.yaml | 6 +++--- plugins/events-backend/src/service/hub/EventHub.ts | 6 +++--- plugins/events-node/src/generated/apis/DefaultApi.client.ts | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index 90a9b10011..d9b3c9045c 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -132,7 +132,7 @@ export const spec = { }, }, paths: { - '/hub/events': { + '/bus/v1/events': { post: { operationId: 'PostEvent', description: 'Publish a new event', @@ -192,7 +192,7 @@ export const spec = { }, }, }, - '/hub/subscriptions/{subscriptionId}': { + '/bus/v1/subscriptions/{subscriptionId}': { put: { operationId: 'PutSubscription', description: @@ -245,7 +245,7 @@ export const spec = { }, }, }, - '/hub/subscriptions/{subscriptionId}/events': { + '/bus/v1/subscriptions/{subscriptionId}/events': { get: { operationId: 'GetSubscriptionEvents', description: 'Get new events for the provided subscription', diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index bec0b2461b..632a884139 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -88,7 +88,7 @@ components: scheme: bearer bearerFormat: JWT paths: - /hub/events: + /bus/v1/events: post: operationId: PostEvent description: Publish a new event @@ -126,7 +126,7 @@ paths: payload: myData: foo - /hub/subscriptions/{subscriptionId}: + /bus/v1/subscriptions/{subscriptionId}: put: operationId: PutSubscription description: Ensures that the subscription exists with the provided configuration @@ -160,7 +160,7 @@ paths: topics: - test-topic - /hub/subscriptions/{subscriptionId}/events: + /bus/v1/subscriptions/{subscriptionId}/events: get: operationId: GetSubscriptionEvents description: Get new events for the provided subscription diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/EventHub.ts index 59be37967e..e70baad882 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/EventHub.ts @@ -53,7 +53,7 @@ export async function createEventBusRouter(options: { router.use(apiRouter); - apiRouter.post('/hub/events', async (req, res) => { + apiRouter.post('/bus/v1/events', async (req, res) => { const credentials = await httpAuth.credentials(req, { allow: ['service'], }); @@ -85,7 +85,7 @@ export async function createEventBusRouter(options: { }); apiRouter.get( - '/hub/subscriptions/:subscriptionId/events', + '/bus/v1/subscriptions/:subscriptionId/events', async (req, res) => { const credentials = await httpAuth.credentials(req, { allow: ['service'], @@ -135,7 +135,7 @@ export async function createEventBusRouter(options: { }, ); - apiRouter.put('/hub/subscriptions/:subscriptionId', async (req, res) => { + apiRouter.put('/bus/v1/subscriptions/:subscriptionId', async (req, res) => { const credentials = await httpAuth.credentials(req, { allow: ['service'], }); diff --git a/plugins/events-node/src/generated/apis/DefaultApi.client.ts b/plugins/events-node/src/generated/apis/DefaultApi.client.ts index 7a2099e79a..4bdeed0c33 100644 --- a/plugins/events-node/src/generated/apis/DefaultApi.client.ts +++ b/plugins/events-node/src/generated/apis/DefaultApi.client.ts @@ -75,7 +75,7 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/hub/subscriptions/{subscriptionId}/events`; + const uriTemplate = `/bus/v1/subscriptions/{subscriptionId}/events`; const uri = parser.parse(uriTemplate).expand({ subscriptionId: request.path.subscriptionId, @@ -103,7 +103,7 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/hub/events`; + const uriTemplate = `/bus/v1/events`; const uri = parser.parse(uriTemplate).expand({}); @@ -134,7 +134,7 @@ export class DefaultApiClient { ): Promise> { const baseUrl = await this.discoveryApi.getBaseUrl(pluginId); - const uriTemplate = `/hub/subscriptions/{subscriptionId}`; + const uriTemplate = `/bus/v1/subscriptions/{subscriptionId}`; const uri = parser.parse(uriTemplate).expand({ subscriptionId: request.path.subscriptionId, From ab035ebc1f13a92e032e116042632bdd6a376528 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 14:00:38 +0200 Subject: [PATCH 23/83] events-backend: EventHub -> EventBus Signed-off-by: Patrik Oldsberg --- ...ntHubStore.ts => DatabaseEventBusStore.ts} | 20 +++++++++---------- ...ventHubStore.ts => MemoryEventBusStore.ts} | 4 ++-- .../{EventHub.ts => createEventBusRouter.ts} | 14 ++++++------- .../events-backend/src/service/hub/index.ts | 2 +- .../events-backend/src/service/hub/types.ts | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) rename plugins/events-backend/src/service/hub/{DatabaseEventHubStore.ts => DatabaseEventBusStore.ts} (95%) rename plugins/events-backend/src/service/hub/{MemoryEventHubStore.ts => MemoryEventBusStore.ts} (96%) rename plugins/events-backend/src/service/hub/{EventHub.ts => createEventBusRouter.ts} (92%) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts similarity index 95% rename from plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts rename to plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index f1c336e5a3..fc2128e2cf 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { EventParams } from '@backstage/plugin-events-node'; -import { EventHubStore } from './types'; +import { EventBusStore } from './types'; import { Knex } from 'knex'; import { DatabaseService, @@ -69,7 +69,7 @@ interface InternalDbConnection { } // This internal class manages a single connection to the database that all listeners share -class DatabaseEventHubListener { +class DatabaseEventBusListener { readonly #client: InternalDbClient; readonly #logger: LoggerService; @@ -84,7 +84,7 @@ class DatabaseEventHubListener { constructor(client: InternalDbClient, logger: LoggerService) { this.#client = client; - this.#logger = logger.child({ type: 'DatabaseEventHubListener' }); + this.#logger = logger.child({ type: 'DatabaseEventBusListener' }); } async listen( @@ -187,16 +187,16 @@ class DatabaseEventHubListener { } } -export class DatabaseEventHubStore implements EventHubStore { +export class DatabaseEventBusStore implements EventBusStore { static async create(options: { database: DatabaseService; logger: LoggerService; - }): Promise { + }): Promise { const db = await options.database.getClient(); if (db.client.config.client !== 'pg') { throw new Error( - `DatabaseEventHubStore only supports PostgreSQL, got '${db.client.config.client}'`, + `DatabaseEventBusStore only supports PostgreSQL, got '${db.client.config.client}'`, ); } @@ -204,20 +204,20 @@ export class DatabaseEventHubStore implements EventHubStore { await db.migrate.latest({ directory: migrationsDir, }); - options.logger.info('DatabaseEventHubStore migrations ran successfully'); + options.logger.info('DatabaseEventBusStore migrations ran successfully'); } - return new DatabaseEventHubStore(db, options.logger); + return new DatabaseEventBusStore(db, options.logger); } readonly #db: Knex; readonly #logger: LoggerService; - readonly #listener: DatabaseEventHubListener; + readonly #listener: DatabaseEventBusListener; private constructor(db: Knex, logger: LoggerService) { this.#db = db; this.#logger = logger; - this.#listener = new DatabaseEventHubListener(db.client, logger); + this.#listener = new DatabaseEventBusListener(db.client, logger); } async publish(options: { diff --git a/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts similarity index 96% rename from plugins/events-backend/src/service/hub/MemoryEventHubStore.ts rename to plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 9b72c18770..8a03c2961a 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventHubStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -14,11 +14,11 @@ * limitations under the License. */ import { EventParams } from '@backstage/plugin-events-node'; -import { EventHubStore } from './types'; +import { EventBusStore } from './types'; const MAX_BATCH_SIZE = 5; -export class MemoryEventHubStore implements EventHubStore { +export class MemoryEventBusStore implements EventBusStore { #events = new Array< EventParams & { seq: number; subscriberIds: Set } >(); diff --git a/plugins/events-backend/src/service/hub/EventHub.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts similarity index 92% rename from plugins/events-backend/src/service/hub/EventHub.ts rename to plugins/events-backend/src/service/hub/createEventBusRouter.ts index e70baad882..23aadacadc 100644 --- a/plugins/events-backend/src/service/hub/EventHub.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -22,9 +22,9 @@ import { import { Handler } from 'express'; import Router from 'express-promise-router'; import { createOpenApiRouter } from '../../schema/openapi.generated'; -import { MemoryEventHubStore } from './MemoryEventHubStore'; -import { DatabaseEventHubStore } from './DatabaseEventHubStore'; -import { EventHubStore } from './types'; +import { MemoryEventBusStore } from './MemoryEventBusStore'; +import { DatabaseEventBusStore } from './DatabaseEventBusStore'; +import { EventBusStore } from './types'; import { EventParams } from '@backstage/plugin-events-node'; export async function createEventBusRouter(options: { @@ -33,20 +33,20 @@ export async function createEventBusRouter(options: { httpAuth: HttpAuthService; }): Promise { const { database, httpAuth } = options; - const logger = options.logger.child({ type: 'EventHub' }); + const logger = options.logger.child({ type: 'EventBus' }); const router = Router(); - let store: EventHubStore; + let store: EventBusStore; const db = await database.getClient(); if (db.client.config.client === 'pg') { logger.info('Database is PostgreSQL, using database store'); - store = await DatabaseEventHubStore.create({ + store = await DatabaseEventBusStore.create({ database, logger, }); } else { logger.info('Database is not PostgreSQL, using memory store'); - store = new MemoryEventHubStore(); + store = new MemoryEventBusStore(); } const apiRouter = await createOpenApiRouter(); diff --git a/plugins/events-backend/src/service/hub/index.ts b/plugins/events-backend/src/service/hub/index.ts index 2e769f24d7..a4f2f263f1 100644 --- a/plugins/events-backend/src/service/hub/index.ts +++ b/plugins/events-backend/src/service/hub/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { createEventBusRouter } from './EventHub'; +export { createEventBusRouter } from './createEventBusRouter'; diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 6cc1cc58e5..f7a162a3a6 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -16,7 +16,7 @@ import { EventParams } from '@backstage/plugin-events-node'; -export type EventHubStore = { +export type EventBusStore = { publish(options: { params: EventParams; subscriberIds: string[]; From 2d25e2791c346e79f8b6f66a0fb65c1c1d3c0712 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 15:33:33 +0200 Subject: [PATCH 24/83] events-backend: refactor event bus notifications and add timeout Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 22 ++++++-- .../src/service/hub/MemoryEventBusStore.ts | 20 ++++--- .../src/service/hub/createEventBusRouter.ts | 55 ++++++++++++++----- .../events-backend/src/service/hub/types.ts | 5 +- 4 files changed, 72 insertions(+), 30 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index fc2128e2cf..c32e2393a7 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -378,27 +378,37 @@ export class DatabaseEventBusStore implements EventBusStore { async listen( subscriptionId: string, - listeners: { + options: { + signal: AbortSignal; onNotify: (topicId: string) => void; onError: () => void; }, - ): Promise<{ cancel(): void }> { + ): Promise { const result = await this.#db(TABLE_SUBSCRIPTIONS) .select('topics') .where({ id: subscriptionId }) .first(); - console.log(`DEBUG: result=`, result); + if (!result) { throw new NotFoundError( `Subscription with ID '${subscriptionId}' not found`, ); } + + if (options.signal.aborted) { + return; + } + const topics = new Set(result.topics ?? []); const cancel = await this.#listener.listen( topics, - listeners.onNotify, - listeners.onError, + options.onNotify, + options.onError, ); - return { cancel }; + if (options.signal.aborted) { + cancel(); + } else { + options.signal.addEventListener('abort', cancel); + } } } diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 8a03c2961a..98739fdb33 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -99,21 +99,25 @@ export class MemoryEventBusStore implements EventBusStore { async listen( subscriptionId: string, - listeners: { + options: { + signal: AbortSignal; onNotify(topicId: string): void; onError(): void; }, - ): Promise<{ cancel(): void }> { + ): Promise { + if (options.signal.aborted) { + return; + } + const sub = this.#subscribers.get(subscriptionId); if (!sub) { throw new Error(`Subscription not found`); } - const listener = { topics: sub.topics, notify: listeners.onNotify }; + const listener = { topics: sub.topics, notify: options.onNotify }; this.#listeners.add(listener); - return { - cancel: () => { - this.#listeners.delete(listener); - }, - }; + + options.signal.addEventListener('abort', () => { + this.#listeners.delete(listener); + }); } } diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 23aadacadc..84d04a00fb 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -27,12 +27,19 @@ import { DatabaseEventBusStore } from './DatabaseEventBusStore'; import { EventBusStore } from './types'; import { EventParams } from '@backstage/plugin-events-node'; +const DEFAULT_NOTIFY_TIMEOUT_MS = 55_000; // Just below 60s, which is a common HTTP timeout + export async function createEventBusRouter(options: { logger: LoggerService; database: DatabaseService; httpAuth: HttpAuthService; + notifyTimeoutMs?: number; // for testing }): Promise { - const { database, httpAuth } = options; + const { + database, + httpAuth, + notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS, + } = options; const logger = options.logger.child({ type: 'EventBus' }); const router = Router(); @@ -92,28 +99,48 @@ export async function createEventBusRouter(options: { }); const id = req.params.subscriptionId; + // Don't notify until we know the outcome of reading events let resolveShouldNotify: (shouldNotify: boolean) => void; const shouldNotifyPromise = new Promise(resolve => { resolveShouldNotify = resolve; }); - const { cancel } = await store.listen(id, { + const controller = new AbortController(); + req.on('end', () => controller.abort()); + + let timeout: NodeJS.Timeout | undefined = undefined; + + let notified = false; + const notify = async (status: number) => { + if (!notified) { + clearTimeout(timeout); + notified = true; + if (await shouldNotifyPromise) { + res.status(status).end(); + } + } + }; + + // By setting up the listener first we make sure we don't miss any events + // that are published while reading. If an event is published we'll receive + // a notification, which depending on the outcome of the read we may ignore + await store.listen(id, { + signal: controller.signal, onNotify() { - shouldNotifyPromise.then(shouldNotify => { - if (shouldNotify) { - res.status(204).end(); - } - }); + notify(204); }, onError() { - shouldNotifyPromise.then(shouldNotify => { - if (shouldNotify) { - res.status(500).end(); - } - }); + notify(500); }, }); - req.on('end', cancel); + + // By timing out requests we make sure they don't stall or that events get stuck. + // For the caller there's no difference between a timeout and a + // notifications, either way they should try reading again. + timeout = setTimeout(() => { + notify(204); + controller.abort(); + }, notifyTimeoutMs); try { const { events } = await store.readSubscription(id); @@ -125,11 +152,11 @@ export async function createEventBusRouter(options: { if (events.length > 0) { res.json({ events }); - resolveShouldNotify!(false); } else { resolveShouldNotify!(true); } } finally { + clearTimeout(timeout); resolveShouldNotify!(false); } }, diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index f7a162a3a6..55159352b4 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -28,9 +28,10 @@ export type EventBusStore = { listen( subscriptionId: string, - listeners: { + options: { + signal: AbortSignal; onNotify(topicId: string): void; onError(): void; }, - ): Promise<{ cancel(): void }>; + ): Promise; }; From d17ce45a4ac0af74bc9326c4dd60976f7df8dc67 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 15:46:49 +0200 Subject: [PATCH 25/83] events-backend: add keepalive loop for listener connection Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index c32e2393a7..4e571f71f0 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -21,10 +21,11 @@ import { LoggerService, resolvePackagePath, } from '@backstage/backend-plugin-api'; -import { NotFoundError } from '@backstage/errors'; +import { ForwardedError, NotFoundError } from '@backstage/errors'; const MAX_BATCH_SIZE = 10; -const LISTENER_CONNECTION_TIMEOUT_MS = 10_000; +const LISTENER_CONNECTION_TIMEOUT_MS = 60_000; +const KEEPALIVE_INTERVAL_MS = 60_000; const TABLE_EVENTS = 'event_bus_events'; const TABLE_SUBSCRIPTIONS = 'event_bus_subscriptions'; @@ -81,6 +82,7 @@ class DatabaseEventBusListener { #connPromise?: Promise; #connTimeout?: NodeJS.Timeout; + #keepaliveInterval?: NodeJS.Timeout; constructor(client: InternalDbClient, logger: LoggerService) { this.#client = client; @@ -104,13 +106,11 @@ class DatabaseEventBusListener { return () => { this.#listeners.delete(listener); - // We don't use any heartbeats for the connection, as clients will be - // driving that for us. We do however need to make sure we don't sit - // idle for too long without any listeners + // If we don't have any listeners, destroy the connection after a timeout if (this.#listeners.size === 0) { this.#connTimeout = setTimeout(() => { this.#connPromise?.then(conn => { - this.#logger.info('Listener connection timed out'); + this.#logger.info('Listener connection timed out, destroying'); this.#connPromise = undefined; this.#destroyConnection(conn); }); @@ -128,7 +128,8 @@ class DatabaseEventBusListener { } } - // We don't try to reconnect on error, instead we notify all listeners and let them try to establish a new connection + // We don't try to reconnect on error, instead we notify all listeners and let + // them try to establish a new connection #handleError(error: Error) { this.#logger.error( `Listener connection failed, notifying all listeners`, @@ -140,6 +141,10 @@ class DatabaseEventBusListener { } #destroyConnection(conn: InternalDbConnection) { + if (this.#keepaliveInterval) { + clearInterval(this.#keepaliveInterval); + this.#keepaliveInterval = undefined; + } this.#client.destroyRawConnection(conn).catch(error => { this.#logger.error(`Listener failed to destroy connection`, error); }); @@ -157,6 +162,18 @@ class DatabaseEventBusListener { try { await conn.query(`LISTEN ${TOPIC_PUBLISH}`); + // Set up a keepalive interval to make sure the connection stays alive + if (this.#keepaliveInterval) { + clearInterval(this.#keepaliveInterval); + } + this.#keepaliveInterval = setInterval(() => { + conn.query('select 1').catch(error => { + this.#connPromise = undefined; + this.#destroyConnection(conn); + this.#handleError(new ForwardedError('Keepalive failed', error)); + }); + }, KEEPALIVE_INTERVAL_MS); + conn.on('notification', event => { this.#handleNotify(event.payload); }); From ed1499b349b2e8c003419eebc11124d9e644bc3d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 17:08:57 +0200 Subject: [PATCH 26/83] events-backend: fix event bus notify timeout Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/createEventBusRouter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 84d04a00fb..14237650fc 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -141,6 +141,11 @@ export async function createEventBusRouter(options: { notify(204); controller.abort(); }, notifyTimeoutMs); + shouldNotifyPromise.then(shouldNotify => { + if (!shouldNotify) { + clearTimeout(timeout); + } + }); try { const { events } = await store.readSubscription(id); @@ -156,7 +161,6 @@ export async function createEventBusRouter(options: { resolveShouldNotify!(true); } } finally { - clearTimeout(timeout); resolveShouldNotify!(false); } }, From 9e65db8cc90926fc33be9cca9af2dd28bdb8d5cf Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 17:09:25 +0200 Subject: [PATCH 27/83] backend: install events backend Signed-off-by: Patrik Oldsberg --- packages/backend/package.json | 27 ++++++++++++++------------- packages/backend/src/index.ts | 1 + yarn.lock | 2 ++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 5b4a753dec..f5d7f88d1a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -1,30 +1,33 @@ { "name": "example-backend", "version": "0.0.30", - "main": "dist/index.cjs.js", - "types": "src/index.ts", - "license": "Apache-2.0", - "private": true, "backstage": { "role": "backend" }, + "private": true, + "keywords": [ + "backstage" + ], "homepage": "https://backstage.io", "repository": { "type": "git", "url": "https://github.com/backstage/backstage", "directory": "packages/backend" }, - "keywords": [ - "backstage" + "license": "Apache-2.0", + "main": "dist/index.cjs.js", + "types": "src/index.ts", + "files": [ + "dist" ], "scripts": { "build": "backstage-cli package build", + "build-image": "docker build ../.. -f Dockerfile --tag example-backend", "clean": "backstage-cli package clean", "lint": "backstage-cli package lint", "start": "backstage-cli package start --require ./src/instrumentation.js", - "test": "backstage-cli package test", - "build-image": "docker build ../.. -f Dockerfile --tag example-backend", - "start:prometheus": "docker run --mount type=bind,source=./prometheus.yml,destination=/etc/prometheus/prometheus.yml --publish published=9090,target=9090,protocol=tcp prom/prometheus" + "start:prometheus": "docker run --mount type=bind,source=./prometheus.yml,destination=/etc/prometheus/prometheus.yml --publish published=9090,target=9090,protocol=tcp prom/prometheus", + "test": "backstage-cli package test" }, "dependencies": { "@backstage/backend-defaults": "workspace:^", @@ -41,6 +44,7 @@ "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^", "@backstage/plugin-devtools-backend": "workspace:^", + "@backstage/plugin-events-backend": "workspace:^", "@backstage/plugin-kubernetes-backend": "workspace:^", "@backstage/plugin-notifications-backend": "workspace:^", "@backstage/plugin-permission-backend": "workspace:^", @@ -64,8 +68,5 @@ }, "devDependencies": { "@backstage/cli": "workspace:^" - }, - "files": [ - "dist" - ] + } } diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 952f3315c4..3e8c2910b1 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -39,6 +39,7 @@ backend.add( import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'), ); backend.add(import('@backstage/plugin-catalog-backend/alpha')); +backend.add(import('@backstage/plugin-events-backend/alpha')); backend.add(import('@backstage/plugin-devtools-backend')); backend.add(import('@backstage/plugin-kubernetes-backend/alpha')); backend.add( diff --git a/yarn.lock b/yarn.lock index 306e96a2c8..58c69ee093 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6166,6 +6166,7 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/types": "workspace:^" cross-fetch: ^4.0.0 uri-template: ^2.0.0 @@ -26382,6 +26383,7 @@ __metadata: "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" "@backstage/plugin-devtools-backend": "workspace:^" + "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" "@backstage/plugin-notifications-backend": "workspace:^" "@backstage/plugin-permission-backend": "workspace:^" From ea33c1240a065fe5c865710480c50051db8d2eaa Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 17:10:57 +0200 Subject: [PATCH 28/83] events-node: update DefaultEventsService to use backend Signed-off-by: Patrik Oldsberg --- plugins/events-node/package.json | 1 + .../src/api/DefaultEventsService.ts | 359 +++++++++++++++--- plugins/events-node/src/service.ts | 11 +- 3 files changed, 307 insertions(+), 64 deletions(-) diff --git a/plugins/events-node/package.json b/plugins/events-node/package.json index de880dc399..2ac1a6922b 100644 --- a/plugins/events-node/package.json +++ b/plugins/events-node/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "@backstage/backend-plugin-api": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/types": "workspace:^", "cross-fetch": "^4.0.0", "uri-template": "^2.0.0" diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index bb5c2a0ca8..d1d26b1723 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -14,9 +14,267 @@ * limitations under the License. */ -import { LoggerService } from '@backstage/backend-plugin-api'; +import { + AuthService, + DiscoveryService, + LoggerService, +} from '@backstage/backend-plugin-api'; import { EventParams } from './EventParams'; import { EventsService, EventsServiceSubscribeOptions } from './EventsService'; +import { DefaultApiClient } from '../generated'; +import { NotImplementedError, ResponseError } from '@backstage/errors'; + +const POLL_BACKOFF_START_MS = 1_000; +const POLL_BACKOFF_MAX_MS = 60_000; +const POLL_BACKOFF_FACTOR = 2; + +/** + * Local event bus for subscribers within the same process. + * + * When publishing events we'll keep track of which subscribers we managed to + * reach locally, and forward those subscriber IDs to the events backend if it + * is in use. The events backend will then both avoid forwarding the same events + * to those subscribers again, but also avoid storing the event altogether if + * there are no other subscribers. + * @internal + */ +export class LocalEventBus { + readonly #logger: LoggerService; + + readonly #subscribers = new Map< + string, + Omit[] + >(); + + constructor(logger: LoggerService) { + this.#logger = logger; + } + + async publish( + params: EventParams, + ): Promise<{ notifiedSubscribers: string[] }> { + this.#logger.debug( + `Event received: topic=${params.topic}, metadata=${JSON.stringify( + params.metadata, + )}, payload=${JSON.stringify(params.eventPayload)}`, + ); + + if (!this.#subscribers.has(params.topic)) { + return { notifiedSubscribers: [] }; + } + + const onEventPromises: Promise[] = []; + this.#subscribers.get(params.topic)?.forEach(subscription => { + onEventPromises.push( + (async () => { + try { + await subscription.onEvent(params); + } catch (error) { + this.#logger.warn( + `Subscriber "${subscription.id}" failed to process event for topic "${params.topic}"`, + error, + ); + } + return subscription.id; + })(), + ); + }); + + return { notifiedSubscribers: await Promise.all(onEventPromises) }; + } + + async subscribe(options: EventsServiceSubscribeOptions): Promise { + options.topics.forEach(topic => { + if (!this.#subscribers.has(topic)) { + this.#subscribers.set(topic, []); + } + + this.#subscribers.get(topic)!.push({ + id: options.id, + onEvent: options.onEvent, + }); + }); + } +} + +/** + * Plugin specific events bus that delegates to the local bus, as well as the + * events backend if it is available. + */ +class PluginEventsService implements EventsService { + constructor( + private readonly pluginId: string, + private readonly localBus: LocalEventBus, + private readonly logger: LoggerService, + private client?: DefaultApiClient, + private readonly auth?: AuthService, + ) {} + + async publish(params: EventParams): Promise { + const { notifiedSubscribers } = await this.localBus.publish(params); + + if (!this.client) { + return; + } + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.postEvent( + { + body: { + event: { payload: params.eventPayload, topic: params.topic }, + subscriptionIds: notifiedSubscribers, + }, + }, + { token }, + ); + + if (!res.ok) { + if (res.status === 404) { + this.logger.warn( + `Event publish request failed with status 404, events backend not found. Future events will not be persisted.`, + ); + delete this.client; + return; + } + throw await ResponseError.fromResponse(res); + } + } + + async subscribe(options: EventsServiceSubscribeOptions): Promise { + const subscriptionId = `${this.pluginId}.${options.id}`; + + await this.localBus.subscribe({ + id: subscriptionId, + topics: options.topics, + onEvent: options.onEvent, + }); + + if (!this.client) { + return; + } + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.putSubscription( + { + path: { subscriptionId }, + body: { topics: options.topics }, + }, + { token }, + ); + if (!res.ok) { + if (res.status === 404) { + this.logger.warn( + `Event subscribe request failed with status 404, events backend not found. Future events will not be persisted.`, + ); + delete this.client; + return; + } + throw await ResponseError.fromResponse(res); + } + + this.#startPolling(subscriptionId, options.onEvent); + } + + #startPolling( + subscriptionId: string, + onEvent: EventsServiceSubscribeOptions['onEvent'], + ) { + let backoffMs = POLL_BACKOFF_START_MS; + const poll = async () => { + if (!this.client) { + return; + } + try { + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.getSubscriptionEvents( + { + path: { subscriptionId }, + }, + { token }, + ); + + if (!res.ok) { + throw await ResponseError.fromResponse(res); + } + backoffMs = POLL_BACKOFF_START_MS; + + // 204 arrives after a blocking response and means there are new events + // available or we timed out, either way should should try to read events + // immediately again + if (res.status === 204) { + process.nextTick(poll); + } else if (res.status === 200) { + const data = await res.json(); + if (data) { + for (const event of data.events ?? []) { + try { + await onEvent({ + topic: event.topic, + eventPayload: event.payload, + }); + } catch (error) { + this.logger.warn( + `Subscriber "${subscriptionId}" failed to process event for topic "${event.topic}"`, + error, + ); + } + } + } + process.nextTick(poll); + } else { + this.logger.warn( + `Unexpected response status ${res.status} from events backend for subscription "${subscriptionId}"`, + ); + } + } catch (error) { + this.logger.warn( + `Poll failed for subscription "${subscriptionId}", retrying in ${backoffMs.toFixed( + 0, + )}ms`, + error, + ); + setTimeout(poll, backoffMs); + backoffMs = Math.min( + backoffMs * POLL_BACKOFF_FACTOR, + POLL_BACKOFF_MAX_MS, + ); + } + }; + poll(); + } + + async #getToken() { + if (!this.auth) { + throw new Error('Auth service not available'); + } + + try { + const { token } = await this.auth.getPluginRequestToken({ + onBehalfOf: await this.auth.getOwnServiceCredentials(), + targetPluginId: 'events', + }); + return token; + } catch (error) { + // This is a bit hacky, but handles the case where new auth is used + // without legacy auth fallback, and the events backend is not installed + if (String(error).includes('Unable to generate legacy token')) { + this.logger.warn( + `The events backend is not available and neither is legacy auth. Future events will not be persisted.`, + ); + delete this.client; + return undefined; + } + throw error; + } + } +} /** * In-process event broker which will pass the event to all registered subscribers @@ -28,15 +286,16 @@ import { EventsService, EventsServiceSubscribeOptions } from './EventsService'; */ // TODO(pjungermann): add opentelemetry? (see plugins/catalog-backend/src/util/opentelemetry.ts, etc.) export class DefaultEventsService implements EventsService { - private readonly subscribers = new Map< - string, - Omit[] - >(); - - private constructor(private readonly logger: LoggerService) {} + private constructor( + private readonly logger: LoggerService, + private readonly localBus: LocalEventBus, + ) {} static create(options: { logger: LoggerService }): DefaultEventsService { - return new DefaultEventsService(options.logger); + return new DefaultEventsService( + options.logger, + new LocalEventBus(options.logger), + ); } /** @@ -45,60 +304,36 @@ export class DefaultEventsService implements EventsService { * * @param pluginId - The plugin that the `EventService` should be created for. */ - forPlugin(pluginId: string): EventsService { - return { - publish: (params: EventParams): Promise => { - return this.publish(params); - }, - subscribe: (options: EventsServiceSubscribeOptions): Promise => { - return this.subscribe({ - ...options, - id: `${pluginId}.${options.id}`, - }); - }, - }; - } - - async publish(params: EventParams): Promise { - this.logger.debug( - `Event received: topic=${params.topic}, metadata=${JSON.stringify( - params.metadata, - )}, payload=${JSON.stringify(params.eventPayload)}`, - ); - - if (!this.subscribers.has(params.topic)) { - return; - } - - const onEventPromises: Promise[] = []; - this.subscribers.get(params.topic)?.forEach(subscription => { - onEventPromises.push( - (async () => { - try { - await subscription.onEvent(params); - } catch (error) { - this.logger.warn( - `Subscriber "${subscription.id}" failed to process event for topic "${params.topic}"`, - error, - ); - } - })(), - ); - }); - - await Promise.all(onEventPromises); - } - - async subscribe(options: EventsServiceSubscribeOptions): Promise { - options.topics.forEach(topic => { - if (!this.subscribers.has(topic)) { - this.subscribers.set(topic, []); - } - - this.subscribers.get(topic)!.push({ - id: options.id, - onEvent: options.onEvent, + forPlugin( + pluginId: string, + options?: { + discovery: DiscoveryService; + logger: LoggerService; + auth: AuthService; + }, + ): EventsService { + const client = + options && + new DefaultApiClient({ + discoveryApi: options.discovery, }); - }); + const logger = options?.logger ?? this.logger; + return new PluginEventsService( + pluginId, + this.localBus, + logger, + client, + options?.auth, + ); + } + + /** @deprecated this method should not be called */ + async publish(_params: EventParams): Promise { + throw new NotImplementedError(); + } + + /** @deprecated this method should not be called */ + async subscribe(_options: EventsServiceSubscribeOptions): Promise { + throw new NotImplementedError(); } } diff --git a/plugins/events-node/src/service.ts b/plugins/events-node/src/service.ts index af0e562c80..8d754f6957 100644 --- a/plugins/events-node/src/service.ts +++ b/plugins/events-node/src/service.ts @@ -38,11 +38,18 @@ export const eventsServiceFactory = createServiceFactory({ deps: { pluginMetadata: coreServices.pluginMetadata, rootLogger: coreServices.rootLogger, + discovery: coreServices.discovery, + logger: coreServices.logger, + auth: coreServices.auth, }, async createRootContext({ rootLogger }) { return DefaultEventsService.create({ logger: rootLogger }); }, - async factory({ pluginMetadata }, eventsService) { - return eventsService.forPlugin(pluginMetadata.getId()); + async factory({ pluginMetadata, discovery, logger, auth }, eventsService) { + return eventsService.forPlugin(pluginMetadata.getId(), { + discovery, + logger, + auth, + }); }, }); From 978a9daca87582673673600a268fde01719bba66 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 24 May 2024 17:13:21 +0200 Subject: [PATCH 29/83] events-backend: clean up local dev setup Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 106 +++------------------------- 1 file changed, 11 insertions(+), 95 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 1c0d38215b..1b36751d59 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -20,7 +20,6 @@ import { createBackendPlugin, } from '@backstage/backend-plugin-api'; import { eventsServiceRef } from '@backstage/plugin-events-node'; -import { DefaultApiClient } from '../../events-node/src/generated'; const backend = createBackend(); @@ -34,16 +33,16 @@ backend.add( deps: { events: eventsServiceRef, logger: coreServices.logger, + rootLifecycle: coreServices.rootLifecycle, }, - async init({ events, logger }) { - // setInterval(() => { - // logger.info(`Publishing event to topic 'test'`); - // events.publish({ - // eventPayload: { foo: 'bar' }, - // topic: 'test', - // metadata: { meta: 'baz' }, - // }); - // }, 5000); + async init({ events, logger, rootLifecycle }) { + rootLifecycle.addStartupHook(async () => { + logger.info(`Publishing event to topic 'test'`); + await events.publish({ + eventPayload: { foo: 'bar' }, + topic: 'test', + }); + }); }, }); }, @@ -58,98 +57,15 @@ backend.add( deps: { events: eventsServiceRef, logger: coreServices.logger, - discovery: coreServices.discovery, - auth: coreServices.auth, - rootLifecycle: coreServices.rootLifecycle, }, - async init({ events, logger, discovery, rootLifecycle, auth }) { - events.subscribe({ + async init({ events, logger }) { + await events.subscribe({ id: 'test-1', topics: ['test'], async onEvent(event) { logger.info(`Received event: ${JSON.stringify(event, null, 2)}`); }, }); - - rootLifecycle.addStartupHook(async () => { - logger.info('Started!'); - - const client = new DefaultApiClient({ discoveryApi: discovery }); - const baseUrl = await discovery.getBaseUrl('events'); - console.log(`DEBUG: baseUrl=`, baseUrl); - const { token } = await auth.getPluginRequestToken({ - onBehalfOf: await auth.getOwnServiceCredentials(), - targetPluginId: 'events', - }); - - const subRes = await client.putSubscription( - { - path: { subscriptionId: '123' }, - body: { topics: ['test'] }, - }, - { token }, - ); - console.log( - `DEBUG: sub create req = ${subRes.status} ${subRes.statusText}`, - ); - const subRes2 = await client.putSubscription( - { - path: { subscriptionId: 'abc' }, - body: { topics: ['test'] }, - }, - { token }, - ); - console.log( - `DEBUG: sub create req = ${subRes2.status} ${subRes2.statusText}`, - ); - - const poll = async () => { - const res = await client.getSubscriptionEvents( - { - path: { subscriptionId: '123' }, - }, - { token }, - ); - - const data = res.status === 200 && (await res.json()); - console.log( - `DEBUG: sub poll req = ${res.status} ${res.statusText}`, - data, - ); - poll(); - }; - poll(); - - const poll2 = async () => { - const res = await client.getSubscriptionEvents( - { - path: { subscriptionId: 'abc' }, - }, - { token }, - ); - - const data = res.status === 200 && (await res.json()); - console.log( - `DEBUG: sub poll2 req = ${res.status} ${res.statusText}`, - data, - ); - poll2(); - }; - poll2(); - - setTimeout(() => { - console.log(`DEBUG: publishing!`); - client.postEvent( - { - body: { - event: { topic: 'test', payload: { foo: 'bar' } }, - subscriptionIds: ['123'], - }, - }, - { token }, - ); - }, 500); - }); }, }); }, From c6e872654368d314d6e83a74afd49e2040342f29 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 08:36:04 +0200 Subject: [PATCH 30/83] events-backend: lock subscription row when reading Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 4e571f71f0..600a038793 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -324,7 +324,9 @@ export class DatabaseEventBusStore implements EventBusStore { async readSubscription(id: string): Promise<{ events: EventParams[] }> { const result = await this.#db(TABLE_SUBSCRIPTIONS) // Read the target subscription so that we can use the read marker and topics - .with('sub', q => q.select().from(TABLE_SUBSCRIPTIONS).where({ id })) + .with('sub', q => + q.select().from(TABLE_SUBSCRIPTIONS).where({ id }).forUpdate(), + ) // Read the next batch of events for the subscription from its read marker .with('events', q => q From c38913a093603ba3289f79142094c6aba2c3d5b0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 08:41:04 +0200 Subject: [PATCH 31/83] events-backend: fix check for already consumed events Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 600a038793..e5e69d697d 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -339,9 +339,9 @@ export class DatabaseEventBusStore implements EventBusStore { this.#db.ref('topics').withSchema('sub').wrap('ANY(', ')'), ) // Skip events that have already been consumed by this subscription - .where( + .whereNot( this.#db.raw('?', id), - '<>', + '=', this.#db.ref('consumed_by').withSchema('event').wrap('ANY(', ')'), ) .where('event.id', '>', this.#db.ref('read_until').withSchema('sub')) From 79f73d9f92510077a09f402f849b7c758f455f9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 09:38:23 +0200 Subject: [PATCH 32/83] events-backend: added cleanup of old events Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.ts | 15 ++- .../src/service/hub/DatabaseEventBusStore.ts | 92 ++++++++++++++++++- .../src/service/hub/createEventBusRouter.ts | 4 + 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index 176d7e2f34..e8769b4a09 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -77,10 +77,19 @@ export const eventsPlugin = createBackendPlugin({ events: eventsServiceRef, database: coreServices.database, logger: coreServices.logger, + scheduler: coreServices.scheduler, httpAuth: coreServices.httpAuth, router: coreServices.httpRouter, }, - async init({ config, events, database, logger, httpAuth, router }) { + async init({ + config, + events, + database, + logger, + scheduler, + httpAuth, + router, + }) { const ingresses = Object.fromEntries( extensionPoint.httpPostIngresses.map(ingress => [ ingress.topic, @@ -97,7 +106,9 @@ export const eventsPlugin = createBackendPlugin({ const eventsRouter = Router(); http.bind(eventsRouter); - router.use(await createEventBusRouter({ database, logger, httpAuth })); + router.use( + await createEventBusRouter({ database, logger, httpAuth, scheduler }), + ); router.use(eventsRouter); router.addAuthPolicy({ diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index e5e69d697d..a6260cd0be 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -19,9 +19,15 @@ import { Knex } from 'knex'; import { DatabaseService, LoggerService, + SchedulerService, resolvePackagePath, } from '@backstage/backend-plugin-api'; import { ForwardedError, NotFoundError } from '@backstage/errors'; +import { HumanDuration, durationToMilliseconds } from '@backstage/types'; + +const WINDOW_SIZE_DEFAULT = 10000; +const WINDOW_MIN_AGE_DEFAULT = { minutes: 10 }; +const WINDOW_MAX_AGE_DEFAULT = { days: 1 }; const MAX_BATCH_SIZE = 10; const LISTENER_CONNECTION_TIMEOUT_MS = 60_000; @@ -208,6 +214,15 @@ export class DatabaseEventBusStore implements EventBusStore { static async create(options: { database: DatabaseService; logger: LoggerService; + scheduler: SchedulerService; + window?: { + /** Events within this range will never be deleted */ + minAge?: HumanDuration; + /** Events outside of this age will always be deleted */ + maxAge?: HumanDuration; + /** Events outside of this count will be deleted if they are outside the minAge window */ + size?: number; + }; }): Promise { const db = await options.database.getClient(); @@ -224,16 +239,43 @@ export class DatabaseEventBusStore implements EventBusStore { options.logger.info('DatabaseEventBusStore migrations ran successfully'); } - return new DatabaseEventBusStore(db, options.logger); + const store = new DatabaseEventBusStore( + db, + options.logger, + options.window?.size ?? WINDOW_SIZE_DEFAULT, + durationToMilliseconds(options.window?.minAge ?? WINDOW_MIN_AGE_DEFAULT), + durationToMilliseconds(options.window?.maxAge ?? WINDOW_MAX_AGE_DEFAULT), + ); + + options.scheduler.scheduleTask({ + id: 'event-bus-cleanup', + frequency: { seconds: 10 }, + timeout: { minutes: 1 }, + fn: store.#cleanup, + }); + + return store; } readonly #db: Knex; readonly #logger: LoggerService; + readonly #windowSize: number; + readonly #windowMinAge: number; + readonly #windowMaxAge: number; readonly #listener: DatabaseEventBusListener; - private constructor(db: Knex, logger: LoggerService) { + private constructor( + db: Knex, + logger: LoggerService, + windowSize: number, + windowMinAge: number, + windowMaxAge: number, + ) { this.#db = db; this.#logger = logger; + this.#windowSize = windowSize; + this.#windowMinAge = windowMinAge; + this.#windowMaxAge = windowMaxAge; this.#listener = new DatabaseEventBusListener(db.client, logger); } @@ -430,4 +472,50 @@ export class DatabaseEventBusStore implements EventBusStore { options.signal.addEventListener('abort', cancel); } } + + #cleanup = async () => { + try { + const eventCount = await this.#db + .delete() + .from(TABLE_EVENTS) + // Delete any events that are outside both the min age and size window + .where('created_at', '<', new Date(Date.now() - this.#windowMinAge)) + .andWhere( + 'id', + '=', + this.#db + .raw( + this.#db + .select('id') + .from(TABLE_EVENTS) + .orderBy('id', 'desc') + .offset(this.#windowSize), + ) + .wrap('ANY(ARRAY(', '))'), + ) + // If events are outside the max age they will always be deleted + .orWhere('created_at', '<', new Date(Date.now() - this.#windowMaxAge)); + this.#logger.info( + `Event cleanup resulted in ${eventCount} old events being deleted`, + ); + } catch (error) { + this.#logger.error('Event cleanup failed', error); + } + + try { + // Delete any subscribers that aren't keeping up with current events + const subscriberCount = await this.#db + .delete() + .from(TABLE_SUBSCRIPTIONS) + .where('read_until', '<', (q: Knex.QueryBuilder) => + q.select(this.#db.raw('MIN(id)')).from(TABLE_EVENTS), + ); + + this.#logger.info( + `Subscription cleanup resulted in ${subscriberCount} stale subscribers being deleted`, + ); + } catch (error) { + this.#logger.error('Subscription cleanup failed', error); + } + }; } diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 14237650fc..30cd137104 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -18,6 +18,7 @@ import { DatabaseService, HttpAuthService, LoggerService, + SchedulerService, } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; import Router from 'express-promise-router'; @@ -32,12 +33,14 @@ const DEFAULT_NOTIFY_TIMEOUT_MS = 55_000; // Just below 60s, which is a common H export async function createEventBusRouter(options: { logger: LoggerService; database: DatabaseService; + scheduler: SchedulerService; httpAuth: HttpAuthService; notifyTimeoutMs?: number; // for testing }): Promise { const { database, httpAuth, + scheduler, notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS, } = options; const logger = options.logger.child({ type: 'EventBus' }); @@ -50,6 +53,7 @@ export async function createEventBusRouter(options: { store = await DatabaseEventBusStore.create({ database, logger, + scheduler, }); } else { logger.info('Database is not PostgreSQL, using memory store'); From 0393323d176c6cf1caa5f717e5e3ca687587610e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 09:38:44 +0200 Subject: [PATCH 33/83] events-backend: more elaborate local development setup with multiple backends Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 100 ++++++++++++++++++++++++---- plugins/events-backend/package.json | 1 + yarn.lock | 1 + 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index 1b36751d59..b629cbf8d0 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -14,18 +14,60 @@ * limitations under the License. */ +import { WinstonLogger } from '@backstage/backend-app-api'; import { createBackend } from '@backstage/backend-defaults'; import { coreServices, createBackendPlugin, + createServiceFactory, } from '@backstage/backend-plugin-api'; +import { mockServices } from '@backstage/backend-test-utils'; import { eventsServiceRef } from '@backstage/plugin-events-node'; -const backend = createBackend(); +function makeBackend(port: number) { + const backend = createBackend(); -backend.add(import('../src/alpha')); + backend.add( + mockServices.rootConfig.factory({ + data: { + backend: { + baseUrl: `http://localhost:${port}`, + listen: { port }, + database: { + client: 'pg', + connection: { + host: 'localhost', + port: 5432, + user: process.env.USER || 'postgres', + }, + }, + }, + }, + }), + ); + backend.add( + createServiceFactory({ + service: coreServices.rootLogger, + deps: {}, + async factory() { + return WinstonLogger.create({ + meta: { + service: 'backstage', + port, + }, + // level: 'debug', + format: WinstonLogger.colorFormat(), + }); + }, + }), + ); + backend.add(import('../src/alpha')); -backend.add( + return backend; +} + +const backend7008 = makeBackend(7008); +backend7008.add( createBackendPlugin({ pluginId: 'producer', register(reg) { @@ -37,19 +79,29 @@ backend.add( }, async init({ events, logger, rootLifecycle }) { rootLifecycle.addStartupHook(async () => { - logger.info(`Publishing event to topic 'test'`); - await events.publish({ - eventPayload: { foo: 'bar' }, - topic: 'test', - }); + const publish = () => { + logger.info(`Publishing event to topic 'test'`); + events + .publish({ + eventPayload: { foo: 'bar' }, + topic: 'test', + }) + .catch(error => { + logger.error(`Failed to publish event from producer`, error); + }); + }; + publish(); + setInterval(publish, 10000); }); }, }); }, }), ); +backend7008.start(); -backend.add( +const backend7009 = makeBackend(7009); +backend7009.add( createBackendPlugin({ pluginId: 'consumer', register(reg) { @@ -60,10 +112,10 @@ backend.add( }, async init({ events, logger }) { await events.subscribe({ - id: 'test-1', + id: 'test', topics: ['test'], async onEvent(event) { - logger.info(`Received event: ${JSON.stringify(event, null, 2)}`); + logger.info(`Received event: ${JSON.stringify(event)}`); }, }); }, @@ -71,5 +123,29 @@ backend.add( }, }), ); +backend7009.start(); -backend.start(); +const backend7010 = makeBackend(7010); +backend7010.add( + createBackendPlugin({ + pluginId: 'consumer', + register(reg) { + reg.registerInit({ + deps: { + events: eventsServiceRef, + logger: coreServices.logger, + }, + async init({ events, logger }) { + await events.subscribe({ + id: 'test', + topics: ['test'], + async onEvent(event) { + logger.info(`Received event: ${JSON.stringify(event)}`); + }, + }); + }, + }); + }, + }), +); +backend7010.start(); diff --git a/plugins/events-backend/package.json b/plugins/events-backend/package.json index f7a5ecc0c2..06addb0546 100644 --- a/plugins/events-backend/package.json +++ b/plugins/events-backend/package.json @@ -67,6 +67,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-app-api": "workspace:^", "@backstage/backend-defaults": "workspace:^", "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^", diff --git a/yarn.lock b/yarn.lock index 58c69ee093..4feebf926f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6137,6 +6137,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-events-backend@workspace:plugins/events-backend" dependencies: + "@backstage/backend-app-api": "workspace:^" "@backstage/backend-common": ^0.25.0 "@backstage/backend-defaults": "workspace:^" "@backstage/backend-openapi-utils": "workspace:^" From d3a7aa37386fea62ac1656e4e08d8363f20e0bc5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 14:41:48 +0200 Subject: [PATCH 34/83] events-backend: refactor listening API to be promise based + return headers early Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 83 +++++++++---------- .../src/service/hub/MemoryEventBusStore.ts | 46 +++++----- .../src/service/hub/createEventBusRouter.ts | 56 +++++-------- .../events-backend/src/service/hub/types.ts | 6 +- .../src/api/DefaultEventsService.ts | 9 +- 5 files changed, 95 insertions(+), 105 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index a6260cd0be..2146df9b84 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -82,8 +82,8 @@ class DatabaseEventBusListener { readonly #listeners = new Set<{ topics: Set; - onNotify: (topicId: string) => void; - onError: () => void; + resolve: (result: { topic: string }) => void; + reject: (error: Error) => void; }>(); #connPromise?: Promise; @@ -95,43 +95,37 @@ class DatabaseEventBusListener { this.#logger = logger.child({ type: 'DatabaseEventBusListener' }); } - async listen( + async waitForUpdate( topics: Set, - onNotify: (topicId: string) => void, - onError: () => void, - ): Promise<() => void> { + signal: AbortSignal, + ): Promise<{ topic: string }> { if (this.#connTimeout) { clearTimeout(this.#connTimeout); this.#connTimeout = undefined; } + await this.#ensureConnection(); - const listener = { topics, onNotify, onError }; - this.#listeners.add(listener); + return new Promise<{ topic: string }>((resolve, reject) => { + const listener = { topics, resolve, reject }; + this.#listeners.add(listener); - return () => { - this.#listeners.delete(listener); - - // If we don't have any listeners, destroy the connection after a timeout - if (this.#listeners.size === 0) { - this.#connTimeout = setTimeout(() => { - this.#connPromise?.then(conn => { - this.#logger.info('Listener connection timed out, destroying'); - this.#connPromise = undefined; - this.#destroyConnection(conn); - }); - }, LISTENER_CONNECTION_TIMEOUT_MS); - } - }; + signal.addEventListener('abort', () => { + this.#listeners.delete(listener); + this.#maybeTimeoutConnection(); + }); + }); } #handleNotify(topic: string) { - this.#logger.info(`Listener received notification for topic '${topic}'`); + this.#logger.debug(`Listener received notification for topic '${topic}'`); for (const l of this.#listeners) { if (l.topics.has(topic)) { - l.onNotify(topic); + l.resolve({ topic }); + this.#listeners.delete(l); } } + this.#maybeTimeoutConnection(); } // We don't try to reconnect on error, instead we notify all listeners and let @@ -142,7 +136,23 @@ class DatabaseEventBusListener { error, ); for (const l of this.#listeners) { - l.onError(); + l.reject(new Error('Listener connection failed')); + } + this.#listeners.clear(); + this.#maybeTimeoutConnection(); + } + + #maybeTimeoutConnection() { + // If we don't have any listeners, destroy the connection after a timeout + if (this.#listeners.size === 0 && !this.#connTimeout) { + this.#connTimeout = setTimeout(() => { + this.#connTimeout = undefined; + this.#connPromise?.then(conn => { + this.#logger.info('Listener connection timed out, destroying'); + this.#connPromise = undefined; + this.#destroyConnection(conn); + }); + }, LISTENER_CONNECTION_TIMEOUT_MS); } } @@ -437,14 +447,12 @@ export class DatabaseEventBusStore implements EventBusStore { }; } - async listen( + async setupListener( subscriptionId: string, options: { signal: AbortSignal; - onNotify: (topicId: string) => void; - onError: () => void; }, - ): Promise { + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { const result = await this.#db(TABLE_SUBSCRIPTIONS) .select('topics') .where({ id: subscriptionId }) @@ -456,21 +464,12 @@ export class DatabaseEventBusStore implements EventBusStore { ); } - if (options.signal.aborted) { - return; - } + options.signal.throwIfAborted(); const topics = new Set(result.topics ?? []); - const cancel = await this.#listener.listen( - topics, - options.onNotify, - options.onError, - ); - if (options.signal.aborted) { - cancel(); - } else { - options.signal.addEventListener('abort', cancel); - } + return { + waitForUpdate: () => this.#listener.waitForUpdate(topics, options.signal), + }; } #cleanup = async () => { diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 98739fdb33..0fc9a2b14f 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -28,19 +28,19 @@ export class MemoryEventBusStore implements EventBusStore { >(); #listeners = new Set<{ topics: Set; - notify(topicId: string): void; + resolve(result: { topic: string }): void; }>(); async publish(options: { params: EventParams; subscriberIds: string[]; }): Promise<{ id: string } | undefined> { - const topicId = options.params.topic; + const topic = options.params.topic; const subscriberIds = new Set(options.subscriberIds); let hasOtherSubscribers = false; for (const sub of this.#subscribers.values()) { - if (sub.topics.has(topicId) && !subscriberIds.has(sub.id)) { + if (sub.topics.has(topic) && !subscriberIds.has(sub.id)) { hasOtherSubscribers = true; break; } @@ -53,8 +53,9 @@ export class MemoryEventBusStore implements EventBusStore { this.#events.push({ ...options.params, subscriberIds, seq: nextSeq }); for (const listener of this.#listeners) { - if (listener.topics.has(topicId)) { - listener.notify(topicId); + if (listener.topics.has(topic)) { + listener.resolve({ topic }); + this.#listeners.delete(listener); } } return { id: String(nextSeq) }; @@ -97,27 +98,30 @@ export class MemoryEventBusStore implements EventBusStore { return { events: events.map(event => ({ ...event, seq: undefined })) }; } - async listen( + async setupListener( subscriptionId: string, options: { signal: AbortSignal; - onNotify(topicId: string): void; - onError(): void; }, - ): Promise { - if (options.signal.aborted) { - return; - } + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { + return { + waitForUpdate: async () => { + options.signal.throwIfAborted(); - const sub = this.#subscribers.get(subscriptionId); - if (!sub) { - throw new Error(`Subscription not found`); - } - const listener = { topics: sub.topics, notify: options.onNotify }; - this.#listeners.add(listener); + const sub = this.#subscribers.get(subscriptionId); + if (!sub) { + throw new Error(`Subscription not found`); + } - options.signal.addEventListener('abort', () => { - this.#listeners.delete(listener); - }); + return new Promise<{ topic: string }>(resolve => { + const listener = { topics: sub.topics, resolve }; + this.#listeners.add(listener); + + options.signal.addEventListener('abort', () => { + this.#listeners.delete(listener); + }); + }); + }, + }; } } diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 30cd137104..d9c2bdf3dc 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -103,53 +103,22 @@ export async function createEventBusRouter(options: { }); const id = req.params.subscriptionId; - // Don't notify until we know the outcome of reading events - let resolveShouldNotify: (shouldNotify: boolean) => void; - const shouldNotifyPromise = new Promise(resolve => { - resolveShouldNotify = resolve; - }); - const controller = new AbortController(); req.on('end', () => controller.abort()); - let timeout: NodeJS.Timeout | undefined = undefined; - - let notified = false; - const notify = async (status: number) => { - if (!notified) { - clearTimeout(timeout); - notified = true; - if (await shouldNotifyPromise) { - res.status(status).end(); - } - } - }; - // By setting up the listener first we make sure we don't miss any events // that are published while reading. If an event is published we'll receive // a notification, which depending on the outcome of the read we may ignore - await store.listen(id, { + const listener = await store.setupListener(id, { signal: controller.signal, - onNotify() { - notify(204); - }, - onError() { - notify(500); - }, }); // By timing out requests we make sure they don't stall or that events get stuck. // For the caller there's no difference between a timeout and a // notifications, either way they should try reading again. - timeout = setTimeout(() => { - notify(204); + const timeout = setTimeout(() => { controller.abort(); }, notifyTimeoutMs); - shouldNotifyPromise.then(shouldNotify => { - if (!shouldNotify) { - clearTimeout(timeout); - } - }); try { const { events } = await store.readSubscription(id); @@ -162,10 +131,27 @@ export async function createEventBusRouter(options: { if (events.length > 0) { res.json({ events }); } else { - resolveShouldNotify!(true); + res.status(202); + res.flushHeaders(); + + try { + const { topic } = await listener.waitForUpdate(); + logger.info( + `Received notification for subscription '${id}' for topic '${topic}'`, + { subject: credentials.principal.subject }, + ); + } finally { + // A small extra delay ensures a more even spread of events across + // consumers in case some consumers are faster than others + await new Promise(resolve => + setTimeout(resolve, 1 + Math.random() * 9), + ); + res.end(); + } } } finally { - resolveShouldNotify!(false); + controller.abort(); + clearTimeout(timeout); } }, ); diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 55159352b4..87e62049a9 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -26,12 +26,10 @@ export type EventBusStore = { readSubscription(id: string): Promise<{ events: EventParams[] }>; - listen( + setupListener( subscriptionId: string, options: { signal: AbortSignal; - onNotify(topicId: string): void; - onError(): void; }, - ): Promise; + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }>; }; diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index d1d26b1723..7811349680 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -205,10 +205,12 @@ class PluginEventsService implements EventsService { } backoffMs = POLL_BACKOFF_START_MS; - // 204 arrives after a blocking response and means there are new events - // available or we timed out, either way should should try to read events + // 202 means there were no immediately available events, but the + // response will block until either new events are available or the + // request times out. In both cases we should should try to read events // immediately again - if (res.status === 204) { + if (res.status === 202) { + await res.body?.getReader()?.closed; process.nextTick(poll); } else if (res.status === 200) { const data = await res.json(); @@ -316,6 +318,7 @@ export class DefaultEventsService implements EventsService { options && new DefaultApiClient({ discoveryApi: options.discovery, + fetchApi: { fetch }, // use native node fetch }); const logger = options?.logger ?? this.logger; return new PluginEventsService( From cd243f994fa3714f22206a38b3fc3de48faa5990 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 15:27:43 +0200 Subject: [PATCH 35/83] events-node: graceful shutdown of events service Signed-off-by: Patrik Oldsberg --- .../src/api/DefaultEventsService.ts | 92 +++++++++++++------ plugins/events-node/src/service.ts | 7 +- 2 files changed, 72 insertions(+), 27 deletions(-) diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 7811349680..86083edeb2 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -17,6 +17,7 @@ import { AuthService, DiscoveryService, + LifecycleService, LoggerService, } from '@backstage/backend-plugin-api'; import { EventParams } from './EventParams'; @@ -111,34 +112,39 @@ class PluginEventsService implements EventsService { ) {} async publish(params: EventParams): Promise { - const { notifiedSubscribers } = await this.localBus.publish(params); + const lock = this.#getShutdownLock(); + try { + const { notifiedSubscribers } = await this.localBus.publish(params); - if (!this.client) { - return; - } - const token = await this.#getToken(); - if (!token) { - return; - } - const res = await this.client.postEvent( - { - body: { - event: { payload: params.eventPayload, topic: params.topic }, - subscriptionIds: notifiedSubscribers, - }, - }, - { token }, - ); - - if (!res.ok) { - if (res.status === 404) { - this.logger.warn( - `Event publish request failed with status 404, events backend not found. Future events will not be persisted.`, - ); - delete this.client; + if (!this.client) { return; } - throw await ResponseError.fromResponse(res); + const token = await this.#getToken(); + if (!token) { + return; + } + const res = await this.client.postEvent( + { + body: { + event: { payload: params.eventPayload, topic: params.topic }, + subscriptionIds: notifiedSubscribers, + }, + }, + { token }, + ); + + if (!res.ok) { + if (res.status === 404) { + this.logger.warn( + `Event publish request failed with status 404, events backend not found. Future events will not be persisted.`, + ); + delete this.client; + return; + } + throw await ResponseError.fromResponse(res); + } + } finally { + lock.release(); } } @@ -188,6 +194,7 @@ class PluginEventsService implements EventsService { if (!this.client) { return; } + const lock = this.#getShutdownLock(); try { const token = await this.#getToken(); if (!token) { @@ -210,6 +217,7 @@ class PluginEventsService implements EventsService { // request times out. In both cases we should should try to read events // immediately again if (res.status === 202) { + lock.release(); await res.body?.getReader()?.closed; process.nextTick(poll); } else if (res.status === 200) { @@ -247,6 +255,8 @@ class PluginEventsService implements EventsService { backoffMs * POLL_BACKOFF_FACTOR, POLL_BACKOFF_MAX_MS, ); + } finally { + lock.release(); } }; poll(); @@ -276,6 +286,31 @@ class PluginEventsService implements EventsService { throw error; } } + + async shutdown() { + this.#isShuttingDown = true; + await Promise.all(this.#shutdownLocks); + } + + #isShuttingDown = false; + #shutdownLocks: Promise[] = []; + + // This locking mechanism helps ensure that we are either idle or waiting for + // a blocked events call before shutting down. It increases out changes of + // never dropping any events on shutdown. + #getShutdownLock(): { release(): void } { + if (this.#isShuttingDown) { + throw new Error('Service is shutting down'); + } + + let release: () => void; + this.#shutdownLocks.push( + new Promise(resolve => { + release = resolve; + }), + ); + return { release: release! }; + } } /** @@ -312,6 +347,7 @@ export class DefaultEventsService implements EventsService { discovery: DiscoveryService; logger: LoggerService; auth: AuthService; + lifecycle: LifecycleService; }, ): EventsService { const client = @@ -321,13 +357,17 @@ export class DefaultEventsService implements EventsService { fetchApi: { fetch }, // use native node fetch }); const logger = options?.logger ?? this.logger; - return new PluginEventsService( + const service = new PluginEventsService( pluginId, this.localBus, logger, client, options?.auth, ); + options?.lifecycle.addShutdownHook(async () => { + await service.shutdown(); + }); + return service; } /** @deprecated this method should not be called */ diff --git a/plugins/events-node/src/service.ts b/plugins/events-node/src/service.ts index 8d754f6957..4f758f44e6 100644 --- a/plugins/events-node/src/service.ts +++ b/plugins/events-node/src/service.ts @@ -40,15 +40,20 @@ export const eventsServiceFactory = createServiceFactory({ rootLogger: coreServices.rootLogger, discovery: coreServices.discovery, logger: coreServices.logger, + lifecycle: coreServices.lifecycle, auth: coreServices.auth, }, async createRootContext({ rootLogger }) { return DefaultEventsService.create({ logger: rootLogger }); }, - async factory({ pluginMetadata, discovery, logger, auth }, eventsService) { + async factory( + { pluginMetadata, discovery, logger, lifecycle, auth }, + eventsService, + ) { return eventsService.forPlugin(pluginMetadata.getId(), { discovery, logger, + lifecycle, auth, }); }, From 2e2fc18dcc721e2c9242317236b82c3620bd575c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 15:34:54 +0200 Subject: [PATCH 36/83] events-backend: update schema Signed-off-by: Patrik Oldsberg --- .../events-backend/src/schema/openapi.generated.ts | 11 +++-------- plugins/events-backend/src/schema/openapi.yaml | 11 ++--------- .../events-node/src/generated/models/Event.model.ts | 5 ++++- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index d9b3c9045c..4b1db5aeb0 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -73,7 +73,6 @@ export const spec = { description: 'The topic that the event is published on', }, payload: { - $ref: '#/components/schemas/JsonObject', description: 'The event payload', }, }, @@ -117,11 +116,6 @@ export const spec = { }, required: ['error', 'request', 'response'], }, - JsonObject: { - type: 'object', - properties: {}, - additionalProperties: {}, - }, }, securitySchemes: { JWT: { @@ -269,8 +263,9 @@ export const spec = { }, }, }, - '201': { - description: 'Block poll response, new events are available', + '202': { + description: + 'No new events are available. Response will block until the client should try again.', }, default: { $ref: '#/components/responses/ErrorResponse', diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index 632a884139..0d228a7b0b 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -39,7 +39,6 @@ components: type: string description: The topic that the event is published on payload: - $ref: '#/components/schemas/JsonObject' description: The event payload Error: @@ -76,12 +75,6 @@ components: - error - request - response - - JsonObject: - type: object - properties: {} - # Free form object. - additionalProperties: {} securitySchemes: JWT: type: http @@ -178,8 +171,8 @@ paths: $ref: '#/components/schemas/Event' required: - results - '201': - description: Block poll response, new events are available + '202': + description: No new events are available. Response will block until the client should try again. default: $ref: '#/components/responses/ErrorResponse' security: diff --git a/plugins/events-node/src/generated/models/Event.model.ts b/plugins/events-node/src/generated/models/Event.model.ts index b5a13b4d8d..a196b42db7 100644 --- a/plugins/events-node/src/generated/models/Event.model.ts +++ b/plugins/events-node/src/generated/models/Event.model.ts @@ -23,5 +23,8 @@ export interface Event { * The topic that the event is published on */ topic: string; - payload: { [key: string]: any }; + /** + * The event payload + */ + payload: any | null; } From 88a1be5ec091c06a9de6c5fe0b042712bda35982 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 15:41:38 +0200 Subject: [PATCH 37/83] events-node: restore EventParams type Signed-off-by: Patrik Oldsberg --- plugins/events-node/src/api/EventParams.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/events-node/src/api/EventParams.ts b/plugins/events-node/src/api/EventParams.ts index 9cd88b160f..19a562aee0 100644 --- a/plugins/events-node/src/api/EventParams.ts +++ b/plugins/events-node/src/api/EventParams.ts @@ -14,12 +14,10 @@ * limitations under the License. */ -import { JsonValue } from '@backstage/types'; - /** * @public */ -export interface EventParams { +export interface EventParams { /** * Topic for which this event should be published. */ @@ -31,5 +29,5 @@ export interface EventParams { /** * Metadata (e.g., HTTP headers and similar for events received from external). */ - metadata?: { [name in string]?: string | string[] }; + metadata?: Record; } From 3f7e67bf33058f73b8e977dc4bc55ed64ecf757a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 16:03:40 +0200 Subject: [PATCH 38/83] events-backend: add event bus docs Signed-off-by: Patrik Oldsberg --- .../src/service/hub/createEventBusRouter.ts | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index d9c2bdf3dc..0ebd25d304 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -30,6 +30,93 @@ import { EventParams } from '@backstage/plugin-events-node'; const DEFAULT_NOTIFY_TIMEOUT_MS = 55_000; // Just below 60s, which is a common HTTP timeout +/* + +# Event Bus + +This comment describes the event bus that is implemented here in the events +backend, and by default used by the events service. + +## Overview + +The events bus implements a subscription mechanism where subscribers must exist +upfront for events to be stored. It uses a single inbox for all events, with +each subscriber having its own pointer for how far into the inbox it has read. + +In order to avoid busy polling, the API uses a long-polling mechanism where a +request is left hanging until the client should try to read again. + +The event bus is not implemented with any guarantees of events being consumed, +but it does aim to make it unlikely that events are dropped + +## API + +### POST /bus/v1/events + +This endpoint is used to publish new events to the event bus on a specific +topic. It can optionally include a set of subscription IDs for subscribers that +have already been notified of the event. This is to enable an optimization where +we notify subscribers locally if possible, and avoid the need for events to be +relayed through the events bus at all of possible. + +For an event to be published and stored there must already exist a subscriber +that is subscribed to the event's topic, and that hasn't already been notified +of the event. If no such subscriber is found, the event will be discarded. + +### PUT /bus/v1/subscriptions/:subscriptionId + +This endpoint is used to create or update a subscriptions. Subscriptions are +shared across the entire bus and divided by subscription ID. Multiple clients +can be reading events from the same subscription at the same time, but only one +of those clients will receive each event. This enables division of work by using +the same subscriber ID across multiple instances, as well as broadcasting by +ensuring separate subscribers IDs. + +### GET /bus/v1/subscriptions/:subscriptionId/events + +This endpoint is used to read events from a subscription. It will return a batch +of events for the subscribed topics that have not yet been read by the +subscription. If no such events are available, the endpoint will return a 202 +response and then hang end response until an event is available or a timeout is +reached. This allows clients to call this endpoint in a loop but will keep +traffic overhead fairly low. + +## Delivery guarantees + +When reading events from the event bus, clients should always implement a +graceful shutdown where they process any events that are received from the +events endpoint before shutting down. This is also the reason that the events +endpoint does not return any events when responding with a 202 blocking the +response, because there would otherwise be a race condition where the events +might be lost in transit if the client shuts down. By always sending an empty +response and requiring the client to send another request, we ensure that the +client is prepared to halt shutdown until the request had been fully processed. + +## Local processing optimization + +When possible, events will be processed locally before sent to the event bus. +The client will also inform the bus of which subscriptions have already been +notified of the event, so that the bus can completely avoid storing an event if +it has already been fully consumed by all subscribers. + +## Automated cleanup & event window + +Events are deleted once they are outside the guaranteed storage window. By +default the window 10 minutes for all events, and 24 hours for the last 10000 +events. This ensures that the event log doesn't grow indefinitely, while still +allowing subscribers with restarts or outages to catch up to past events, +ensuring that events are likely not lost. + +Subscriptions are also cleaned up if their read pointer falls outside of the +current event window. This ensures that stale subscribers don't accumulate and +cause unnecessary storage of events. + +*/ + +/** + * Creates a new event bus router + * @internal + */ export async function createEventBusRouter(options: { logger: LoggerService; database: DatabaseService; From 4ea914f12e14e95c27e3a5610a79e67c3ae70216 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 16:03:58 +0200 Subject: [PATCH 39/83] events-node: update API report Signed-off-by: Patrik Oldsberg --- plugins/events-node/api-report.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/plugins/events-node/api-report.md b/plugins/events-node/api-report.md index d8a2281faa..a735007e3b 100644 --- a/plugins/events-node/api-report.md +++ b/plugins/events-node/api-report.md @@ -3,6 +3,9 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AuthService } from '@backstage/backend-plugin-api'; +import { DiscoveryService } from '@backstage/backend-plugin-api'; +import { LifecycleService } from '@backstage/backend-plugin-api'; import { LoggerService } from '@backstage/backend-plugin-api'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; @@ -11,11 +14,19 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; export class DefaultEventsService implements EventsService { // (undocumented) static create(options: { logger: LoggerService }): DefaultEventsService; - forPlugin(pluginId: string): EventsService; - // (undocumented) - publish(params: EventParams): Promise; - // (undocumented) - subscribe(options: EventsServiceSubscribeOptions): Promise; + forPlugin( + pluginId: string, + options?: { + discovery: DiscoveryService; + logger: LoggerService; + auth: AuthService; + lifecycle: LifecycleService; + }, + ): EventsService; + // @deprecated (undocumented) + publish(_params: EventParams): Promise; + // @deprecated (undocumented) + subscribe(_options: EventsServiceSubscribeOptions): Promise; } // @public @deprecated From 32371edb4b793d9fc5206ab01ca11fa7300cbf9c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 16:21:59 +0200 Subject: [PATCH 40/83] events-backend: fix event bus timeout handling Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 1 + .../events-backend/src/service/hub/createEventBusRouter.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 2146df9b84..720b62bb9c 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -113,6 +113,7 @@ class DatabaseEventBusListener { signal.addEventListener('abort', () => { this.#listeners.delete(listener); this.#maybeTimeoutConnection(); + reject(signal.reason); }); }); } diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 0ebd25d304..4a6c34bb6c 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -227,6 +227,12 @@ export async function createEventBusRouter(options: { `Received notification for subscription '${id}' for topic '${topic}'`, { subject: credentials.principal.subject }, ); + } catch (error) { + if (error === controller.signal.reason) { + res.end(); + } else { + throw error; + } } finally { // A small extra delay ensures a more even spread of events across // consumers in case some consumers are faster than others From f13bd3e987ce012c153a547a2d022fa860a65903 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 25 May 2024 16:22:32 +0200 Subject: [PATCH 41/83] events-node: try to recreate subscription if it is cleaned up during polling Signed-off-by: Patrik Oldsberg --- .../src/api/DefaultEventsService.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 86083edeb2..74e37100e0 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -182,11 +182,12 @@ class PluginEventsService implements EventsService { throw await ResponseError.fromResponse(res); } - this.#startPolling(subscriptionId, options.onEvent); + this.#startPolling(subscriptionId, options.topics, options.onEvent); } #startPolling( subscriptionId: string, + topics: string[], onEvent: EventsServiceSubscribeOptions['onEvent'], ) { let backoffMs = POLL_BACKOFF_START_MS; @@ -208,6 +209,21 @@ class PluginEventsService implements EventsService { ); if (!res.ok) { + if (res.status === 404) { + this.logger.info( + `Polling event subscription resulted in a 404, recreating subscription`, + ); + const putRes = await this.client.putSubscription( + { + path: { subscriptionId }, + body: { topics }, + }, + { token }, + ); + if (!putRes.ok) { + throw await ResponseError.fromResponse(res); + } + } throw await ResponseError.fromResponse(res); } backoffMs = POLL_BACKOFF_START_MS; From 1154a730f68cbd0da3a10104e52a3c22cfe7f888 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 17:56:43 +0200 Subject: [PATCH 42/83] events-backend: fix schema and response shape Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/schema/openapi.generated.ts | 2 +- plugins/events-backend/src/schema/openapi.yaml | 4 ++-- .../events-backend/src/service/hub/createEventBusRouter.ts | 7 ++++++- .../models/GetSubscriptionEvents200Response.model.ts | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index 4b1db5aeb0..d6278ff070 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -250,6 +250,7 @@ export const spec = { 'application/json': { schema: { type: 'object', + required: ['events'], properties: { events: { type: 'array', @@ -258,7 +259,6 @@ export const spec = { }, }, }, - required: ['results'], }, }, }, diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index 0d228a7b0b..48f2c74e5e 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -164,13 +164,13 @@ paths: application/json: schema: type: object + required: + - events properties: events: type: array items: $ref: '#/components/schemas/Event' - required: - - results '202': description: No new events are available. Response will block until the client should try again. default: diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 4a6c34bb6c..a5f67538bf 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -216,7 +216,12 @@ export async function createEventBusRouter(options: { ); if (events.length > 0) { - res.json({ events }); + res.json({ + events: events.map(event => ({ + topic: event.topic, + payload: event.eventPayload, + })), + }); } else { res.status(202); res.flushHeaders(); diff --git a/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts index 653bfaf3fc..c9fc37f5bb 100644 --- a/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts +++ b/plugins/events-node/src/generated/models/GetSubscriptionEvents200Response.model.ts @@ -20,5 +20,5 @@ import { Event } from '../models/Event.model'; export interface GetSubscriptionEvents200Response { - events?: Array; + events: Array; } From 39ec9c97b838b40d857976f10eaee27db0b769e1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 17:57:04 +0200 Subject: [PATCH 43/83] events-backend: add event bus integration tests Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 155 +++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 48ce1090fb..21be572a20 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -14,11 +14,17 @@ * limitations under the License. */ +/* eslint-disable jest/expect-expect */ + import { createBackendModule, createServiceFactory, } from '@backstage/backend-plugin-api'; -import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { + mockCredentials, + mockServices, + startTestBackend, +} from '@backstage/backend-test-utils'; import { eventsServiceRef } from '@backstage/plugin-events-node'; import { eventsExtensionPoint } from '@backstage/plugin-events-node/alpha'; import { TestEventsService } from '@backstage/plugin-events-backend-test-utils'; @@ -92,4 +98,151 @@ describe('eventsPlugin', () => { test: 'fake-ext', }); }); + + describe('event bus', () => { + it('should be possible to publish events as a service', async () => { + const { server } = await startTestBackend({ + features: [eventsPlugin()], + }); + + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.none.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(401); + + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.user.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(403); + + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(204); // 204, since there are no subscribers + }); + + it('should be possible to subscribe as a service and receive an event', async () => { + const { server } = await startTestBackend({ + features: [eventsPlugin()], + }); + + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.none.header()) + .send({ topics: ['test'] }) + .expect(401); + + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.user.header()) + .send({ topics: ['test'] }) + .expect(403); + + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.none.header()) + .send({ topics: ['test'] }) + .expect(401); + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.user.header()) + .send({ topics: ['test'] }) + .expect(403); + + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(201); // 201, since there is a subscriber + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + }); + }); + + it('should only send an event for each subscriber once', async () => { + const { server } = await startTestBackend({ + features: [eventsPlugin()], + }); + + // 2 subscribers + await request(server) + .put('/api/events/bus/v1/subscriptions/tester-1') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + await request(server) + .put('/api/events/bus/v1/subscriptions/tester-2') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + + // A single event + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(201); + + // Single client for subscriber 1 gets the event + await request(server) + .get('/api/events/bus/v1/subscriptions/tester-1/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + // Two clients for subscriber 2, only one gets the event + const res1 = request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }); + const res2 = request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }); + + const res = await Promise.race([res1, res2]); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + // Post another event, which triggers the other client to return + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 2 } } }) + .expect(201); + + const otherRes = await Promise.all([res1, res2]).then(rs => + rs.find(r => r !== res), + ); + expect(otherRes?.status).toBe(202); + + // Reading subscriber 2 should now return the second event only + await request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 2 } }], + }); + }); }); From 973278ffe1abaf2321d631227e19a7549374cf3f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 19:22:29 +0200 Subject: [PATCH 44/83] events-backend: graceful shutdown of listener Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.ts | 10 ++++++- .../src/service/hub/DatabaseEventBusStore.ts | 29 +++++++++++++++++-- .../src/service/hub/createEventBusRouter.ts | 4 +++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.ts b/plugins/events-backend/src/service/EventsPlugin.ts index e8769b4a09..1369ee98ca 100644 --- a/plugins/events-backend/src/service/EventsPlugin.ts +++ b/plugins/events-backend/src/service/EventsPlugin.ts @@ -78,6 +78,7 @@ export const eventsPlugin = createBackendPlugin({ database: coreServices.database, logger: coreServices.logger, scheduler: coreServices.scheduler, + lifecycle: coreServices.lifecycle, httpAuth: coreServices.httpAuth, router: coreServices.httpRouter, }, @@ -87,6 +88,7 @@ export const eventsPlugin = createBackendPlugin({ database, logger, scheduler, + lifecycle, httpAuth, router, }) { @@ -107,7 +109,13 @@ export const eventsPlugin = createBackendPlugin({ http.bind(eventsRouter); router.use( - await createEventBusRouter({ database, logger, httpAuth, scheduler }), + await createEventBusRouter({ + database, + logger, + httpAuth, + scheduler, + lifecycle, + }), ); router.use(eventsRouter); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 720b62bb9c..dfcf10ece4 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -18,6 +18,7 @@ import { EventBusStore } from './types'; import { Knex } from 'knex'; import { DatabaseService, + LifecycleService, LoggerService, SchedulerService, resolvePackagePath, @@ -86,6 +87,7 @@ class DatabaseEventBusListener { reject: (error: Error) => void; }>(); + #isShuttingDown = false; #connPromise?: Promise; #connTimeout?: NodeJS.Timeout; #keepaliveInterval?: NodeJS.Timeout; @@ -118,6 +120,17 @@ class DatabaseEventBusListener { }); } + async shutdown() { + if (this.#isShuttingDown) { + return; + } + this.#isShuttingDown = true; + const conn = await this.#connPromise?.catch(() => undefined); + if (conn) { + this.#destroyConnection(conn); + } + } + #handleNotify(topic: string) { this.#logger.debug(`Listener received notification for topic '${topic}'`); for (const l of this.#listeners) { @@ -169,6 +182,9 @@ class DatabaseEventBusListener { } async #ensureConnection() { + if (this.#isShuttingDown) { + throw new Error('Listener is shutting down'); + } if (this.#connPromise) { await this.#connPromise; return; @@ -226,6 +242,7 @@ export class DatabaseEventBusStore implements EventBusStore { database: DatabaseService; logger: LoggerService; scheduler: SchedulerService; + lifecycle: LifecycleService; window?: { /** Events within this range will never be deleted */ minAge?: HumanDuration; @@ -250,9 +267,12 @@ export class DatabaseEventBusStore implements EventBusStore { options.logger.info('DatabaseEventBusStore migrations ran successfully'); } + const listener = new DatabaseEventBusListener(db.client, options.logger); + const store = new DatabaseEventBusStore( db, options.logger, + listener, options.window?.size ?? WINDOW_SIZE_DEFAULT, durationToMilliseconds(options.window?.minAge ?? WINDOW_MIN_AGE_DEFAULT), durationToMilliseconds(options.window?.maxAge ?? WINDOW_MAX_AGE_DEFAULT), @@ -265,29 +285,34 @@ export class DatabaseEventBusStore implements EventBusStore { fn: store.#cleanup, }); + options.lifecycle.addShutdownHook(async () => { + await listener.shutdown(); + }); + return store; } readonly #db: Knex; readonly #logger: LoggerService; + readonly #listener: DatabaseEventBusListener; readonly #windowSize: number; readonly #windowMinAge: number; readonly #windowMaxAge: number; - readonly #listener: DatabaseEventBusListener; private constructor( db: Knex, logger: LoggerService, + listener: DatabaseEventBusListener, windowSize: number, windowMinAge: number, windowMaxAge: number, ) { this.#db = db; this.#logger = logger; + this.#listener = listener; this.#windowSize = windowSize; this.#windowMinAge = windowMinAge; this.#windowMaxAge = windowMaxAge; - this.#listener = new DatabaseEventBusListener(db.client, logger); } async publish(options: { diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index a5f67538bf..1a702c624b 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -17,6 +17,7 @@ import { DatabaseService, HttpAuthService, + LifecycleService, LoggerService, SchedulerService, } from '@backstage/backend-plugin-api'; @@ -121,6 +122,7 @@ export async function createEventBusRouter(options: { logger: LoggerService; database: DatabaseService; scheduler: SchedulerService; + lifecycle: LifecycleService; httpAuth: HttpAuthService; notifyTimeoutMs?: number; // for testing }): Promise { @@ -128,6 +130,7 @@ export async function createEventBusRouter(options: { database, httpAuth, scheduler, + lifecycle, notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS, } = options; const logger = options.logger.child({ type: 'EventBus' }); @@ -141,6 +144,7 @@ export async function createEventBusRouter(options: { database, logger, scheduler, + lifecycle, }); } else { logger.info('Database is not PostgreSQL, using memory store'); From 105d68cd09d8a8e8953072b3d6b66f751b6f8c62 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 21:12:55 +0200 Subject: [PATCH 45/83] eventa-backend: fix for pg listener not actually waiting for initial setup Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index dfcf10ece4..53f55ce67f 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -97,10 +97,10 @@ class DatabaseEventBusListener { this.#logger = logger.child({ type: 'DatabaseEventBusListener' }); } - async waitForUpdate( + async setupListener( topics: Set, signal: AbortSignal, - ): Promise<{ topic: string }> { + ): Promise<{ waitForUpdate(): Promise<{ topic: string }> }> { if (this.#connTimeout) { clearTimeout(this.#connTimeout); this.#connTimeout = undefined; @@ -108,7 +108,7 @@ class DatabaseEventBusListener { await this.#ensureConnection(); - return new Promise<{ topic: string }>((resolve, reject) => { + const updatePromise = new Promise<{ topic: string }>((resolve, reject) => { const listener = { topics, resolve, reject }; this.#listeners.add(listener); @@ -118,6 +118,11 @@ class DatabaseEventBusListener { reject(signal.reason); }); }); + + // Ignore unhandled rejections + updatePromise.catch(() => {}); + + return { waitForUpdate: () => updatePromise }; } async shutdown() { @@ -492,10 +497,10 @@ export class DatabaseEventBusStore implements EventBusStore { options.signal.throwIfAborted(); - const topics = new Set(result.topics ?? []); - return { - waitForUpdate: () => this.#listener.waitForUpdate(topics, options.signal), - }; + return this.#listener.setupListener( + new Set(result.topics ?? []), + options.signal, + ); } #cleanup = async () => { From d2e733c8b6ef860b91c734f2dfb44c9b0ac37f25 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 21:13:25 +0200 Subject: [PATCH 46/83] events-backend: avoid immediate cleanup job to avoid it running for integration tests Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 53f55ce67f..9248e1c0e5 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -287,6 +287,7 @@ export class DatabaseEventBusStore implements EventBusStore { id: 'event-bus-cleanup', frequency: { seconds: 10 }, timeout: { minutes: 1 }, + initialDelay: { seconds: 10 }, fn: store.#cleanup, }); From 65e58b53d805deb4f50abc35cad306131bd2df06 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 21:14:02 +0200 Subject: [PATCH 47/83] events-backend: fix for subscription failing if there are no events Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 9248e1c0e5..0a8dddfe51 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -392,7 +392,9 @@ export class DatabaseEventBusStore implements EventBusStore { id, updated_at: this.#db.fn.now(), topics, - read_until: this.#db(TABLE_EVENTS).max('id') as any, // TODO: figure out TS, + read_until: this.#db.raw( + `( SELECT COALESCE(MAX("id"), 0) FROM "${TABLE_EVENTS}" )`, + ), }) .onConflict('id') .merge(['topics', 'updated_at']) From 840cab000095004ff14c63a537084973383113f6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sun, 26 May 2024 21:14:20 +0200 Subject: [PATCH 48/83] events-backend: add integration tests for postgres Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 287 ++++++++++-------- 1 file changed, 159 insertions(+), 128 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 21be572a20..e73e636d24 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -21,6 +21,8 @@ import { createServiceFactory, } from '@backstage/backend-plugin-api'; import { + TestDatabaseId, + TestDatabases, mockCredentials, mockServices, startTestBackend, @@ -100,149 +102,178 @@ describe('eventsPlugin', () => { }); describe('event bus', () => { - it('should be possible to publish events as a service', async () => { - const { server } = await startTestBackend({ - features: [eventsPlugin()], - }); - - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.none.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(401); - - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.user.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(403); - - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(204); // 204, since there are no subscribers + const databases = TestDatabases.create({ + ids: ['SQLITE_3', 'POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], }); - it('should be possible to subscribe as a service and receive an event', async () => { - const { server } = await startTestBackend({ - features: [eventsPlugin()], - }); + async function mockKnexFactory(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + return mockServices.database.mock({ + getClient: async () => knex, + }).factory; + } - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') - .set('authorization', mockCredentials.none.header()) - .send({ topics: ['test'] }) - .expect(401); + it.each(databases.eachSupportedId())( + 'should be possible to publish events as a service, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const { server } = backend; - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') - .set('authorization', mockCredentials.user.header()) - .send({ topics: ['test'] }) - .expect(403); + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.none.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(401); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.user.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(403); - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') - .set('authorization', mockCredentials.none.header()) - .send({ topics: ['test'] }) - .expect(401); + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(204); // 204, since there are no subscribers - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') - .set('authorization', mockCredentials.user.header()) - .send({ topics: ['test'] }) - .expect(403); + await backend.stop(); + }, + ); - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(201); // 201, since there is a subscriber + it.each(databases.eachSupportedId())( + 'should be possible to subscribe as a service and receive an event, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const { server } = backend; - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.none.header()) + .send({ topics: ['test'] }) + .expect(401); + + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.user.header()) + .send({ topics: ['test'] }) + .expect(403); + + await request(server) + .put('/api/events/bus/v1/subscriptions/tester') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.none.header()) + .send({ topics: ['test'] }) + .expect(401); + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.user.header()) + .send({ topics: ['test'] }) + .expect(403); + + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(201); // 201, since there is a subscriber + + await request(server) + .get('/api/events/bus/v1/subscriptions/tester/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + await backend.stop(); + }, + ); + + it.each(databases.eachSupportedId())( + 'should only send an event for each subscriber once, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const { server } = backend; + + // 2 subscribers + await request(server) + .put('/api/events/bus/v1/subscriptions/tester-1') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + await request(server) + .put('/api/events/bus/v1/subscriptions/tester-2') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(201); + + // A single event + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 1 } } }) + .expect(201); + + // Single client for subscriber 1 gets the event + await request(server) + .get('/api/events/bus/v1/subscriptions/tester-1/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); + + // Two clients for subscriber 2, only one gets the event + const res1 = request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }); + const res2 = request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }); + + const res = await Promise.race([res1, res2]); + expect(res.status).toBe(200); + expect(res.body).toEqual({ events: [{ topic: 'test', payload: { n: 1 } }], }); - }); - }); - it('should only send an event for each subscriber once', async () => { - const { server } = await startTestBackend({ - features: [eventsPlugin()], - }); + // Post another event, which triggers the other client to return + await request(server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic: 'test', payload: { n: 2 } } }) + .expect(201); - // 2 subscribers - await request(server) - .put('/api/events/bus/v1/subscriptions/tester-1') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester-2') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); + const otherRes = await Promise.all([res1, res2]).then(rs => + rs.find(r => r !== res), + ); + expect(otherRes?.status).toBe(202); - // A single event - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(201); + // Reading subscriber 2 should now return the second event only + await request(server) + .get('/api/events/bus/v1/subscriptions/tester-2/events') + .set('authorization', mockCredentials.service.header()) + .send({ topics: ['test'] }) + .expect(200, { + events: [{ topic: 'test', payload: { n: 2 } }], + }); - // Single client for subscriber 1 gets the event - await request(server) - .get('/api/events/bus/v1/subscriptions/tester-1/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { - events: [{ topic: 'test', payload: { n: 1 } }], - }); - - // Two clients for subscriber 2, only one gets the event - const res1 = request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }); - const res2 = request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }); - - const res = await Promise.race([res1, res2]); - expect(res.status).toBe(200); - expect(res.body).toEqual({ - events: [{ topic: 'test', payload: { n: 1 } }], - }); - - // Post another event, which triggers the other client to return - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 2 } } }) - .expect(201); - - const otherRes = await Promise.all([res1, res2]).then(rs => - rs.find(r => r !== res), + await backend.stop(); + }, ); - expect(otherRes?.status).toBe(202); - - // Reading subscriber 2 should now return the second event only - await request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { - events: [{ topic: 'test', payload: { n: 2 } }], - }); }); }); From 12dab612507fd08fa42c318158f5d1785b4196a2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 09:47:28 +0200 Subject: [PATCH 49/83] events-backend: use consumedBy everywhere instead of subscriptionIds Signed-off-by: Patrik Oldsberg --- .../src/schema/openapi.generated.ts | 2 +- .../events-backend/src/schema/openapi.yaml | 2 +- .../src/service/hub/DatabaseEventBusStore.ts | 7 ++--- .../src/service/hub/MemoryEventBusStore.ts | 14 +++++----- .../src/service/hub/createEventBusRouter.ts | 26 +++++++++++-------- .../events-backend/src/service/hub/types.ts | 2 +- .../src/api/DefaultEventsService.ts | 2 +- .../models/PostEventRequest.model.ts | 2 +- 8 files changed, 30 insertions(+), 27 deletions(-) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index d6278ff070..ef712e2f86 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -159,7 +159,7 @@ export const spec = { event: { $ref: '#/components/schemas/Event', }, - subscriptionIds: { + consumedBy: { type: 'array', description: 'The IDs of subscriptions that have already received this event', diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index 48f2c74e5e..8b2639a189 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -106,7 +106,7 @@ paths: properties: event: $ref: '#/components/schemas/Event' - subscriptionIds: + consumedBy: type: array description: The IDs of subscriptions that have already received this event items: diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 0a8dddfe51..87c9263e79 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -323,9 +323,10 @@ export class DatabaseEventBusStore implements EventBusStore { async publish(options: { params: EventParams; - subscriberIds: string[]; + consumedBy?: string[]; }): Promise<{ id: string } | undefined> { const topic = options.params.topic; + const consumedBy = options.consumedBy ?? []; // This query inserts a new event into the database, but only if there are // subscribers to the topic that have not already been notified const result = await this.#db @@ -351,12 +352,12 @@ export class DatabaseEventBusStore implements EventBusStore { metadata: options.params.metadata, }), ]), - this.#db.raw('?', [options.subscriberIds]), + this.#db.raw('?', [consumedBy]), ) // The rest of this query is to check whether there are any // subscribers that have not been notified yet .from(TABLE_SUBSCRIPTIONS) - .whereNotIn('id', options.subscriberIds) // Skip notified subscribers + .whereNotIn('id', consumedBy) // Skip notified subscribers .andWhere(this.#db.raw('? = ANY(topics)', [topic])) // Match topic .having(this.#db.raw('count(*)'), '>', 0), // Check if there are any results ) diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 0fc9a2b14f..23aec6ba97 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -19,9 +19,7 @@ import { EventBusStore } from './types'; const MAX_BATCH_SIZE = 5; export class MemoryEventBusStore implements EventBusStore { - #events = new Array< - EventParams & { seq: number; subscriberIds: Set } - >(); + #events = new Array }>(); #subscribers = new Map< string, { id: string; seq: number; topics: Set } @@ -33,14 +31,14 @@ export class MemoryEventBusStore implements EventBusStore { async publish(options: { params: EventParams; - subscriberIds: string[]; + consumedBy: string[]; }): Promise<{ id: string } | undefined> { const topic = options.params.topic; - const subscriberIds = new Set(options.subscriberIds); + const consumedBy = new Set(options.consumedBy); let hasOtherSubscribers = false; for (const sub of this.#subscribers.values()) { - if (sub.topics.has(topic) && !subscriberIds.has(sub.id)) { + if (sub.topics.has(topic) && !consumedBy.has(sub.id)) { hasOtherSubscribers = true; break; } @@ -50,7 +48,7 @@ export class MemoryEventBusStore implements EventBusStore { } const nextSeq = this.#getMaxSeq() + 1; - this.#events.push({ ...options.params, subscriberIds, seq: nextSeq }); + this.#events.push({ ...options.params, consumedBy, seq: nextSeq }); for (const listener of this.#listeners) { if (listener.topics.has(topic)) { @@ -89,7 +87,7 @@ export class MemoryEventBusStore implements EventBusStore { event => event.seq > sub.seq && sub.topics.has(event.topic) && - !event.subscriberIds.has(id), + !event.consumedBy.has(id), ) .slice(0, MAX_BATCH_SIZE); diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 1a702c624b..566574f6e8 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -160,13 +160,13 @@ export async function createEventBusRouter(options: { allow: ['service'], }); const topic = req.body.event.topic; - const subscriberIds = req.body.subscriptionIds ?? []; + const consumedBy = req.body.consumedBy; const result = await store.publish({ params: { topic, eventPayload: req.body.event.payload, } as EventParams, - subscriberIds, + consumedBy: req.body.consumedBy, }); if (result) { logger.info(`Published event to '${topic}' with ID '${result.id}'`, { @@ -174,14 +174,18 @@ export async function createEventBusRouter(options: { }); res.status(201).end(); } else { - logger.info( - `Skipped publishing of event to '${topic}', subscribers have already been notified: '${subscriberIds.join( - "', '", - )}'`, - { - subject: credentials.principal.subject, - }, - ); + if (consumedBy) { + const notified = `'${consumedBy.join("', '")}'`; + logger.info( + `Skipped publishing of event to '${topic}', subscribers have already been notified: ${notified}`, + { subject: credentials.principal.subject }, + ); + } else { + logger.info( + `Skipped publishing of event to '${topic}', no subscribers present`, + { subject: credentials.principal.subject }, + ); + } res.status(204).end(); } }); @@ -267,7 +271,7 @@ export async function createEventBusRouter(options: { await store.upsertSubscription(id, req.body.topics); logger.info( - `New subscription '${id}' topics='${req.body.topics.join("', '")}'`, + `New subscription '${id}' for topics '${req.body.topics.join("', '")}'`, { subject: credentials.principal.subject }, ); diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 87e62049a9..315a4a7220 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -19,7 +19,7 @@ import { EventParams } from '@backstage/plugin-events-node'; export type EventBusStore = { publish(options: { params: EventParams; - subscriberIds: string[]; + consumedBy?: string[]; }): Promise<{ id: string } | undefined>; upsertSubscription(id: string, topics: string[]): Promise; diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 74e37100e0..5a2712fc89 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -127,7 +127,7 @@ class PluginEventsService implements EventsService { { body: { event: { payload: params.eventPayload, topic: params.topic }, - subscriptionIds: notifiedSubscribers, + consumedBy: notifiedSubscribers, }, }, { token }, diff --git a/plugins/events-node/src/generated/models/PostEventRequest.model.ts b/plugins/events-node/src/generated/models/PostEventRequest.model.ts index 527064bc13..04eaaab461 100644 --- a/plugins/events-node/src/generated/models/PostEventRequest.model.ts +++ b/plugins/events-node/src/generated/models/PostEventRequest.model.ts @@ -24,5 +24,5 @@ export interface PostEventRequest { /** * The IDs of subscriptions that have already received this event */ - subscriptionIds?: Array; + consumedBy?: Array; } From 6bc5b882d24a699b795863700d99e809ee36fa9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 09:56:40 +0200 Subject: [PATCH 50/83] events-backend: refactor event bus tests to use helper Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 150 ++++++++---------- 1 file changed, 67 insertions(+), 83 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index e73e636d24..f6e0799695 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -21,6 +21,7 @@ import { createServiceFactory, } from '@backstage/backend-plugin-api'; import { + TestBackend, TestDatabaseId, TestDatabases, mockCredentials, @@ -102,6 +103,38 @@ describe('eventsPlugin', () => { }); describe('event bus', () => { + class ReqHelper { + constructor(private readonly backend: TestBackend) {} + + subscribe(id: string, topics: string[], options?: { auth?: string }) { + return request(this.backend.server) + .put(`/api/events/bus/v1/subscriptions/${id}`) + .set( + 'authorization', + options?.auth ?? mockCredentials.service.header(), + ) + .send({ topics }); + } + + publish( + topic: string, + payload: unknown, + options?: { consumedBy?: string[] }, + ) { + return request(this.backend.server) + .post('/api/events/bus/v1/events') + .set('authorization', mockCredentials.service.header()) + .send({ event: { topic, payload }, consumedBy: options?.consumedBy }); + } + + readEvents(id: string) { + return request(this.backend.server) + .get(`/api/events/bus/v1/subscriptions/${id}/events`) + .set('authorization', mockCredentials.service.header()) + .send(); + } + } + const databases = TestDatabases.create({ ids: ['SQLITE_3', 'POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], }); @@ -119,24 +152,21 @@ describe('eventsPlugin', () => { const backend = await startTestBackend({ features: [eventsPlugin(), await mockKnexFactory(databaseId)], }); - const { server } = backend; + const helper = new ReqHelper(backend); - await request(server) - .post('/api/events/bus/v1/events') + await helper + .publish('test', { n: 1 }) .set('authorization', mockCredentials.none.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) .expect(401); - await request(server) - .post('/api/events/bus/v1/events') + await helper + .publish('test', { n: 1 }) .set('authorization', mockCredentials.user.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) .expect(403); - await request(server) - .post('/api/events/bus/v1/events') + await helper + .publish('test', { n: 1 }) .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) .expect(204); // 204, since there are no subscribers await backend.stop(); @@ -149,51 +179,35 @@ describe('eventsPlugin', () => { const backend = await startTestBackend({ features: [eventsPlugin(), await mockKnexFactory(databaseId)], }); - const { server } = backend; + const helper = new ReqHelper(backend); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') + await helper + .subscribe('tester', ['test']) .set('authorization', mockCredentials.none.header()) - .send({ topics: ['test'] }) .expect(401); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') + await helper + .subscribe('tester', ['test']) .set('authorization', mockCredentials.user.header()) - .send({ topics: ['test'] }) .expect(403); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); + await helper.subscribe('tester', ['test']).expect(201); - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') + await helper + .readEvents('tester') .set('authorization', mockCredentials.none.header()) - .send({ topics: ['test'] }) .expect(401); - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') + await helper + .readEvents('tester') .set('authorization', mockCredentials.user.header()) - .send({ topics: ['test'] }) .expect(403); - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(201); // 201, since there is a subscriber + await helper.publish('test', { n: 1 }).expect(201); // 201, since there is a subscriber - await request(server) - .get('/api/events/bus/v1/subscriptions/tester/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { - events: [{ topic: 'test', payload: { n: 1 } }], - }); + await helper.readEvents('tester').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); await backend.stop(); }, @@ -205,45 +219,23 @@ describe('eventsPlugin', () => { const backend = await startTestBackend({ features: [eventsPlugin(), await mockKnexFactory(databaseId)], }); - const { server } = backend; + const helper = new ReqHelper(backend); // 2 subscribers - await request(server) - .put('/api/events/bus/v1/subscriptions/tester-1') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); - await request(server) - .put('/api/events/bus/v1/subscriptions/tester-2') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(201); + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); // A single event - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 1 } } }) - .expect(201); + await helper.publish('test', { n: 1 }).expect(201); // Single client for subscriber 1 gets the event - await request(server) - .get('/api/events/bus/v1/subscriptions/tester-1/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { - events: [{ topic: 'test', payload: { n: 1 } }], - }); + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { n: 1 } }], + }); // Two clients for subscriber 2, only one gets the event - const res1 = request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }); - const res2 = request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }); + const res1 = helper.readEvents('tester-2'); + const res2 = helper.readEvents('tester-2'); const res = await Promise.race([res1, res2]); expect(res.status).toBe(200); @@ -252,11 +244,7 @@ describe('eventsPlugin', () => { }); // Post another event, which triggers the other client to return - await request(server) - .post('/api/events/bus/v1/events') - .set('authorization', mockCredentials.service.header()) - .send({ event: { topic: 'test', payload: { n: 2 } } }) - .expect(201); + await helper.publish('test', { n: 2 }).expect(201); const otherRes = await Promise.all([res1, res2]).then(rs => rs.find(r => r !== res), @@ -264,13 +252,9 @@ describe('eventsPlugin', () => { expect(otherRes?.status).toBe(202); // Reading subscriber 2 should now return the second event only - await request(server) - .get('/api/events/bus/v1/subscriptions/tester-2/events') - .set('authorization', mockCredentials.service.header()) - .send({ topics: ['test'] }) - .expect(200, { - events: [{ topic: 'test', payload: { n: 2 } }], - }); + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { n: 2 } }], + }); await backend.stop(); }, From 24b86599c72610e720243f44e303a293bfed6a2b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 11:26:56 +0200 Subject: [PATCH 51/83] events-backend: always use batch size 10 for events bus Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/hub/MemoryEventBusStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 23aec6ba97..1e95efc3de 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -16,7 +16,7 @@ import { EventParams } from '@backstage/plugin-events-node'; import { EventBusStore } from './types'; -const MAX_BATCH_SIZE = 5; +const MAX_BATCH_SIZE = 10; export class MemoryEventBusStore implements EventBusStore { #events = new Array }>(); From 9631d2a9e4974b333c7bf307415aa5b99e571192 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 11:27:20 +0200 Subject: [PATCH 52/83] events-backend: more tests for events bus Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index f6e0799695..dbf7249e0e 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -259,5 +259,114 @@ describe('eventsPlugin', () => { await backend.stop(); }, ); + + it.each(databases.eachSupportedId())( + 'should should not notify subscribers that have already consumed the event, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester-1', ['test']).expect(201); + await helper.subscribe('tester-2', ['test']).expect(201); + + // A single event for each subscriber, that should not be sent to the other one + await helper + .publish( + 'test', + { for: 'tester-2' }, + { + consumedBy: ['tester-1'], + }, + ) + .expect(201); + await helper + .publish( + 'test', + { for: 'tester-1' }, + { + consumedBy: ['tester-2'], + }, + ) + .expect(201); + + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-1').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-1' } }], + }); + // Single client for subscriber 1 gets the event + await helper.readEvents('tester-2').expect(200, { + events: [{ topic: 'test', payload: { for: 'tester-2' } }], + }); + + await backend.stop(); + }, + ); + + it.each(databases.eachSupportedId())( + 'should return multiple events in order, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + // 2 subscribers + await helper.subscribe('tester', ['test']).expect(201); + + // A single event for each subscriber, that should not be sent to the other one + for (let n = 0; n < 15; ++n) { + await helper.publish('test', { n }).expect(201); + } + + // Batch size it 10 + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 0 } }, + { topic: 'test', payload: { n: 1 } }, + { topic: 'test', payload: { n: 2 } }, + { topic: 'test', payload: { n: 3 } }, + { topic: 'test', payload: { n: 4 } }, + { topic: 'test', payload: { n: 5 } }, + { topic: 'test', payload: { n: 6 } }, + { topic: 'test', payload: { n: 7 } }, + { topic: 'test', payload: { n: 8 } }, + { topic: 'test', payload: { n: 9 } }, + ], + }); + + await helper.readEvents('tester').expect(200, { + events: [ + { topic: 'test', payload: { n: 10 } }, + { topic: 'test', payload: { n: 11 } }, + { topic: 'test', payload: { n: 12 } }, + { topic: 'test', payload: { n: 13 } }, + { topic: 'test', payload: { n: 14 } }, + ], + }); + + await backend.stop(); + }, + ); + + it.each(databases.eachSupportedId())( + 'should skip publishing if all subscribers have already consumed the event, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper.subscribe('tester', ['test']).expect(201); + + await helper + .publish('test', { for: 'tester-2' }, { consumedBy: ['tester'] }) + .expect(204); + + await backend.stop(); + }, + ); }); }); From 66487229ccd28da113b9d98613e96b8fa22d9fc9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 11:51:41 +0200 Subject: [PATCH 53/83] events-backend: fix for memory store not timing out listeners Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/hub/MemoryEventBusStore.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 1e95efc3de..629c341a76 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -111,12 +111,13 @@ export class MemoryEventBusStore implements EventBusStore { throw new Error(`Subscription not found`); } - return new Promise<{ topic: string }>(resolve => { + return new Promise<{ topic: string }>((resolve, reject) => { const listener = { topics: sub.topics, resolve }; this.#listeners.add(listener); options.signal.addEventListener('abort', () => { this.#listeners.delete(listener); + reject(options.signal.reason); }); }); }, From a97b71ab07a7cca2276f6766d2f608dd050314ff Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 11:51:59 +0200 Subject: [PATCH 54/83] events-backend: fix blocking request being ended too early Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/createEventBusRouter.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 566574f6e8..60dc5fac3f 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -241,9 +241,7 @@ export async function createEventBusRouter(options: { { subject: credentials.principal.subject }, ); } catch (error) { - if (error === controller.signal.reason) { - res.end(); - } else { + if (error !== controller.signal.reason) { throw error; } } finally { From 692c505a38bf1b4ae4b46712741213db6596cee1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 11:52:19 +0200 Subject: [PATCH 55/83] events-backend: added event hub test for blocking events response Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index dbf7249e0e..3c08637b1f 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -368,5 +368,53 @@ describe('eventsPlugin', () => { await backend.stop(); }, ); + + it.each(databases.eachSupportedId())( + 'should time out when no events are available, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + await helper.subscribe('tester', ['test']).expect(201); + + jest.useFakeTimers({ + doNotFake: ['nextTick'], + }); + + try { + // Can't use supertest for this one because it can't handle the partially blocking response + const res = await fetch( + `http://localhost:${backend.server.port()}/api/events/bus/v1/subscriptions/tester/events`, + { + headers: { + authorization: mockCredentials.service.header(), + }, + }, + ); + + expect(res.status).toBe(202); + + const { closed } = res.body!.getReader(); + const checkClosed = () => + Promise.race([ + closed.then(() => true), + new Promise(r => process.nextTick(() => r(false))), + ]); + + await expect(checkClosed()).resolves.toBe(false); + + await jest.advanceTimersByTimeAsync(30000); + await expect(checkClosed()).resolves.toBe(false); + + await jest.advanceTimersByTimeAsync(30000); + await expect(checkClosed()).resolves.toBe(true); + } finally { + jest.useRealTimers(); + } + + await backend.stop(); + }, + ); }); }); From a19caf26973f501549f2d7cdb51124c64b2428ef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 13:34:02 +0200 Subject: [PATCH 56/83] events-backend: fix error when reading subscription with empty events table Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/EventsPlugin.test.ts | 2 +- plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 3c08637b1f..215d4e9190 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -379,7 +379,7 @@ describe('eventsPlugin', () => { await helper.subscribe('tester', ['test']).expect(201); jest.useFakeTimers({ - doNotFake: ['nextTick'], + advanceTimers: true, }); try { diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 87c9263e79..6694703a2e 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -447,7 +447,7 @@ export class DatabaseEventBusStore implements EventBusStore { // events where read, the last ID out of all events .update({ read_until: this.#db.raw( - 'COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events))', + 'COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events), 0)', ), }) .updateFrom({ From 362571f6771e5d2cfee7c34e5fcfc399f43e26ea Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 13:34:59 +0200 Subject: [PATCH 57/83] events-backend: log blocking event response errors instead of attempting to return them Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/hub/createEventBusRouter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 60dc5fac3f..2e1dc043db 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -242,7 +242,7 @@ export async function createEventBusRouter(options: { ); } catch (error) { if (error !== controller.signal.reason) { - throw error; + logger.error(`Error listening for subscription '${id}'`, error); } } finally { // A small extra delay ensures a more even spread of events across From 1e8e3f9283243e600d6b0c875c0b7986d4af96d1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 13:38:47 +0200 Subject: [PATCH 58/83] events-backend: test for reading events without subscription + fix error Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 14 ++++++++++++++ .../src/service/hub/MemoryEventBusStore.ts | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 215d4e9190..53f84ffed1 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -416,5 +416,19 @@ describe('eventsPlugin', () => { await backend.stop(); }, ); + + it.each(databases.eachSupportedId())( + 'should refuse listen without a subscription, %p', + async databaseId => { + const backend = await startTestBackend({ + features: [eventsPlugin(), await mockKnexFactory(databaseId)], + }); + const helper = new ReqHelper(backend); + + await helper.readEvents('nonexistent').expect(404); + + await backend.stop(); + }, + ); }); }); diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 629c341a76..3071a51edf 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -15,6 +15,7 @@ */ import { EventParams } from '@backstage/plugin-events-node'; import { EventBusStore } from './types'; +import { NotFoundError } from '@backstage/errors'; const MAX_BATCH_SIZE = 10; @@ -80,7 +81,7 @@ export class MemoryEventBusStore implements EventBusStore { async readSubscription(id: string): Promise<{ events: EventParams[] }> { const sub = this.#subscribers.get(id); if (!sub) { - throw new Error(`Subscription not found`); + throw new NotFoundError(`Subscription not found`); } const events = this.#events .filter( @@ -108,7 +109,7 @@ export class MemoryEventBusStore implements EventBusStore { const sub = this.#subscribers.get(subscriptionId); if (!sub) { - throw new Error(`Subscription not found`); + throw new NotFoundError(`Subscription not found`); } return new Promise<{ topic: string }>((resolve, reject) => { From 2c9ea899c7c07857e98ba42deb40607b7f8388e9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 15:07:34 +0200 Subject: [PATCH 59/83] events-node: restore behavior of top-level DefaltEventsService methods Signed-off-by: Patrik Oldsberg --- plugins/events-node/api-report.md | 8 ++++---- plugins/events-node/src/api/DefaultEventsService.ts | 12 +++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/plugins/events-node/api-report.md b/plugins/events-node/api-report.md index a735007e3b..9ebaf1a536 100644 --- a/plugins/events-node/api-report.md +++ b/plugins/events-node/api-report.md @@ -23,10 +23,10 @@ export class DefaultEventsService implements EventsService { lifecycle: LifecycleService; }, ): EventsService; - // @deprecated (undocumented) - publish(_params: EventParams): Promise; - // @deprecated (undocumented) - subscribe(_options: EventsServiceSubscribeOptions): Promise; + // (undocumented) + publish(params: EventParams): Promise; + // (undocumented) + subscribe(options: EventsServiceSubscribeOptions): Promise; } // @public @deprecated diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 5a2712fc89..9e2509539a 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -23,7 +23,7 @@ import { import { EventParams } from './EventParams'; import { EventsService, EventsServiceSubscribeOptions } from './EventsService'; import { DefaultApiClient } from '../generated'; -import { NotImplementedError, ResponseError } from '@backstage/errors'; +import { ResponseError } from '@backstage/errors'; const POLL_BACKOFF_START_MS = 1_000; const POLL_BACKOFF_MAX_MS = 60_000; @@ -386,13 +386,11 @@ export class DefaultEventsService implements EventsService { return service; } - /** @deprecated this method should not be called */ - async publish(_params: EventParams): Promise { - throw new NotImplementedError(); + async publish(params: EventParams): Promise { + await this.localBus.publish(params); } - /** @deprecated this method should not be called */ - async subscribe(_options: EventsServiceSubscribeOptions): Promise { - throw new NotImplementedError(); + async subscribe(options: EventsServiceSubscribeOptions): Promise { + this.localBus.subscribe(options); } } From 0de66764e4072055524d6e99ada8c1c9121a2b5c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 27 May 2024 18:23:41 +0200 Subject: [PATCH 60/83] events-backend: more reliable timeout test Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/EventsPlugin.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 53f84ffed1..18b994bff9 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -396,19 +396,21 @@ describe('eventsPlugin', () => { expect(res.status).toBe(202); const { closed } = res.body!.getReader(); - const checkClosed = () => + const checkNotClosed = () => Promise.race([ - closed.then(() => true), + closed.then(() => { + throw new Error('Closed'); + }), new Promise(r => process.nextTick(() => r(false))), ]); - await expect(checkClosed()).resolves.toBe(false); + await expect(checkNotClosed()).resolves.toBe(false); await jest.advanceTimersByTimeAsync(30000); - await expect(checkClosed()).resolves.toBe(false); + await expect(checkNotClosed()).resolves.toBe(false); await jest.advanceTimersByTimeAsync(30000); - await expect(checkClosed()).resolves.toBe(true); + await closed; } finally { jest.useRealTimers(); } From b9daa73fedebea4c1b8d2db41aea2f8c853e29b4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Sep 2024 20:46:30 +0200 Subject: [PATCH 61/83] Apply suggestions from code review Co-authored-by: Patrick Jungermann Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/service/EventsPlugin.test.ts | 8 ++++---- .../src/service/hub/DatabaseEventBusStore.ts | 2 +- .../src/service/hub/createEventBusRouter.ts | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 18b994bff9..0cfb68eb05 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -233,7 +233,7 @@ describe('eventsPlugin', () => { events: [{ topic: 'test', payload: { n: 1 } }], }); - // Two clients for subscriber 2, only one gets the event + // Two clients for subscriber 2, only one gets the event const res1 = helper.readEvents('tester-2'); const res2 = helper.readEvents('tester-2'); @@ -261,7 +261,7 @@ describe('eventsPlugin', () => { ); it.each(databases.eachSupportedId())( - 'should should not notify subscribers that have already consumed the event, %p', + 'should not notify subscribers that have already consumed the event, %p', async databaseId => { const backend = await startTestBackend({ features: [eventsPlugin(), await mockKnexFactory(databaseId)], @@ -296,7 +296,7 @@ describe('eventsPlugin', () => { await helper.readEvents('tester-1').expect(200, { events: [{ topic: 'test', payload: { for: 'tester-1' } }], }); - // Single client for subscriber 1 gets the event + // Single client for subscriber 2 gets the event await helper.readEvents('tester-2').expect(200, { events: [{ topic: 'test', payload: { for: 'tester-2' } }], }); @@ -316,7 +316,7 @@ describe('eventsPlugin', () => { // 2 subscribers await helper.subscribe('tester', ['test']).expect(201); - // A single event for each subscriber, that should not be sent to the other one + // A sequence of events published one at a time for (let n = 0; n < 15; ++n) { await helper.publish('test', { n }).expect(201); } diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 6694703a2e..c9e97e22b1 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -26,7 +26,7 @@ import { import { ForwardedError, NotFoundError } from '@backstage/errors'; import { HumanDuration, durationToMilliseconds } from '@backstage/types'; -const WINDOW_SIZE_DEFAULT = 10000; +const WINDOW_SIZE_DEFAULT = 10_000; const WINDOW_MIN_AGE_DEFAULT = { minutes: 10 }; const WINDOW_MAX_AGE_DEFAULT = { days: 1 }; diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 2e1dc043db..a3ba7779b1 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -203,14 +203,14 @@ export async function createEventBusRouter(options: { // By setting up the listener first we make sure we don't miss any events // that are published while reading. If an event is published we'll receive - // a notification, which depending on the outcome of the read we may ignore + // a notification, which we may ignore depending on the outcome of the read const listener = await store.setupListener(id, { signal: controller.signal, }); // By timing out requests we make sure they don't stall or that events get stuck. // For the caller there's no difference between a timeout and a - // notifications, either way they should try reading again. + // notification, either way they should try reading again. const timeout = setTimeout(() => { controller.abort(); }, notifyTimeoutMs); From f06211e44757939d5fc6076e768bd8b332aa8c98 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Sep 2024 22:18:57 +0200 Subject: [PATCH 62/83] events-backend: couple of simple review fixes Signed-off-by: Patrik Oldsberg --- .../migrations/20240523100528_init.js | 3 +- .../src/service/EventsPlugin.test.ts | 16 ++++---- .../src/service/hub/DatabaseEventBusStore.ts | 23 ++++++----- .../src/service/hub/createEventBusRouter.ts | 40 +++++++++---------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js index 90cdb770a5..7ec85d7917 100644 --- a/plugins/events-backend/migrations/20240523100528_init.js +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -82,6 +82,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { if (knex.client.config.client === 'pg') { - await knex.schema.dropTable('events'); + await knex.schema.dropTable('event_bus_subscriptions'); + await knex.schema.dropTable('event_bus_events'); } }; diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 0cfb68eb05..ba9a236800 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -150,7 +150,7 @@ describe('eventsPlugin', () => { 'should be possible to publish events as a service, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -177,7 +177,7 @@ describe('eventsPlugin', () => { 'should be possible to subscribe as a service and receive an event, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -217,7 +217,7 @@ describe('eventsPlugin', () => { 'should only send an event for each subscriber once, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -264,7 +264,7 @@ describe('eventsPlugin', () => { 'should not notify subscribers that have already consumed the event, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -309,7 +309,7 @@ describe('eventsPlugin', () => { 'should return multiple events in order, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -355,7 +355,7 @@ describe('eventsPlugin', () => { 'should skip publishing if all subscribers have already consumed the event, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -373,7 +373,7 @@ describe('eventsPlugin', () => { 'should time out when no events are available, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); await helper.subscribe('tester', ['test']).expect(201); @@ -423,7 +423,7 @@ describe('eventsPlugin', () => { 'should refuse listen without a subscription, %p', async databaseId => { const backend = await startTestBackend({ - features: [eventsPlugin(), await mockKnexFactory(databaseId)], + features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index c9e97e22b1..e8f0da444d 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -26,7 +26,7 @@ import { import { ForwardedError, NotFoundError } from '@backstage/errors'; import { HumanDuration, durationToMilliseconds } from '@backstage/types'; -const WINDOW_SIZE_DEFAULT = 10_000; +const WINDOW_MAX_COUNT_DEFAULT = 10_000; const WINDOW_MIN_AGE_DEFAULT = { minutes: 10 }; const WINDOW_MAX_AGE_DEFAULT = { days: 1 }; @@ -254,7 +254,7 @@ export class DatabaseEventBusStore implements EventBusStore { /** Events outside of this age will always be deleted */ maxAge?: HumanDuration; /** Events outside of this count will be deleted if they are outside the minAge window */ - size?: number; + maxCount?: number; }; }): Promise { const db = await options.database.getClient(); @@ -278,12 +278,12 @@ export class DatabaseEventBusStore implements EventBusStore { db, options.logger, listener, - options.window?.size ?? WINDOW_SIZE_DEFAULT, + options.window?.maxCount ?? WINDOW_MAX_COUNT_DEFAULT, durationToMilliseconds(options.window?.minAge ?? WINDOW_MIN_AGE_DEFAULT), durationToMilliseconds(options.window?.maxAge ?? WINDOW_MAX_AGE_DEFAULT), ); - options.scheduler.scheduleTask({ + await options.scheduler.scheduleTask({ id: 'event-bus-cleanup', frequency: { seconds: 10 }, timeout: { minutes: 1 }, @@ -301,7 +301,7 @@ export class DatabaseEventBusStore implements EventBusStore { readonly #db: Knex; readonly #logger: LoggerService; readonly #listener: DatabaseEventBusListener; - readonly #windowSize: number; + readonly #windowMaxCount: number; readonly #windowMinAge: number; readonly #windowMaxAge: number; @@ -309,14 +309,14 @@ export class DatabaseEventBusStore implements EventBusStore { db: Knex, logger: LoggerService, listener: DatabaseEventBusListener, - windowSize: number, + windowMaxCount: number, windowMinAge: number, windowMaxAge: number, ) { this.#db = db; this.#logger = logger; this.#listener = listener; - this.#windowSize = windowSize; + this.#windowMaxCount = windowMaxCount; this.#windowMinAge = windowMinAge; this.#windowMaxAge = windowMaxAge; } @@ -393,9 +393,10 @@ export class DatabaseEventBusStore implements EventBusStore { id, updated_at: this.#db.fn.now(), topics, - read_until: this.#db.raw( - `( SELECT COALESCE(MAX("id"), 0) FROM "${TABLE_EVENTS}" )`, - ), + // TODO(Rugvip): Might be that there's a more performant way to do this + read_until: this.#db.raw(`( SELECT COALESCE(MAX("id"), 0) FROM ?? )`, [ + TABLE_EVENTS, + ]), }) .onConflict('id') .merge(['topics', 'updated_at']) @@ -523,7 +524,7 @@ export class DatabaseEventBusStore implements EventBusStore { .select('id') .from(TABLE_EVENTS) .orderBy('id', 'desc') - .offset(this.#windowSize), + .offset(this.#windowMaxCount), ) .wrap('ANY(ARRAY(', '))'), ) diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index a3ba7779b1..532ceaf2b8 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -114,6 +114,23 @@ cause unnecessary storage of events. */ +async function createEventBusStore(deps: { + logger: LoggerService; + database: DatabaseService; + scheduler: SchedulerService; + lifecycle: LifecycleService; + httpAuth: HttpAuthService; +}): Promise { + const db = await deps.database.getClient(); + if (db.client.config.client === 'pg') { + deps.logger.info('Database is PostgreSQL, using database store'); + return await DatabaseEventBusStore.create(deps); + } + + deps.logger.info('Database is not PostgreSQL, using memory store'); + return new MemoryEventBusStore(); +} + /** * Creates a new event bus router * @internal @@ -126,30 +143,11 @@ export async function createEventBusRouter(options: { httpAuth: HttpAuthService; notifyTimeoutMs?: number; // for testing }): Promise { - const { - database, - httpAuth, - scheduler, - lifecycle, - notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS, - } = options; + const { httpAuth, notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS } = options; const logger = options.logger.child({ type: 'EventBus' }); const router = Router(); - let store: EventBusStore; - const db = await database.getClient(); - if (db.client.config.client === 'pg') { - logger.info('Database is PostgreSQL, using database store'); - store = await DatabaseEventBusStore.create({ - database, - logger, - scheduler, - lifecycle, - }); - } else { - logger.info('Database is not PostgreSQL, using memory store'); - store = new MemoryEventBusStore(); - } + const store = await createEventBusStore(options); const apiRouter = await createOpenApiRouter(); From c412d48980ed3ed6780fe3c056611f6ef6825647 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Sep 2024 23:17:17 +0200 Subject: [PATCH 63/83] events-backend: add cleanup test for DatabaseEventBusStore Signed-off-by: Patrik Oldsberg --- .../service/hub/DatabaseEventBusStore.test.ts | 62 +++++++++++++++++++ .../src/service/hub/DatabaseEventBusStore.ts | 16 +++++ 2 files changed, 78 insertions(+) create mode 100644 plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts new file mode 100644 index 0000000000..ed9fa3a7af --- /dev/null +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TestDatabases, mockServices } from '@backstage/backend-test-utils'; +import { DatabaseEventBusStore } from './DatabaseEventBusStore'; + +const logger = mockServices.logger.mock(); + +const databases = TestDatabases.create({ + ids: ['POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], +}); + +describe('DatabaseEventBusStore', () => { + it.each(databases.eachSupportedId())( + 'should clean up old events, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ logger, db }); + + await store.upsertSubscription('tester-1', ['test']); + await store.upsertSubscription('tester-2', ['test']); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + params: { topic: 'test', eventPayload: { n: i } }, + }); + } + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + + await store.clean(); + + await expect(store.readSubscription('tester-2')).rejects.toThrow( + "Subscription with ID 'tester-2' not found", + ); + + await store.upsertSubscription('tester-3', ['test']); + + // Reset read pointer to read form the beginning + await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ + read_until: 0, + }); + + const { events: events3 } = await store.readSubscription('tester-3'); + expect(events3.length).toBe(5); + }, + ); +}); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index e8f0da444d..744e9d3569 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -298,6 +298,22 @@ export class DatabaseEventBusStore implements EventBusStore { return store; } + /** @internal */ + static async forTest({ db, logger }: { db: Knex; logger: LoggerService }) { + await db.migrate.latest({ directory: migrationsDir }); + + const store = new DatabaseEventBusStore( + db, + logger, + new DatabaseEventBusListener(db.client, logger), + 5, + 0, + 10, + ); + + return Object.assign(store, { clean: () => store.#cleanup() }); + } + readonly #db: Knex; readonly #logger: LoggerService; readonly #listener: DatabaseEventBusListener; From e4d661091345859ddc8682cfd80065cc31a322a8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 12 Sep 2024 23:49:03 +0200 Subject: [PATCH 64/83] events-backend: more cleanup tests and refactor DB access to avoid some raw Signed-off-by: Patrik Oldsberg --- .../service/hub/DatabaseEventBusStore.test.ts | 65 +++++++++++++++++++ .../src/service/hub/DatabaseEventBusStore.ts | 62 ++++++++++-------- 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index ed9fa3a7af..830ee1ae46 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -59,4 +59,69 @@ describe('DatabaseEventBusStore', () => { expect(events3.length).toBe(5); }, ); + + it.each(databases.eachSupportedId())( + 'should always clean up events outside the max age window, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + maxAge: 0, + }); + + await store.upsertSubscription('tester-1', ['test']); + await store.upsertSubscription('tester-2', ['test']); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + params: { topic: 'test', eventPayload: { n: i } }, + }); + } + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + + await store.clean(); + + await expect(store.readSubscription('tester-2')).rejects.toThrow( + "Subscription with ID 'tester-2' not found", + ); + + await store.upsertSubscription('tester-3', ['test']); + + // Reset read pointer to read form the beginning + await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ + read_until: 0, + }); + + const { events: events3 } = await store.readSubscription('tester-3'); + expect(events3.length).toBe(0); + }, + ); + + it.each(databases.eachSupportedId())( + 'should not clean up events within the min age window, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + minAge: 1000, + }); + + await store.upsertSubscription('tester-1', ['test']); + + for (let i = 0; i < 10; ++i) { + await store.publish({ + params: { topic: 'test', eventPayload: { n: i } }, + }); + } + + await store.clean(); + + const { events: events1 } = await store.readSubscription('tester-1'); + expect(events1.length).toBe(10); + }, + ); }); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 744e9d3569..63cc06d40d 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -299,7 +299,17 @@ export class DatabaseEventBusStore implements EventBusStore { } /** @internal */ - static async forTest({ db, logger }: { db: Knex; logger: LoggerService }) { + static async forTest({ + db, + logger, + minAge = 0, + maxAge = 10_000, + }: { + db: Knex; + logger: LoggerService; + minAge?: number; + maxAge?: number; + }) { await db.migrate.latest({ directory: migrationsDir }); const store = new DatabaseEventBusStore( @@ -307,8 +317,8 @@ export class DatabaseEventBusStore implements EventBusStore { logger, new DatabaseEventBusListener(db.client, logger), 5, - 0, - 10, + minAge, + maxAge, ); return Object.assign(store, { clean: () => store.#cleanup() }); @@ -404,15 +414,13 @@ export class DatabaseEventBusStore implements EventBusStore { } async upsertSubscription(id: string, topics: string[]): Promise { + const [{ max: maxId }] = await this.#db(TABLE_EVENTS).max('id'); const result = await this.#db(TABLE_SUBSCRIPTIONS) .insert({ id, updated_at: this.#db.fn.now(), topics, - // TODO(Rugvip): Might be that there's a more performant way to do this - read_until: this.#db.raw(`( SELECT COALESCE(MAX("id"), 0) FROM ?? )`, [ - TABLE_EVENTS, - ]), + read_until: maxId || 0, }) .onConflict('id') .merge(['topics', 'updated_at']) @@ -526,26 +534,21 @@ export class DatabaseEventBusStore implements EventBusStore { #cleanup = async () => { try { - const eventCount = await this.#db + const eventCount = await this.#db(TABLE_EVENTS) .delete() - .from(TABLE_EVENTS) // Delete any events that are outside both the min age and size window - .where('created_at', '<', new Date(Date.now() - this.#windowMinAge)) - .andWhere( + .whereIn( 'id', - '=', this.#db - .raw( - this.#db - .select('id') - .from(TABLE_EVENTS) - .orderBy('id', 'desc') - .offset(this.#windowMaxCount), - ) - .wrap('ANY(ARRAY(', '))'), + .select('id') + .from(TABLE_EVENTS) + .orderBy('id', 'desc') + .offset(this.#windowMaxCount), ) + .andWhere('created_at', '<', new Date(Date.now() - this.#windowMinAge)) // If events are outside the max age they will always be deleted .orWhere('created_at', '<', new Date(Date.now() - this.#windowMaxAge)); + this.#logger.info( `Event cleanup resulted in ${eventCount} old events being deleted`, ); @@ -555,12 +558,19 @@ export class DatabaseEventBusStore implements EventBusStore { try { // Delete any subscribers that aren't keeping up with current events - const subscriberCount = await this.#db - .delete() - .from(TABLE_SUBSCRIPTIONS) - .where('read_until', '<', (q: Knex.QueryBuilder) => - q.select(this.#db.raw('MIN(id)')).from(TABLE_EVENTS), - ); + const [{ min: minId }] = await this.#db(TABLE_EVENTS).min('id'); + + let subscriberCount; + if (minId === null) { + // No events left, remove all subscribers. This can happen if no events + // are published within the max age window. + subscriberCount = await this.#db(TABLE_SUBSCRIPTIONS).delete(); + } else { + subscriberCount = await this.#db(TABLE_SUBSCRIPTIONS) + .delete() + // Read pointer points to the ID that has been read, so we need an additional offset + .where('read_until', '<', minId - 1); + } this.#logger.info( `Subscription cleanup resulted in ${subscriberCount} stale subscribers being deleted`, From d98d202f813a84cec0bdae6ad4751dfddee9ea56 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 08:59:45 +0200 Subject: [PATCH 65/83] events-backend: rough performance test for event cleanup Signed-off-by: Patrik Oldsberg --- .../service/hub/DatabaseEventBusStore.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index 830ee1ae46..c962fe75a0 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -124,4 +124,39 @@ describe('DatabaseEventBusStore', () => { expect(events1.length).toBe(10); }, ); + + it.each(databases.eachSupportedId())( + 'should clean up a large number of events, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + }); + + const COUNT = '100000'; + + await db.raw(` + INSERT INTO event_bus_events (id, topic, data_json) + SELECT id, 'test', '{}' + FROM generate_series(1, ${COUNT}) AS id + `); + + await expect(db('event_bus_events').count()).resolves.toEqual([ + { count: COUNT }, + ]); + + const start = Date.now(); + + await store.clean(); + + // Local testing shows this takes about 80ms, but if this is flaky we can + // reduce the count down to 10_000. + expect(Date.now() - start).toBeLessThan(500); + + await expect(db('event_bus_events').count()).resolves.toEqual([ + { count: '5' }, + ]); + }, + ); }); From d75fdf178ebdb8af904bd5ba0c5c8bcb88ee3adb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 09:03:03 +0200 Subject: [PATCH 66/83] events-backend: fix moved import Signed-off-by: Patrik Oldsberg --- plugins/events-backend/dev/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/events-backend/dev/index.ts b/plugins/events-backend/dev/index.ts index b629cbf8d0..d75834884e 100644 --- a/plugins/events-backend/dev/index.ts +++ b/plugins/events-backend/dev/index.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { WinstonLogger } from '@backstage/backend-app-api'; +import { WinstonLogger } from '@backstage/backend-defaults/rootLogger'; import { createBackend } from '@backstage/backend-defaults'; import { coreServices, From ab0afa9bacbd5bae5fa8ffb02b39dc088d7a72a3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 09:15:29 +0200 Subject: [PATCH 67/83] events-backend: made some names in EventBusStore more explicit Signed-off-by: Patrik Oldsberg --- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 4 ++-- .../events-backend/src/service/hub/MemoryEventBusStore.ts | 4 ++-- .../events-backend/src/service/hub/createEventBusRouter.ts | 2 +- plugins/events-backend/src/service/hub/types.ts | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 63cc06d40d..b8a3e54ab8 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -350,7 +350,7 @@ export class DatabaseEventBusStore implements EventBusStore { async publish(options: { params: EventParams; consumedBy?: string[]; - }): Promise<{ id: string } | undefined> { + }): Promise<{ eventId: string } | undefined> { const topic = options.params.topic; const consumedBy = options.consumedBy ?? []; // This query inserts a new event into the database, but only if there are @@ -410,7 +410,7 @@ export class DatabaseEventBusStore implements EventBusStore { ); } - return { id }; + return { eventId: id }; } async upsertSubscription(id: string, topics: string[]): Promise { diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 3071a51edf..1527eeb1fa 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -33,7 +33,7 @@ export class MemoryEventBusStore implements EventBusStore { async publish(options: { params: EventParams; consumedBy: string[]; - }): Promise<{ id: string } | undefined> { + }): Promise<{ eventId: string } | undefined> { const topic = options.params.topic; const consumedBy = new Set(options.consumedBy); @@ -57,7 +57,7 @@ export class MemoryEventBusStore implements EventBusStore { this.#listeners.delete(listener); } } - return { id: String(nextSeq) }; + return { eventId: String(nextSeq) }; } #getMaxSeq() { diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 532ceaf2b8..fd0732e279 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -167,7 +167,7 @@ export async function createEventBusRouter(options: { consumedBy: req.body.consumedBy, }); if (result) { - logger.info(`Published event to '${topic}' with ID '${result.id}'`, { + logger.info(`Published event to '${topic}' with ID '${result.eventId}'`, { subject: credentials.principal.subject, }); res.status(201).end(); diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 315a4a7220..b221f6696d 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -20,11 +20,11 @@ export type EventBusStore = { publish(options: { params: EventParams; consumedBy?: string[]; - }): Promise<{ id: string } | undefined>; + }): Promise<{ eventId: string } | undefined>; - upsertSubscription(id: string, topics: string[]): Promise; + upsertSubscription(subscriptionId: string, topics: string[]): Promise; - readSubscription(id: string): Promise<{ events: EventParams[] }>; + readSubscription(subscriptionId: string): Promise<{ events: EventParams[] }>; setupListener( subscriptionId: string, From f1fb54af5b605e9db0fc2440fb1e220e41aae598 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 09:29:15 +0200 Subject: [PATCH 68/83] events-backend: rename consumedBy -> notifiedSubscribers Signed-off-by: Patrik Oldsberg --- .../src/schema/openapi.generated.ts | 2 +- plugins/events-backend/src/schema/openapi.yaml | 2 +- .../src/service/EventsPlugin.test.ts | 17 ++++++++++++----- .../src/service/hub/DatabaseEventBusStore.ts | 8 ++++---- .../src/service/hub/MemoryEventBusStore.ts | 14 ++++++++------ .../src/service/hub/createEventBusRouter.ts | 8 ++++---- plugins/events-backend/src/service/hub/types.ts | 2 +- .../events-node/src/api/DefaultEventsService.ts | 2 +- .../generated/models/PostEventRequest.model.ts | 2 +- 9 files changed, 33 insertions(+), 24 deletions(-) diff --git a/plugins/events-backend/src/schema/openapi.generated.ts b/plugins/events-backend/src/schema/openapi.generated.ts index ef712e2f86..8f8d6c6a82 100644 --- a/plugins/events-backend/src/schema/openapi.generated.ts +++ b/plugins/events-backend/src/schema/openapi.generated.ts @@ -159,7 +159,7 @@ export const spec = { event: { $ref: '#/components/schemas/Event', }, - consumedBy: { + notifiedSubscribers: { type: 'array', description: 'The IDs of subscriptions that have already received this event', diff --git a/plugins/events-backend/src/schema/openapi.yaml b/plugins/events-backend/src/schema/openapi.yaml index 8b2639a189..97f7705f87 100644 --- a/plugins/events-backend/src/schema/openapi.yaml +++ b/plugins/events-backend/src/schema/openapi.yaml @@ -106,7 +106,7 @@ paths: properties: event: $ref: '#/components/schemas/Event' - consumedBy: + notifiedSubscribers: type: array description: The IDs of subscriptions that have already received this event items: diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index ba9a236800..dcf901b1a0 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -119,12 +119,15 @@ describe('eventsPlugin', () => { publish( topic: string, payload: unknown, - options?: { consumedBy?: string[] }, + options?: { notifiedSubscribers?: string[] }, ) { return request(this.backend.server) .post('/api/events/bus/v1/events') .set('authorization', mockCredentials.service.header()) - .send({ event: { topic, payload }, consumedBy: options?.consumedBy }); + .send({ + event: { topic, payload }, + notifiedSubscribers: options?.notifiedSubscribers, + }); } readEvents(id: string) { @@ -278,7 +281,7 @@ describe('eventsPlugin', () => { 'test', { for: 'tester-2' }, { - consumedBy: ['tester-1'], + notifiedSubscribers: ['tester-1'], }, ) .expect(201); @@ -287,7 +290,7 @@ describe('eventsPlugin', () => { 'test', { for: 'tester-1' }, { - consumedBy: ['tester-2'], + notifiedSubscribers: ['tester-2'], }, ) .expect(201); @@ -362,7 +365,11 @@ describe('eventsPlugin', () => { await helper.subscribe('tester', ['test']).expect(201); await helper - .publish('test', { for: 'tester-2' }, { consumedBy: ['tester'] }) + .publish( + 'test', + { for: 'tester-2' }, + { notifiedSubscribers: ['tester'] }, + ) .expect(204); await backend.stop(); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index b8a3e54ab8..a41e95733d 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -349,10 +349,10 @@ export class DatabaseEventBusStore implements EventBusStore { async publish(options: { params: EventParams; - consumedBy?: string[]; + notifiedSubscribers?: string[]; }): Promise<{ eventId: string } | undefined> { const topic = options.params.topic; - const consumedBy = options.consumedBy ?? []; + const notifiedSubscribers = options.notifiedSubscribers ?? []; // This query inserts a new event into the database, but only if there are // subscribers to the topic that have not already been notified const result = await this.#db @@ -378,12 +378,12 @@ export class DatabaseEventBusStore implements EventBusStore { metadata: options.params.metadata, }), ]), - this.#db.raw('?', [consumedBy]), + this.#db.raw('?', [notifiedSubscribers]), ) // The rest of this query is to check whether there are any // subscribers that have not been notified yet .from(TABLE_SUBSCRIPTIONS) - .whereNotIn('id', consumedBy) // Skip notified subscribers + .whereNotIn('id', notifiedSubscribers) // Skip notified subscribers .andWhere(this.#db.raw('? = ANY(topics)', [topic])) // Match topic .having(this.#db.raw('count(*)'), '>', 0), // Check if there are any results ) diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 1527eeb1fa..6c7b4e6df7 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -20,7 +20,9 @@ import { NotFoundError } from '@backstage/errors'; const MAX_BATCH_SIZE = 10; export class MemoryEventBusStore implements EventBusStore { - #events = new Array }>(); + #events = new Array< + EventParams & { seq: number; notifiedSubscribers: Set } + >(); #subscribers = new Map< string, { id: string; seq: number; topics: Set } @@ -32,14 +34,14 @@ export class MemoryEventBusStore implements EventBusStore { async publish(options: { params: EventParams; - consumedBy: string[]; + notifiedSubscribers: string[]; }): Promise<{ eventId: string } | undefined> { const topic = options.params.topic; - const consumedBy = new Set(options.consumedBy); + const notifiedSubscribers = new Set(options.notifiedSubscribers); let hasOtherSubscribers = false; for (const sub of this.#subscribers.values()) { - if (sub.topics.has(topic) && !consumedBy.has(sub.id)) { + if (sub.topics.has(topic) && !notifiedSubscribers.has(sub.id)) { hasOtherSubscribers = true; break; } @@ -49,7 +51,7 @@ export class MemoryEventBusStore implements EventBusStore { } const nextSeq = this.#getMaxSeq() + 1; - this.#events.push({ ...options.params, consumedBy, seq: nextSeq }); + this.#events.push({ ...options.params, notifiedSubscribers, seq: nextSeq }); for (const listener of this.#listeners) { if (listener.topics.has(topic)) { @@ -88,7 +90,7 @@ export class MemoryEventBusStore implements EventBusStore { event => event.seq > sub.seq && sub.topics.has(event.topic) && - !event.consumedBy.has(id), + !event.notifiedSubscribers.has(id), ) .slice(0, MAX_BATCH_SIZE); diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index fd0732e279..193c1108d5 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -158,13 +158,13 @@ export async function createEventBusRouter(options: { allow: ['service'], }); const topic = req.body.event.topic; - const consumedBy = req.body.consumedBy; + const notifiedSubscribers = req.body.notifiedSubscribers; const result = await store.publish({ params: { topic, eventPayload: req.body.event.payload, } as EventParams, - consumedBy: req.body.consumedBy, + notifiedSubscribers: req.body.notifiedSubscribers, }); if (result) { logger.info(`Published event to '${topic}' with ID '${result.eventId}'`, { @@ -172,8 +172,8 @@ export async function createEventBusRouter(options: { }); res.status(201).end(); } else { - if (consumedBy) { - const notified = `'${consumedBy.join("', '")}'`; + if (notifiedSubscribers) { + const notified = `'${notifiedSubscribers.join("', '")}'`; logger.info( `Skipped publishing of event to '${topic}', subscribers have already been notified: ${notified}`, { subject: credentials.principal.subject }, diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index b221f6696d..3a4c8df9f8 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -19,7 +19,7 @@ import { EventParams } from '@backstage/plugin-events-node'; export type EventBusStore = { publish(options: { params: EventParams; - consumedBy?: string[]; + notifiedSubscribers?: string[]; }): Promise<{ eventId: string } | undefined>; upsertSubscription(subscriptionId: string, topics: string[]): Promise; diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 9e2509539a..9055afaeb8 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -127,7 +127,7 @@ class PluginEventsService implements EventsService { { body: { event: { payload: params.eventPayload, topic: params.topic }, - consumedBy: notifiedSubscribers, + notifiedSubscribers, }, }, { token }, diff --git a/plugins/events-node/src/generated/models/PostEventRequest.model.ts b/plugins/events-node/src/generated/models/PostEventRequest.model.ts index 04eaaab461..75289ed25b 100644 --- a/plugins/events-node/src/generated/models/PostEventRequest.model.ts +++ b/plugins/events-node/src/generated/models/PostEventRequest.model.ts @@ -24,5 +24,5 @@ export interface PostEventRequest { /** * The IDs of subscriptions that have already received this event */ - consumedBy?: Array; + notifiedSubscribers?: Array; } From 94b1caee5e30e16094c9e287dbb848da6fff4bfb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 09:56:09 +0200 Subject: [PATCH 69/83] events-backend: renamed EventBusStore params -> event Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.test.ts | 6 +++--- .../src/service/hub/DatabaseEventBusStore.ts | 8 ++++---- .../events-backend/src/service/hub/MemoryEventBusStore.ts | 6 +++--- .../src/service/hub/createEventBusRouter.ts | 2 +- plugins/events-backend/src/service/hub/types.ts | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index c962fe75a0..3c8d2cb377 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -35,7 +35,7 @@ describe('DatabaseEventBusStore', () => { for (let i = 0; i < 10; ++i) { await store.publish({ - params: { topic: 'test', eventPayload: { n: i } }, + event: { topic: 'test', eventPayload: { n: i } }, }); } @@ -75,7 +75,7 @@ describe('DatabaseEventBusStore', () => { for (let i = 0; i < 10; ++i) { await store.publish({ - params: { topic: 'test', eventPayload: { n: i } }, + event: { topic: 'test', eventPayload: { n: i } }, }); } @@ -114,7 +114,7 @@ describe('DatabaseEventBusStore', () => { for (let i = 0; i < 10; ++i) { await store.publish({ - params: { topic: 'test', eventPayload: { n: i } }, + event: { topic: 'test', eventPayload: { n: i } }, }); } diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index a41e95733d..c94fa2aa2c 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -348,10 +348,10 @@ export class DatabaseEventBusStore implements EventBusStore { } async publish(options: { - params: EventParams; + event: EventParams; notifiedSubscribers?: string[]; }): Promise<{ eventId: string } | undefined> { - const topic = options.params.topic; + const topic = options.event.topic; const notifiedSubscribers = options.notifiedSubscribers ?? []; // This query inserts a new event into the database, but only if there are // subscribers to the topic that have not already been notified @@ -374,8 +374,8 @@ export class DatabaseEventBusStore implements EventBusStore { this.#db.raw('?', [topic]), this.#db.raw('?', [ JSON.stringify({ - payload: options.params.eventPayload, - metadata: options.params.metadata, + payload: options.event.eventPayload, + metadata: options.event.metadata, }), ]), this.#db.raw('?', [notifiedSubscribers]), diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 6c7b4e6df7..2e4dba34bd 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -33,10 +33,10 @@ export class MemoryEventBusStore implements EventBusStore { }>(); async publish(options: { - params: EventParams; + event: EventParams; notifiedSubscribers: string[]; }): Promise<{ eventId: string } | undefined> { - const topic = options.params.topic; + const topic = options.event.topic; const notifiedSubscribers = new Set(options.notifiedSubscribers); let hasOtherSubscribers = false; @@ -51,7 +51,7 @@ export class MemoryEventBusStore implements EventBusStore { } const nextSeq = this.#getMaxSeq() + 1; - this.#events.push({ ...options.params, notifiedSubscribers, seq: nextSeq }); + this.#events.push({ ...options.event, notifiedSubscribers, seq: nextSeq }); for (const listener of this.#listeners) { if (listener.topics.has(topic)) { diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 193c1108d5..b1e99596b1 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -160,7 +160,7 @@ export async function createEventBusRouter(options: { const topic = req.body.event.topic; const notifiedSubscribers = req.body.notifiedSubscribers; const result = await store.publish({ - params: { + event: { topic, eventPayload: req.body.event.payload, } as EventParams, diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 3a4c8df9f8..0d363bec8b 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -18,7 +18,7 @@ import { EventParams } from '@backstage/plugin-events-node'; export type EventBusStore = { publish(options: { - params: EventParams; + event: EventParams; notifiedSubscribers?: string[]; }): Promise<{ eventId: string } | undefined>; From 5c728ee6425351eee9e9fd68decc3dbb51d90b1b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 10:40:19 +0200 Subject: [PATCH 70/83] changesets: changesets for new events backend bus Signed-off-by: Patrik Oldsberg --- .changeset/slow-gorillas-thank.md | 9 +++++++++ .changeset/stale-roses-serve.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .changeset/slow-gorillas-thank.md diff --git a/.changeset/slow-gorillas-thank.md b/.changeset/slow-gorillas-thank.md new file mode 100644 index 0000000000..2c52964a47 --- /dev/null +++ b/.changeset/slow-gorillas-thank.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-events-backend': patch +--- + +The events backend now has its own built-in event bus for distributing events across multiple backend instances. It exposes a new HTTP API under `/bus/v1/` for publishing and reading events from the bus, as well as its own storage and notification mechanism for events. + +The backing event store for the bus only supports scaled deployment if PostgreSQL is used as the DBMS. If SQLite or MySQL is used, the event bus will fall back to an in-memory store that does not support multiple backend instances. + +The default `EventsService` implementation from `@backstage/plugin-events-node` has also been updated to use the new events bus. diff --git a/.changeset/stale-roses-serve.md b/.changeset/stale-roses-serve.md index dc1ab50f88..29dee56e22 100644 --- a/.changeset/stale-roses-serve.md +++ b/.changeset/stale-roses-serve.md @@ -2,4 +2,4 @@ '@backstage/plugin-events-node': patch --- -The `EventParams` payload must now be a `JsonValue`, and the `metadata` type has been tweaked. +The default implementation of the `EventsService` now uses the new event bus for distributing events across multiple backend instances if the events backend plugin is installed. From 03cc3ce2692a9cd1d4dcee626e720120cb142dba Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 14:37:30 +0200 Subject: [PATCH 71/83] events-backend: clean up EventsPlugin test a bit Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index dcf901b1a0..700eb37bb7 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -149,10 +149,18 @@ describe('eventsPlugin', () => { }).factory; } + let backend: TestBackend | undefined = undefined; + afterEach(async () => { + if (backend) { + await backend.stop(); + backend = undefined; + } + }); + it.each(databases.eachSupportedId())( 'should be possible to publish events as a service, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -171,15 +179,13 @@ describe('eventsPlugin', () => { .publish('test', { n: 1 }) .set('authorization', mockCredentials.service.header()) .expect(204); // 204, since there are no subscribers - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should be possible to subscribe as a service and receive an event, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -211,15 +217,13 @@ describe('eventsPlugin', () => { await helper.readEvents('tester').expect(200, { events: [{ topic: 'test', payload: { n: 1 } }], }); - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should only send an event for each subscriber once, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -258,15 +262,13 @@ describe('eventsPlugin', () => { await helper.readEvents('tester-2').expect(200, { events: [{ topic: 'test', payload: { n: 2 } }], }); - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should not notify subscribers that have already consumed the event, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -303,15 +305,13 @@ describe('eventsPlugin', () => { await helper.readEvents('tester-2').expect(200, { events: [{ topic: 'test', payload: { for: 'tester-2' } }], }); - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should return multiple events in order, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -349,15 +349,13 @@ describe('eventsPlugin', () => { { topic: 'test', payload: { n: 14 } }, ], }); - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should skip publishing if all subscribers have already consumed the event, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -371,15 +369,13 @@ describe('eventsPlugin', () => { { notifiedSubscribers: ['tester'] }, ) .expect(204); - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should time out when no events are available, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); @@ -408,35 +404,31 @@ describe('eventsPlugin', () => { closed.then(() => { throw new Error('Closed'); }), - new Promise(r => process.nextTick(() => r(false))), + new Promise(r => process.nextTick(r)), ]); - await expect(checkNotClosed()).resolves.toBe(false); + await checkNotClosed(); await jest.advanceTimersByTimeAsync(30000); - await expect(checkNotClosed()).resolves.toBe(false); + await checkNotClosed(); await jest.advanceTimersByTimeAsync(30000); await closed; } finally { jest.useRealTimers(); } - - await backend.stop(); }, ); it.each(databases.eachSupportedId())( 'should refuse listen without a subscription, %p', async databaseId => { - const backend = await startTestBackend({ + backend = await startTestBackend({ features: [eventsPlugin, await mockKnexFactory(databaseId)], }); const helper = new ReqHelper(backend); await helper.readEvents('nonexistent').expect(404); - - await backend.stop(); }, ); }); From 4adf6e14cff91831ae36e87e69e86dd112b2325b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 13 Sep 2024 17:25:25 +0200 Subject: [PATCH 72/83] events-backend: fix for time out test in later versions of Node.js v20 Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 700eb37bb7..37b4cfd961 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -398,22 +398,21 @@ describe('eventsPlugin', () => { expect(res.status).toBe(202); - const { closed } = res.body!.getReader(); - const checkNotClosed = () => + const reader = res.body!.getReader(); + const isClosed = () => Promise.race([ - closed.then(() => { - throw new Error('Closed'); - }), - new Promise(r => process.nextTick(r)), + reader + .read() + .then(() => reader.closed) + .then(() => true), + new Promise(r => setTimeout(r, 100, false)), ]); - await checkNotClosed(); - + await expect(isClosed()).resolves.toBe(false); await jest.advanceTimersByTimeAsync(30000); - await checkNotClosed(); - + await expect(isClosed()).resolves.toBe(false); await jest.advanceTimersByTimeAsync(30000); - await closed; + await expect(isClosed()).resolves.toBe(true); } finally { jest.useRealTimers(); } From f14e127e4cf8646d3338b0348a589ffc470b9fc5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Sep 2024 17:47:26 +0200 Subject: [PATCH 73/83] Update plugins/events-node/src/api/DefaultEventsService.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Patrik Oldsberg --- plugins/events-node/src/api/DefaultEventsService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/events-node/src/api/DefaultEventsService.ts b/plugins/events-node/src/api/DefaultEventsService.ts index 9055afaeb8..aa77d6e696 100644 --- a/plugins/events-node/src/api/DefaultEventsService.ts +++ b/plugins/events-node/src/api/DefaultEventsService.ts @@ -174,7 +174,7 @@ class PluginEventsService implements EventsService { if (!res.ok) { if (res.status === 404) { this.logger.warn( - `Event subscribe request failed with status 404, events backend not found. Future events will not be persisted.`, + `Event subscribe request failed with status 404, events backend not found. Will only receive events that were sent locally on this process.`, ); delete this.client; return; From 74ebf8bcae6ea80453bb29c0497534f34a4a1127 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Sep 2024 20:36:24 +0200 Subject: [PATCH 74/83] events-backend: minor review fixes Signed-off-by: Patrik Oldsberg --- .../src/service/EventsPlugin.test.ts | 2 +- .../src/service/hub/DatabaseEventBusStore.ts | 23 +++++++++-------- .../src/service/hub/createEventBusRouter.ts | 25 +++++++++---------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/plugins/events-backend/src/service/EventsPlugin.test.ts b/plugins/events-backend/src/service/EventsPlugin.test.ts index 37b4cfd961..2c0befe1fd 100644 --- a/plugins/events-backend/src/service/EventsPlugin.test.ts +++ b/plugins/events-backend/src/service/EventsPlugin.test.ts @@ -139,7 +139,7 @@ describe('eventsPlugin', () => { } const databases = TestDatabases.create({ - ids: ['SQLITE_3', 'POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], + ids: ['SQLITE_3', 'MYSQL_8', 'POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], }); async function mockKnexFactory(databaseId: TestDatabaseId) { diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index c94fa2aa2c..0f4085e794 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -269,7 +269,6 @@ export class DatabaseEventBusStore implements EventBusStore { await db.migrate.latest({ directory: migrationsDir, }); - options.logger.info('DatabaseEventBusStore migrations ran successfully'); } const listener = new DatabaseEventBusListener(db.client, options.logger); @@ -288,7 +287,7 @@ export class DatabaseEventBusStore implements EventBusStore { frequency: { seconds: 10 }, timeout: { minutes: 1 }, initialDelay: { seconds: 10 }, - fn: store.#cleanup, + fn: () => store.#cleanup(), }); options.lifecycle.addShutdownHook(async () => { @@ -532,7 +531,7 @@ export class DatabaseEventBusStore implements EventBusStore { ); } - #cleanup = async () => { + async #cleanup() { try { const eventCount = await this.#db(TABLE_EVENTS) .delete() @@ -549,9 +548,11 @@ export class DatabaseEventBusStore implements EventBusStore { // If events are outside the max age they will always be deleted .orWhere('created_at', '<', new Date(Date.now() - this.#windowMaxAge)); - this.#logger.info( - `Event cleanup resulted in ${eventCount} old events being deleted`, - ); + if (eventCount > 0) { + this.#logger.info( + `Event cleanup resulted in ${eventCount} old events being deleted`, + ); + } } catch (error) { this.#logger.error('Event cleanup failed', error); } @@ -572,11 +573,13 @@ export class DatabaseEventBusStore implements EventBusStore { .where('read_until', '<', minId - 1); } - this.#logger.info( - `Subscription cleanup resulted in ${subscriberCount} stale subscribers being deleted`, - ); + if (subscriberCount > 0) { + this.#logger.info( + `Subscription cleanup resulted in ${subscriberCount} stale subscribers being deleted`, + ); + } } catch (error) { this.#logger.error('Subscription cleanup failed', error); } - }; + } } diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index b1e99596b1..485f1b0e58 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -22,7 +22,6 @@ import { SchedulerService, } from '@backstage/backend-plugin-api'; import { Handler } from 'express'; -import Router from 'express-promise-router'; import { createOpenApiRouter } from '../../schema/openapi.generated'; import { MemoryEventBusStore } from './MemoryEventBusStore'; import { DatabaseEventBusStore } from './DatabaseEventBusStore'; @@ -145,14 +144,11 @@ export async function createEventBusRouter(options: { }): Promise { const { httpAuth, notifyTimeoutMs = DEFAULT_NOTIFY_TIMEOUT_MS } = options; const logger = options.logger.child({ type: 'EventBus' }); - const router = Router(); const store = await createEventBusStore(options); const apiRouter = await createOpenApiRouter(); - router.use(apiRouter); - apiRouter.post('/bus/v1/events', async (req, res) => { const credentials = await httpAuth.credentials(req, { allow: ['service'], @@ -164,22 +160,25 @@ export async function createEventBusRouter(options: { topic, eventPayload: req.body.event.payload, } as EventParams, - notifiedSubscribers: req.body.notifiedSubscribers, + notifiedSubscribers, }); if (result) { - logger.info(`Published event to '${topic}' with ID '${result.eventId}'`, { - subject: credentials.principal.subject, - }); + logger.debug( + `Published event to '${topic}' with ID '${result.eventId}'`, + { + subject: credentials.principal.subject, + }, + ); res.status(201).end(); } else { if (notifiedSubscribers) { const notified = `'${notifiedSubscribers.join("', '")}'`; - logger.info( + logger.debug( `Skipped publishing of event to '${topic}', subscribers have already been notified: ${notified}`, { subject: credentials.principal.subject }, ); } else { - logger.info( + logger.debug( `Skipped publishing of event to '${topic}', no subscribers present`, { subject: credentials.principal.subject }, ); @@ -216,7 +215,7 @@ export async function createEventBusRouter(options: { try { const { events } = await store.readSubscription(id); - logger.info( + logger.debug( `Reading subscription '${id}' resulted in ${events.length} events`, { subject: credentials.principal.subject }, ); @@ -234,7 +233,7 @@ export async function createEventBusRouter(options: { try { const { topic } = await listener.waitForUpdate(); - logger.info( + logger.debug( `Received notification for subscription '${id}' for topic '${topic}'`, { subject: credentials.principal.subject }, ); @@ -266,7 +265,7 @@ export async function createEventBusRouter(options: { await store.upsertSubscription(id, req.body.topics); - logger.info( + logger.debug( `New subscription '${id}' for topics '${req.body.topics.join("', '")}'`, { subject: credentials.principal.subject }, ); From 483c8dc24329c9ccc385711e4c8cd9492b5d66ea Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Sep 2024 20:36:50 +0200 Subject: [PATCH 75/83] events-backend: proper cleanup of signal abort listeners Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 23 ++++++++++++++++--- .../src/service/hub/MemoryEventBusStore.ts | 19 ++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 0f4085e794..c51179896b 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -109,14 +109,31 @@ class DatabaseEventBusListener { await this.#ensureConnection(); const updatePromise = new Promise<{ topic: string }>((resolve, reject) => { - const listener = { topics, resolve, reject }; + const listener = { + topics, + resolve(result: { topic: string }) { + resolve(result); + cleanup(); + }, + reject(err: Error) { + reject(err); + cleanup(); + }, + }; this.#listeners.add(listener); - signal.addEventListener('abort', () => { + const onAbort = () => { this.#listeners.delete(listener); this.#maybeTimeoutConnection(); reject(signal.reason); - }); + cleanup(); + }; + + function cleanup() { + signal.removeEventListener('abort', onAbort); + } + + signal.addEventListener('abort', onAbort); }); // Ignore unhandled rejections diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 2e4dba34bd..b7b1d78052 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -115,13 +115,26 @@ export class MemoryEventBusStore implements EventBusStore { } return new Promise<{ topic: string }>((resolve, reject) => { - const listener = { topics: sub.topics, resolve }; + const listener = { + topics: sub.topics, + resolve(result: { topic: string }) { + resolve(result); + cleanup(); + }, + }; this.#listeners.add(listener); - options.signal.addEventListener('abort', () => { + const onAbort = () => { this.#listeners.delete(listener); reject(options.signal.reason); - }); + cleanup(); + }; + + function cleanup() { + options.signal.removeEventListener('abort', onAbort); + } + + options.signal.addEventListener('abort', onAbort); }); }, }; From cda26b88225de7536acf0aeb65907c67e114bd9e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Sep 2024 23:45:44 +0200 Subject: [PATCH 76/83] events-backend: add cleanup of old events for memory store Signed-off-by: Patrik Oldsberg --- .../service/hub/MemoryEventBusStore.test.ts | 86 +++++++++++++++++++ .../src/service/hub/MemoryEventBusStore.ts | 19 +++- 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts new file mode 100644 index 0000000000..6866f290b5 --- /dev/null +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventParams } from '@backstage/plugin-events-node'; +import { MemoryEventBusStore } from './MemoryEventBusStore'; + +function mkEvent(message: string): EventParams { + return { + topic: 'test', + eventPayload: { message }, + }; +} + +describe('MemoryEventBusStore', () => { + it('should publish to subscribers', async () => { + const store = new MemoryEventBusStore(); + + await expect( + store.publish({ + event: mkEvent('hello'), + notifiedSubscribers: [], + }), + ).resolves.toEqual(undefined); + + await expect(store.readSubscription('test')).rejects.toThrow( + 'Subscription not found', + ); + + await store.upsertSubscription('tester', ['test']); + + await expect( + store.publish({ + event: mkEvent('hello'), + notifiedSubscribers: [], + }), + ).resolves.toEqual({ eventId: '1' }); + + await expect( + store.publish({ + event: mkEvent('ignored'), + notifiedSubscribers: ['tester'], + }), + ).resolves.toEqual(undefined); + + await expect(store.readSubscription('tester')).resolves.toEqual({ + events: [mkEvent('hello')], + }); + }); + + it('should clean up old events', async () => { + const store = new MemoryEventBusStore({ maxEvents: 5 }); + + await store.upsertSubscription('tester', ['test']); + + for (let i = 0; i < 20; ++i) { + await expect( + store.publish({ + event: mkEvent(`hello ${i}`), + notifiedSubscribers: [], + }), + ).resolves.toEqual({ eventId: String(i + 1) }); + } + + await expect(store.readSubscription('tester')).resolves.toEqual({ + events: [ + mkEvent('hello 15'), + mkEvent('hello 16'), + mkEvent('hello 17'), + mkEvent('hello 18'), + mkEvent('hello 19'), + ], + }); + }); +}); diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index b7b1d78052..67d384b1fa 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -18,8 +18,10 @@ import { EventBusStore } from './types'; import { NotFoundError } from '@backstage/errors'; const MAX_BATCH_SIZE = 10; +const MAX_EVENTS_DEFAULT = 1_000; export class MemoryEventBusStore implements EventBusStore { + #maxEvents: number; #events = new Array< EventParams & { seq: number; notifiedSubscribers: Set } >(); @@ -32,6 +34,10 @@ export class MemoryEventBusStore implements EventBusStore { resolve(result: { topic: string }): void; }>(); + constructor(options: { maxEvents?: number } = {}) { + this.#maxEvents = options.maxEvents ?? MAX_EVENTS_DEFAULT; + } + async publish(options: { event: EventParams; notifiedSubscribers: string[]; @@ -59,6 +65,12 @@ export class MemoryEventBusStore implements EventBusStore { this.#listeners.delete(listener); } } + + // Trim old events + if (this.#events.length > this.#maxEvents) { + this.#events.shift(); + } + return { eventId: String(nextSeq) }; } @@ -96,7 +108,12 @@ export class MemoryEventBusStore implements EventBusStore { sub.seq = events[events.length - 1]?.seq ?? sub.seq; - return { events: events.map(event => ({ ...event, seq: undefined })) }; + return { + events: events.map(({ topic, eventPayload }) => ({ + topic, + eventPayload, + })), + }; } async setupListener( From 38edbc387ba0c7d01b8c7debac6b61f8003283c4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 18 Sep 2024 23:50:00 +0200 Subject: [PATCH 77/83] events-backend: refactor db query to clarify precedence Signed-off-by: Patrik Oldsberg --- .../src/service/hub/DatabaseEventBusStore.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index c51179896b..3a6a98d919 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -553,15 +553,22 @@ export class DatabaseEventBusStore implements EventBusStore { const eventCount = await this.#db(TABLE_EVENTS) .delete() // Delete any events that are outside both the min age and size window - .whereIn( - 'id', - this.#db - .select('id') - .from(TABLE_EVENTS) - .orderBy('id', 'desc') - .offset(this.#windowMaxCount), + .orWhere(inner => + inner + .whereIn( + 'id', + this.#db + .select('id') + .from(TABLE_EVENTS) + .orderBy('id', 'desc') + .offset(this.#windowMaxCount), + ) + .andWhere( + 'created_at', + '<', + new Date(Date.now() - this.#windowMinAge), + ), ) - .andWhere('created_at', '<', new Date(Date.now() - this.#windowMinAge)) // If events are outside the max age they will always be deleted .orWhere('created_at', '<', new Date(Date.now() - this.#windowMaxAge)); From e7a0b6572c539166543440aed84d15956b2a6564 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 19 Sep 2024 00:01:18 +0200 Subject: [PATCH 78/83] events-backend: added migration test Signed-off-by: Patrik Oldsberg --- plugins/events-backend/src/migrations.test.ts | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 plugins/events-backend/src/migrations.test.ts diff --git a/plugins/events-backend/src/migrations.test.ts b/plugins/events-backend/src/migrations.test.ts new file mode 100644 index 0000000000..16853f6052 --- /dev/null +++ b/plugins/events-backend/src/migrations.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import fs from 'fs'; + +const migrationsDir = `${__dirname}/../migrations`; +const migrationsFiles = fs.readdirSync(migrationsDir).sort(); + +async function migrateUpOnce(knex: Knex): Promise { + await knex.migrate.up({ directory: migrationsDir }); +} + +async function migrateDownOnce(knex: Knex): Promise { + await knex.migrate.down({ directory: migrationsDir }); +} + +async function migrateUntilBefore(knex: Knex, target: string): Promise { + const index = migrationsFiles.indexOf(target); + if (index === -1) { + throw new Error(`Migration ${target} not found`); + } + for (let i = 0; i < index; i++) { + await migrateUpOnce(knex); + } +} + +jest.setTimeout(60_000); + +describe('migrations', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_9', 'POSTGRES_13', 'POSTGRES_16'], + }); + + it.each(databases.eachSupportedId())( + '20240523100528_init.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore(knex, '20240523100528_init.js'); + await migrateUpOnce(knex); + + await knex('event_bus_events').insert({ + topic: 'test', + data_json: JSON.stringify({ message: 'hello' }), + consumed_by: ['tester'], + }); + await knex('event_bus_subscriptions').insert({ + id: 'tester', + read_until: '5', + topics: ['test', 'test2'], + }); + + await expect(knex('event_bus_events')).resolves.toEqual([ + { + id: '1', + topic: 'test', + data_json: JSON.stringify({ message: 'hello' }), + created_at: expect.anything(), + consumed_by: ['tester'], + }, + ]); + await expect(knex('event_bus_subscriptions')).resolves.toEqual([ + { + id: 'tester', + created_at: expect.anything(), + updated_at: expect.anything(), + read_until: '5', + topics: ['test', 'test2'], + }, + ]); + + await migrateDownOnce(knex); + + // This looks odd - you might expect a .toThrow at the end but that + // actually is flaky for some reason specifically on sqlite when + // performing multiple runs in sequence + await expect(knex('event_bus_events')).rejects.toEqual(expect.anything()); + + await knex.destroy(); + }, + ); +}); From 82412894273addaac64689c72f2afc62f3a06d2e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 19 Sep 2024 01:19:31 +0200 Subject: [PATCH 79/83] events-backend: track creator of events and subscriptions Signed-off-by: Patrik Oldsberg --- .../migrations/20240523100528_init.js | 8 +++ plugins/events-backend/src/migrations.test.ts | 4 ++ .../service/hub/DatabaseEventBusStore.test.ts | 55 +++++++++++++++---- .../src/service/hub/DatabaseEventBusStore.ts | 24 +++++++- .../service/hub/MemoryEventBusStore.test.ts | 5 ++ .../src/service/hub/MemoryEventBusStore.ts | 5 ++ .../src/service/hub/createEventBusRouter.ts | 3 +- .../events-backend/src/service/hub/types.ts | 11 +++- 8 files changed, 100 insertions(+), 15 deletions(-) diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js index 7ec85d7917..c3b66658fd 100644 --- a/plugins/events-backend/migrations/20240523100528_init.js +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -29,6 +29,10 @@ exports.up = async function up(knex) { .bigIncrements('id') .primary() .comment('The unique ID of this event'); + table + .text('created_by') + .notNullable() + .comment('The principal that published the event'); table .dateTime('created_at') .defaultTo(knex.fn.now()) @@ -53,6 +57,10 @@ exports.up = async function up(knex) { .primary() .notNullable() .comment('The unique ID of this particular subscription'); + table + .text('created_by') + .notNullable() + .comment('The principal that created the subscription'); table .dateTime('created_at') .defaultTo(knex.fn.now()) diff --git a/plugins/events-backend/src/migrations.test.ts b/plugins/events-backend/src/migrations.test.ts index 16853f6052..069b326075 100644 --- a/plugins/events-backend/src/migrations.test.ts +++ b/plugins/events-backend/src/migrations.test.ts @@ -56,11 +56,13 @@ describe('migrations', () => { await knex('event_bus_events').insert({ topic: 'test', + created_by: 'abc', data_json: JSON.stringify({ message: 'hello' }), consumed_by: ['tester'], }); await knex('event_bus_subscriptions').insert({ id: 'tester', + created_by: 'abc', read_until: '5', topics: ['test', 'test2'], }); @@ -68,6 +70,7 @@ describe('migrations', () => { await expect(knex('event_bus_events')).resolves.toEqual([ { id: '1', + created_by: 'abc', topic: 'test', data_json: JSON.stringify({ message: 'hello' }), created_at: expect.anything(), @@ -77,6 +80,7 @@ describe('migrations', () => { await expect(knex('event_bus_subscriptions')).resolves.toEqual([ { id: 'tester', + created_by: 'abc', created_at: expect.anything(), updated_at: expect.anything(), read_until: '5', diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index 3c8d2cb377..bfc68b7663 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { TestDatabases, mockServices } from '@backstage/backend-test-utils'; +import { + TestDatabases, + mockCredentials, + mockServices, +} from '@backstage/backend-test-utils'; import { DatabaseEventBusStore } from './DatabaseEventBusStore'; const logger = mockServices.logger.mock(); @@ -30,12 +34,21 @@ describe('DatabaseEventBusStore', () => { const db = await databases.init(databaseId); const store = await DatabaseEventBusStore.forTest({ logger, db }); - await store.upsertSubscription('tester-1', ['test']); - await store.upsertSubscription('tester-2', ['test']); + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); + await store.upsertSubscription( + 'tester-2', + ['test'], + mockCredentials.service(), + ); for (let i = 0; i < 10; ++i) { await store.publish({ event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), }); } @@ -48,7 +61,11 @@ describe('DatabaseEventBusStore', () => { "Subscription with ID 'tester-2' not found", ); - await store.upsertSubscription('tester-3', ['test']); + await store.upsertSubscription( + 'tester-3', + ['test'], + mockCredentials.service(), + ); // Reset read pointer to read form the beginning await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ @@ -70,12 +87,21 @@ describe('DatabaseEventBusStore', () => { maxAge: 0, }); - await store.upsertSubscription('tester-1', ['test']); - await store.upsertSubscription('tester-2', ['test']); + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); + await store.upsertSubscription( + 'tester-2', + ['test'], + mockCredentials.service(), + ); for (let i = 0; i < 10; ++i) { await store.publish({ event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), }); } @@ -88,7 +114,11 @@ describe('DatabaseEventBusStore', () => { "Subscription with ID 'tester-2' not found", ); - await store.upsertSubscription('tester-3', ['test']); + await store.upsertSubscription( + 'tester-3', + ['test'], + mockCredentials.service(), + ); // Reset read pointer to read form the beginning await db('event_bus_subscriptions').select({ id: 'tester-3' }).update({ @@ -110,11 +140,16 @@ describe('DatabaseEventBusStore', () => { minAge: 1000, }); - await store.upsertSubscription('tester-1', ['test']); + await store.upsertSubscription( + 'tester-1', + ['test'], + mockCredentials.service(), + ); for (let i = 0; i < 10; ++i) { await store.publish({ event: { topic: 'test', eventPayload: { n: i } }, + credentials: mockCredentials.service(), }); } @@ -137,8 +172,8 @@ describe('DatabaseEventBusStore', () => { const COUNT = '100000'; await db.raw(` - INSERT INTO event_bus_events (id, topic, data_json) - SELECT id, 'test', '{}' + INSERT INTO event_bus_events (id, created_by, topic, data_json) + SELECT id, 'abc', 'test', '{}' FROM generate_series(1, ${COUNT}) AS id `); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 3a6a98d919..a25856e88d 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -17,6 +17,8 @@ import { EventParams } from '@backstage/plugin-events-node'; import { EventBusStore } from './types'; import { Knex } from 'knex'; import { + BackstageCredentials, + BackstageServicePrincipal, DatabaseService, LifecycleService, LoggerService, @@ -40,6 +42,7 @@ const TOPIC_PUBLISH = 'event_bus_publish'; type EventsRow = { id: string; + created_by: string; created_at: Date; topic: string; data_json: string; @@ -48,12 +51,19 @@ type EventsRow = { type SubscriptionsRow = { id: string; + created_by: string; created_at: Date; updated_at: Date; read_until: string; topics: string[]; }; +function creatorId( + credentials: BackstageCredentials, +) { + return `service=${credentials.principal.subject}`; +} + const migrationsDir = resolvePackagePath( '@backstage/plugin-events-backend', 'migrations', @@ -366,6 +376,7 @@ export class DatabaseEventBusStore implements EventBusStore { async publish(options: { event: EventParams; notifiedSubscribers?: string[]; + credentials: BackstageCredentials; }): Promise<{ eventId: string } | undefined> { const topic = options.event.topic; const notifiedSubscribers = options.notifiedSubscribers ?? []; @@ -374,9 +385,10 @@ export class DatabaseEventBusStore implements EventBusStore { const result = await this.#db // There's no clean way to create a INSERT INTO .. SELECT with knex, so we end up with quite a lot of .raw(...) .into( - this.#db.raw('?? (??, ??, ??)', [ + this.#db.raw('?? (??, ??, ??, ??)', [ TABLE_EVENTS, // These are the rows that we insert, and should match the SELECT below + 'created_by', 'topic', 'data_json', 'consumed_by', @@ -387,6 +399,7 @@ export class DatabaseEventBusStore implements EventBusStore { q // We're not reading data to insert from anywhere else, just raw data .select( + this.#db.raw('?', [creatorId(options.credentials)]), this.#db.raw('?', [topic]), this.#db.raw('?', [ JSON.stringify({ @@ -429,17 +442,22 @@ export class DatabaseEventBusStore implements EventBusStore { return { eventId: id }; } - async upsertSubscription(id: string, topics: string[]): Promise { + async upsertSubscription( + id: string, + topics: string[], + credentials: BackstageCredentials, + ): Promise { const [{ max: maxId }] = await this.#db(TABLE_EVENTS).max('id'); const result = await this.#db(TABLE_SUBSCRIPTIONS) .insert({ id, + created_by: creatorId(credentials), updated_at: this.#db.fn.now(), topics, read_until: maxId || 0, }) .onConflict('id') - .merge(['topics', 'updated_at']) + .merge(['created_by', 'topics', 'updated_at']) .returning('*'); if (result.length !== 1) { diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts index 6866f290b5..52d075951d 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.test.ts @@ -15,6 +15,7 @@ */ import { EventParams } from '@backstage/plugin-events-node'; import { MemoryEventBusStore } from './MemoryEventBusStore'; +import { mockCredentials } from '@backstage/backend-test-utils'; function mkEvent(message: string): EventParams { return { @@ -31,6 +32,7 @@ describe('MemoryEventBusStore', () => { store.publish({ event: mkEvent('hello'), notifiedSubscribers: [], + credentials: mockCredentials.service(), }), ).resolves.toEqual(undefined); @@ -44,6 +46,7 @@ describe('MemoryEventBusStore', () => { store.publish({ event: mkEvent('hello'), notifiedSubscribers: [], + credentials: mockCredentials.service(), }), ).resolves.toEqual({ eventId: '1' }); @@ -51,6 +54,7 @@ describe('MemoryEventBusStore', () => { store.publish({ event: mkEvent('ignored'), notifiedSubscribers: ['tester'], + credentials: mockCredentials.service(), }), ).resolves.toEqual(undefined); @@ -69,6 +73,7 @@ describe('MemoryEventBusStore', () => { store.publish({ event: mkEvent(`hello ${i}`), notifiedSubscribers: [], + credentials: mockCredentials.service(), }), ).resolves.toEqual({ eventId: String(i + 1) }); } diff --git a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts index 67d384b1fa..6c999dc390 100644 --- a/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/MemoryEventBusStore.ts @@ -16,6 +16,10 @@ import { EventParams } from '@backstage/plugin-events-node'; import { EventBusStore } from './types'; import { NotFoundError } from '@backstage/errors'; +import { + BackstageCredentials, + BackstageServicePrincipal, +} from '@backstage/backend-plugin-api'; const MAX_BATCH_SIZE = 10; const MAX_EVENTS_DEFAULT = 1_000; @@ -41,6 +45,7 @@ export class MemoryEventBusStore implements EventBusStore { async publish(options: { event: EventParams; notifiedSubscribers: string[]; + credentials: BackstageCredentials; }): Promise<{ eventId: string } | undefined> { const topic = options.event.topic; const notifiedSubscribers = new Set(options.notifiedSubscribers); diff --git a/plugins/events-backend/src/service/hub/createEventBusRouter.ts b/plugins/events-backend/src/service/hub/createEventBusRouter.ts index 485f1b0e58..4a332d5ce5 100644 --- a/plugins/events-backend/src/service/hub/createEventBusRouter.ts +++ b/plugins/events-backend/src/service/hub/createEventBusRouter.ts @@ -161,6 +161,7 @@ export async function createEventBusRouter(options: { eventPayload: req.body.event.payload, } as EventParams, notifiedSubscribers, + credentials, }); if (result) { logger.debug( @@ -263,7 +264,7 @@ export async function createEventBusRouter(options: { }); const id = req.params.subscriptionId; - await store.upsertSubscription(id, req.body.topics); + await store.upsertSubscription(id, req.body.topics, credentials); logger.debug( `New subscription '${id}' for topics '${req.body.topics.join("', '")}'`, diff --git a/plugins/events-backend/src/service/hub/types.ts b/plugins/events-backend/src/service/hub/types.ts index 0d363bec8b..316d9e561f 100644 --- a/plugins/events-backend/src/service/hub/types.ts +++ b/plugins/events-backend/src/service/hub/types.ts @@ -14,15 +14,24 @@ * limitations under the License. */ +import { + BackstageCredentials, + BackstageServicePrincipal, +} from '@backstage/backend-plugin-api'; import { EventParams } from '@backstage/plugin-events-node'; export type EventBusStore = { publish(options: { event: EventParams; notifiedSubscribers?: string[]; + credentials: BackstageCredentials; }): Promise<{ eventId: string } | undefined>; - upsertSubscription(subscriptionId: string, topics: string[]): Promise; + upsertSubscription( + subscriptionId: string, + topics: string[], + credentials: BackstageCredentials, + ): Promise; readSubscription(subscriptionId: string): Promise<{ events: EventParams[] }>; From 359ab5936fb4f045ddf4a495520b9ce47f8a7ba7 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Sep 2024 11:14:06 +0200 Subject: [PATCH 80/83] events-backend: add index for event topics Signed-off-by: Patrik Oldsberg --- plugins/events-backend/migrations/20240523100528_init.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js index c3b66658fd..e94c6eb8dd 100644 --- a/plugins/events-backend/migrations/20240523100528_init.js +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -48,6 +48,8 @@ exports.up = async function up(knex) { .comment( 'The IDs of the subscribers that have already consumed this event', ); + + table.index('topic', 'event_bus_events_topic_idx'); }); await knex.schema.createTable('event_bus_subscriptions', table => { From ca3587e81d4f02f77a73b501c51ed81a63f4733e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Sep 2024 11:38:54 +0200 Subject: [PATCH 81/83] events-backend: rewrite subscription read query to force optimal path Signed-off-by: Patrik Oldsberg --- .../service/hub/DatabaseEventBusStore.test.ts | 40 +++++++ .../src/service/hub/DatabaseEventBusStore.ts | 103 ++++++++++-------- 2 files changed, 96 insertions(+), 47 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index bfc68b7663..4e5f72e1dd 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -194,4 +194,44 @@ describe('DatabaseEventBusStore', () => { ]); }, ); + + it.each(databases.eachSupportedId())( + 'should perform well when looking up events by topic, %p', + async databaseId => { + const db = await databases.init(databaseId); + const store = await DatabaseEventBusStore.forTest({ + logger, + db, + }); + + const COUNT = '100000'; + + // Insert 100,000 events, a lot more than we'd expect to ever have + // in a real-world scenario given our count window size is 10,000. + await db.raw(` + INSERT INTO event_bus_events (id, created_by, topic, data_json, consumed_by) + SELECT id, 'abc', CONCAT('test-', id), '{}', '{"${String( + Math.random(), + ).slice(2, 6)}"}' + FROM generate_series(1, ${COUNT}) AS id + `); + await db('event_bus_subscriptions').insert({ + id: 'tester', + created_by: 'abc', + read_until: 0, + topics: ['test-1000'], + }); + + const start = Date.now(); + const { events } = await store.readSubscription('tester'); + const duration = Date.now() - start; + + expect(events).toEqual([{ topic: 'test-1000', eventPayload: undefined }]); + + // When not run in an optimized way this takes anywhere from 10ms to + // 100ms. When optimized it's closed to 1ms, but we leave a bit of room + // for CI load spikes. + expect(duration).toBeLessThan(20); + }, + ); }); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index a25856e88d..86646cf4a8 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -468,53 +468,62 @@ export class DatabaseEventBusStore implements EventBusStore { } async readSubscription(id: string): Promise<{ events: EventParams[] }> { - const result = await this.#db(TABLE_SUBSCRIPTIONS) - // Read the target subscription so that we can use the read marker and topics - .with('sub', q => - q.select().from(TABLE_SUBSCRIPTIONS).where({ id }).forUpdate(), + // The below query selects the subscription we're reading from, locks it for + // an update, reads events for the subscription up to the limit, and then + // updates the pointer to the last read event. + // + // The query for selected_events is written in this particular way to force + // evaluation of the notified subscribers last. Without this, the query + // planner loves to first do a sequential scan of the events table to filter + // out the events that have already been consumed, but that is often a small + // subset and extremely wasteful. We instead want to make sure that the + // query is executed such that we select the events that are relevant to the + // subscription first, and then filter out any events that have already been + // consumed last. + // + // This is written as a plain SQL query to spare us all the horrors of + // expressing this in knex. + + const { rows: result } = await this.#db.raw<{ + rows: [] | [{ events: EventsRow[] }]; + }>( + ` + WITH subscription AS ( + SELECT topics, read_until + FROM event_bus_subscriptions + WHERE id = :id + FOR UPDATE + ), + selected_events AS ( + SELECT event_bus_events.* + FROM event_bus_events + INNER JOIN subscription + ON event_bus_events.topic = ANY(subscription.topics) + WHERE ( + CASE WHEN event_bus_events.id > subscription.read_until THEN + CASE WHEN NOT :id = ANY(event_bus_events.consumed_by) + THEN 1 + END + END + ) = 1 + ORDER BY event_bus_events.id ASC LIMIT :limit + ), + last_event_id AS ( + SELECT max(id) AS last_event_id + FROM selected_events + ), + events_array AS ( + SELECT json_agg(row_to_json(selected_events)) AS events + FROM selected_events ) - // Read the next batch of events for the subscription from its read marker - .with('events', q => - q - .select('event.*') - .from({ event: 'event_bus_events', sub: 'sub' }) - // For each event, check if it matches any of the topics that we're subscribed to - .where( - 'event.topic', - '=', - this.#db.ref('topics').withSchema('sub').wrap('ANY(', ')'), - ) - // Skip events that have already been consumed by this subscription - .whereNot( - this.#db.raw('?', id), - '=', - this.#db.ref('consumed_by').withSchema('event').wrap('ANY(', ')'), - ) - .where('event.id', '>', this.#db.ref('read_until').withSchema('sub')) - .orderBy('event.id', 'asc') - .limit(MAX_BATCH_SIZE), - ) - // Find the ID of the last event in the batch, for use as the new read_until marker - .with('last_event_id', q => q.max({ last_event_id: 'id' }).from('events')) - // Aggregate the events into a JSON array so that we can return all of them with the UPDATE - .with('events_array', q => - q - .select({ events: this.#db.raw('json_agg(row_to_json(events))') }) - .from('events'), - ) - // Update the read_until marker to the ID of the last event, or if no - // events where read, the last ID out of all events - .update({ - read_until: this.#db.raw( - 'COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events), 0)', - ), - }) - .updateFrom({ - events_array: 'events_array', - last_event_id: 'last_event_id', - }) - .where(`${TABLE_SUBSCRIPTIONS}.id`, id) - .returning<[{ events: EventsRow[] }]>('events_array.events'); + UPDATE event_bus_subscriptions + SET read_until = COALESCE(last_event_id, (SELECT MAX(id) FROM event_bus_events), 0) + FROM events_array, last_event_id + WHERE event_bus_subscriptions.id = :id + RETURNING events_array.events + `, + { id, limit: MAX_BATCH_SIZE }, + ); if (result.length === 0) { throw new NotFoundError(`Subscription with ID '${id}' not found`); @@ -530,7 +539,7 @@ export class DatabaseEventBusStore implements EventBusStore { } return { - events: result[0].events.map(row => { + events: rows.map(row => { const { payload, metadata } = JSON.parse(row.data_json); return { topic: row.topic, From 3bd926905efb514cb3434ff63ffefdd827bd8724 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Sep 2024 11:42:20 +0200 Subject: [PATCH 82/83] events-backend: rename column consumed_by -> notified_subscribers Signed-off-by: Patrik Oldsberg --- plugins/events-backend/migrations/20240523100528_init.js | 2 +- plugins/events-backend/src/migrations.test.ts | 4 ++-- .../src/service/hub/DatabaseEventBusStore.test.ts | 2 +- .../events-backend/src/service/hub/DatabaseEventBusStore.ts | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/events-backend/migrations/20240523100528_init.js b/plugins/events-backend/migrations/20240523100528_init.js index e94c6eb8dd..9fbef79a75 100644 --- a/plugins/events-backend/migrations/20240523100528_init.js +++ b/plugins/events-backend/migrations/20240523100528_init.js @@ -44,7 +44,7 @@ exports.up = async function up(knex) { .notNullable() .comment('The payload data of this event'); table - .specificType('consumed_by', 'text ARRAY') + .specificType('notified_subscribers', 'text ARRAY') .comment( 'The IDs of the subscribers that have already consumed this event', ); diff --git a/plugins/events-backend/src/migrations.test.ts b/plugins/events-backend/src/migrations.test.ts index 069b326075..1c90260ed3 100644 --- a/plugins/events-backend/src/migrations.test.ts +++ b/plugins/events-backend/src/migrations.test.ts @@ -58,7 +58,7 @@ describe('migrations', () => { topic: 'test', created_by: 'abc', data_json: JSON.stringify({ message: 'hello' }), - consumed_by: ['tester'], + notified_subscribers: ['tester'], }); await knex('event_bus_subscriptions').insert({ id: 'tester', @@ -74,7 +74,7 @@ describe('migrations', () => { topic: 'test', data_json: JSON.stringify({ message: 'hello' }), created_at: expect.anything(), - consumed_by: ['tester'], + notified_subscribers: ['tester'], }, ]); await expect(knex('event_bus_subscriptions')).resolves.toEqual([ diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index 4e5f72e1dd..e1a60f9f05 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -209,7 +209,7 @@ describe('DatabaseEventBusStore', () => { // Insert 100,000 events, a lot more than we'd expect to ever have // in a real-world scenario given our count window size is 10,000. await db.raw(` - INSERT INTO event_bus_events (id, created_by, topic, data_json, consumed_by) + INSERT INTO event_bus_events (id, created_by, topic, data_json, notified_subscribers) SELECT id, 'abc', CONCAT('test-', id), '{}', '{"${String( Math.random(), ).slice(2, 6)}"}' diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index 86646cf4a8..b1f26ab841 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -46,7 +46,7 @@ type EventsRow = { created_at: Date; topic: string; data_json: string; - consumed_by: string[]; + notified_subscribers: string[]; }; type SubscriptionsRow = { @@ -391,7 +391,7 @@ export class DatabaseEventBusStore implements EventBusStore { 'created_by', 'topic', 'data_json', - 'consumed_by', + 'notified_subscribers', ]), ) .insert( @@ -501,7 +501,7 @@ export class DatabaseEventBusStore implements EventBusStore { ON event_bus_events.topic = ANY(subscription.topics) WHERE ( CASE WHEN event_bus_events.id > subscription.read_until THEN - CASE WHEN NOT :id = ANY(event_bus_events.consumed_by) + CASE WHEN NOT :id = ANY(event_bus_events.notified_subscribers) THEN 1 END END From 6d28caf285ad731ec157a67568a9590944f786f8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 20 Sep 2024 12:17:46 +0200 Subject: [PATCH 83/83] events-backend: drop optimization for sparse events Signed-off-by: Patrik Oldsberg --- .../service/hub/DatabaseEventBusStore.test.ts | 20 +++++++++++++------ .../src/service/hub/DatabaseEventBusStore.ts | 18 ++--------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts index e1a60f9f05..7086d90cbf 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.test.ts @@ -210,7 +210,7 @@ describe('DatabaseEventBusStore', () => { // in a real-world scenario given our count window size is 10,000. await db.raw(` INSERT INTO event_bus_events (id, created_by, topic, data_json, notified_subscribers) - SELECT id, 'abc', CONCAT('test-', id), '{}', '{"${String( + SELECT id, 'abc', CONCAT('test-', MOD(id, 10)), CONCAT('{"payload":{"id":"', id, '"}}'), '{"${String( Math.random(), ).slice(2, 6)}"}' FROM generate_series(1, ${COUNT}) AS id @@ -219,18 +219,26 @@ describe('DatabaseEventBusStore', () => { id: 'tester', created_by: 'abc', read_until: 0, - topics: ['test-1000'], + topics: ['test-5'], }); const start = Date.now(); const { events } = await store.readSubscription('tester'); const duration = Date.now() - start; - expect(events).toEqual([{ topic: 'test-1000', eventPayload: undefined }]); + expect(events).toEqual([ + { topic: 'test-5', eventPayload: { id: '5' } }, + { topic: 'test-5', eventPayload: { id: '15' } }, + { topic: 'test-5', eventPayload: { id: '25' } }, + { topic: 'test-5', eventPayload: { id: '35' } }, + { topic: 'test-5', eventPayload: { id: '45' } }, + { topic: 'test-5', eventPayload: { id: '55' } }, + { topic: 'test-5', eventPayload: { id: '65' } }, + { topic: 'test-5', eventPayload: { id: '75' } }, + { topic: 'test-5', eventPayload: { id: '85' } }, + { topic: 'test-5', eventPayload: { id: '95' } }, + ]); - // When not run in an optimized way this takes anywhere from 10ms to - // 100ms. When optimized it's closed to 1ms, but we leave a bit of room - // for CI load spikes. expect(duration).toBeLessThan(20); }, ); diff --git a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts index b1f26ab841..107dabfc27 100644 --- a/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts +++ b/plugins/events-backend/src/service/hub/DatabaseEventBusStore.ts @@ -472,15 +472,6 @@ export class DatabaseEventBusStore implements EventBusStore { // an update, reads events for the subscription up to the limit, and then // updates the pointer to the last read event. // - // The query for selected_events is written in this particular way to force - // evaluation of the notified subscribers last. Without this, the query - // planner loves to first do a sequential scan of the events table to filter - // out the events that have already been consumed, but that is often a small - // subset and extremely wasteful. We instead want to make sure that the - // query is executed such that we select the events that are relevant to the - // subscription first, and then filter out any events that have already been - // consumed last. - // // This is written as a plain SQL query to spare us all the horrors of // expressing this in knex. @@ -499,13 +490,8 @@ export class DatabaseEventBusStore implements EventBusStore { FROM event_bus_events INNER JOIN subscription ON event_bus_events.topic = ANY(subscription.topics) - WHERE ( - CASE WHEN event_bus_events.id > subscription.read_until THEN - CASE WHEN NOT :id = ANY(event_bus_events.notified_subscribers) - THEN 1 - END - END - ) = 1 + WHERE event_bus_events.id > subscription.read_until + AND NOT :id = ANY(event_bus_events.notified_subscribers) ORDER BY event_bus_events.id ASC LIMIT :limit ), last_event_id AS (