paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 49 Packages Projects Releases Wiki Activity

Merge pull request #16678 from RoadieHQ/add-export-for-in-memory-broker

Browse Source 
Add export for in memory broker
This commit is contained in:
Patrik Oldsberg
2023-04-04 17:29:17 +02:00
committed by GitHub
parent d34708e81d 386cfe0fc9
commit 6aa6c32f14
16 changed files with 240 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/tall-meals-dress.md
+7
View File
@@ -0,0 +1,7 @@
---
'@backstage/plugin-events-backend': patch
---
Export `DefaultEventBroker` to allow decoupling of the catalog and events backends in the `example-backend`.
Please look at `plugins/events-backend/README.md` for the currently advised way to set up the event backend and catalog providers.
docs/integrations/bitbucketCloud/discovery.md
+18 -17
View File
@@ -76,36 +76,37 @@ Additionally, you need to decide how you want to receive events from external so
Set up your provider
```ts title="packages/backend/src/plugins/catalogEventBasedProviders.ts"
```ts title="packages/backend/src/plugins/catalog.ts"
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
/* highlight-add-start */
import { CatalogClient } from '@backstage/catalog-client';
import { BitbucketCloudEntityProvider } from '@backstage/plugin-catalog-backend-module-bitbucket-cloud';
/* highlight-add-end */
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createCatalogEventBasedProviders(
/* highlight-remove-next-line */
_: PluginEnvironment,
/* highlight-add-next-line */
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
// add your event-based entity providers here
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
/* highlight-add-start */
providers.push(
BitbucketCloudEntityProvider.fromConfig(env.config, {
const bitbucketCloudProvider = BitbucketCloudEntityProvider.fromConfig(
env.config,
{
catalogApi: new CatalogClient({ discoveryApi: env.discovery }),
logger: env.logger,
scheduler: env.scheduler,
tokenManager: env.tokenManager,
}),
},
);
env.eventBroker.subscribe(bitbucketCloudProvider);
builder.addEntityProvider(bitbucketCloudProvider);
/* highlight-add-end */
return providers.flat();
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
docs/integrations/github/discovery.md
+23 -24
View File
@@ -69,37 +69,36 @@ Additionally, you need to decide how you want to receive events from external so
Set up your provider
```ts title="packages/backend/src/plugins/catalogEventBasedProviders.ts"
```ts title="packages/backend/src/plugins/catalog.ts"
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
/* highlight-add-next-line */
import { GithubEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createCatalogEventBasedProviders(
/* highlight-remove-next-line */
_: PluginEnvironment,
/* highlight-add-next-line */
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
// add your event-based entity providers here
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
/* highlight-add-start */
providers.push(
GithubEntityProvider.fromConfig(env.config, {
logger: env.logger,
// optional: alternatively, use scheduler with schedule defined in app-config.yaml
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 3 },
}),
// optional: alternatively, use schedule
scheduler: env.scheduler,
const githubProvider = GithubEntityProvider.fromConfig(env.config, {
logger: env.logger,
// optional: alternatively, use scheduler with schedule defined in app-config.yaml
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 3 },
}),
);
// optional: alternatively, use schedule
scheduler: env.scheduler,
});
env.eventBroker.subscribe(githubProvider);
builder.addEntityProvider(demoProvider);
/* highlight-add-end */
return providers.flat();
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
docs/integrations/github/org.md
+15 -17
View File
@@ -79,25 +79,21 @@ Additionally, you need to decide how you want to receive events from external so
Set up your provider
```ts title="packages/backend/src/plugins/catalogEventBasedProviders.ts"
```ts title="packages/backend/src/plugins/catalog.ts"
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
/* highlight-add-next-line */
import { GithubOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createCatalogEventBasedProviders(
/* highlight-remove-next-line */
_: PluginEnvironment,
/* highlight-add-next-line */
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
// add your event-based entity providers here
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
/* highlight-add-start */
providers.push(
GithubOrgEntityProvider.fromConfig(env.config, {
const githubOrgProvider = GithubOrgEntityProvider.fromConfig(env.config, {
id: 'production',
orgUrl: 'https://github.com/backstage',
logger: env.logger,
@@ -105,10 +101,12 @@ export default async function createCatalogEventBasedProviders(
frequency: { minutes: 60 },
timeout: { minutes: 15 },
}),
}),
);
env.eventBroker.subscribe(githubOrgProvider);
builder.addEntityProvider(githubOrgProvider);
/* highlight-add-end */
return providers.flat();
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
packages/backend/src/index.ts
+6 -9
View File
@@ -42,7 +42,6 @@ import { metricsInit, metricsHandler } from './metrics';
import auth from './plugins/auth';
import azureDevOps from './plugins/azure-devops';
import catalog from './plugins/catalog';
import catalogEventBasedProviders from './plugins/catalogEventBasedProviders';
import codeCoverage from './plugins/codecoverage';
import entityFeedback from './plugins/entityFeedback';
import events from './plugins/events';
@@ -68,6 +67,7 @@ import linguist from './plugins/linguist';
import { PluginEnvironment } from './types';
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
import { DefaultIdentityClient } from '@backstage/plugin-auth-node';
import { DefaultEventBroker } from '@backstage/plugin-events-backend';
function makeCreateEnv(config: Config) {
const root = getRootLogger();
@@ -85,6 +85,8 @@ function makeCreateEnv(config: Config) {
discovery,
});
const eventBroker = new DefaultEventBroker(root.child({ type: 'plugin' }));
root.info(`Created UrlReader ${reader}`);
return (plugin: string): PluginEnvironment => {
@@ -99,6 +101,7 @@ function makeCreateEnv(config: Config) {
database,
config,
reader,
eventBroker,
discovery,
tokenManager,
permissions,
@@ -156,18 +159,12 @@ async function main() {
const exploreEnv = useHotMemoize(module, () => createEnv('explore'));
const lighthouseEnv = useHotMemoize(module, () => createEnv('lighthouse'));
const eventBasedEntityProviders = await catalogEventBasedProviders(
catalogEnv,
);
const linguistEnv = useHotMemoize(module, () => createEnv('linguist'));
const apiRouter = Router();
apiRouter.use(
'/catalog',
await catalog(catalogEnv, eventBasedEntityProviders),
);
apiRouter.use('/catalog', await catalog(catalogEnv));
apiRouter.use('/code-coverage', await codeCoverage(codeCoverageEnv));
apiRouter.use('/events', await events(eventsEnv, eventBasedEntityProviders));
apiRouter.use('/events', await events(eventsEnv));
apiRouter.use('/rollbar', await rollbar(rollbarEnv));
apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv));
apiRouter.use('/tech-insights', await techInsights(techInsightsEnv));
packages/backend/src/plugins/catalogEventBasedProviders.ts → packages/backend/src/plugins/DemoEventBasedEntityProvider.ts
+21 -18
View File
@@ -18,15 +18,29 @@ import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-node';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import {
EventBroker,
EventParams,
EventSubscriber,
} from '@backstage/plugin-events-node';
import { Logger } from 'winston';
import { PluginEnvironment } from '../types';
class DemoEventBasedEntityProvider implements EntityProvider, EventSubscriber {
constructor(
private readonly logger: Logger,
private readonly topics: string[],
) {}
export class DemoEventBasedEntityProvider
implements EntityProvider, EventSubscriber
{
private readonly logger: Logger;
private readonly topics: string[];
constructor(opts: {
eventBroker: EventBroker;
logger: Logger;
topics: string[];
}) {
const { eventBroker, logger, topics } = opts;
eventBroker.subscribe(this);
this.logger = logger;
this.topics = topics;
}
async onEvent(params: EventParams): Promise<void> {
this.logger.info(
@@ -48,14 +62,3 @@ class DemoEventBasedEntityProvider implements EntityProvider, EventSubscriber {
return DemoEventBasedEntityProvider.name;
}
}
export default async function createCatalogEventBasedProviders(
env: PluginEnvironment,
): Promise<Array<EntityProvider & EventSubscriber>> {
const providers: Array<
(EntityProvider & EventSubscriber) | Array<EntityProvider & EventSubscriber>
> = [];
providers.push(new DemoEventBasedEntityProvider(env.logger, ['example']));
// add your event-based entity providers here
return providers.flat();
}
packages/backend/src/plugins/catalog.ts
+9 -3
View File
@@ -15,18 +15,24 @@
*/
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
import { EntityProvider } from '@backstage/plugin-catalog-node';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
import { DemoEventBasedEntityProvider } from './DemoEventBasedEntityProvider';
export default async function createPlugin(
env: PluginEnvironment,
providers?: Array<EntityProvider>,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
builder.addEntityProvider(providers ?? []);
const demoProvider = new DemoEventBasedEntityProvider({
logger: env.logger,
topics: ['example'],
eventBroker: env.eventBroker,
});
builder.addEntityProvider(demoProvider);
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
packages/backend/src/plugins/events.ts
+1 -3
View File
@@ -18,13 +18,11 @@ import {
EventsBackend,
HttpPostIngressEventPublisher,
} from '@backstage/plugin-events-backend';
import { EventSubscriber } from '@backstage/plugin-events-node';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createPlugin(
env: PluginEnvironment,
subscribers: EventSubscriber[],
): Promise<Router> {
const eventsRouter = Router();
@@ -35,8 +33,8 @@ export default async function createPlugin(
http.bind(eventsRouter);
await new EventsBackend(env.logger)
.setEventBroker(env.eventBroker)
.addPublishers(http)
.addSubscribers(subscribers)
.start();
return eventsRouter;
packages/backend/src/types.ts
+2
View File
@@ -26,6 +26,7 @@ import {
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { IdentityApi } from '@backstage/plugin-auth-node';
import { PermissionEvaluator } from '@backstage/plugin-permission-common';
import { EventBroker } from '@backstage/plugin-events-node';
export type PluginEnvironment = {
logger: Logger;
@@ -38,4 +39,5 @@ export type PluginEnvironment = {
permissions: PermissionEvaluator;
scheduler: PluginTaskScheduler;
identity: IdentityApi;
eventBroker: EventBroker;
};
plugins/events-backend/README.md
+113 -49
View File
@@ -6,7 +6,7 @@ This plugin provides the wiring of all extension points
for managing events as defined by [plugin-events-node](../events-node)
including backend plugin `EventsPlugin` and `EventsBackend`.
Additionally, it uses a simple in-memory implementation for
Additionally, it uses a simple in-process implementation for
the `EventBroker` by default which you can replace with a more sophisticated
implementation of your choice as you need (e.g., via module).
@@ -24,10 +24,89 @@ to the used event broker.
yarn add --cwd packages/backend @backstage/plugin-events-backend
```
Add a file [`packages/backend/src/plugins/events.ts`](../../packages/backend/src/plugins/events.ts)
to your Backstage project.
### Event Broker
There, you can add all publishers, subscribers, etc. you want.
First you will need to add and implementation of the `EventBroker` interface to the backend plugin environment.
This will allow event broker instance any backend plugins to publish and subscribe to events in order to communicate
between them.
Add the following to `makeCreateEnv`
```diff
// packages/backend/src/index.ts
+ const eventBroker = new DefaultEventBroker(root.child({ type: 'plugin' }));
```
Then update plugin environment to include the event broker.
```diff
// packages/backend/src/types.ts
+ eventBroker: EventBroker;
```
### Publishing and Subscribing to events with the broker
Backend plugins are passed the event broker in the plugin environment at startup of the application. The plugin can
make use of this to communicate between parts of the application.
Here is an example of a plugin publishing a payload to a topic.
```typescript jsx
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
env.eventBroker.publish({
topic: 'publish.example',
eventPayload: { message: 'Hello, World!' },
metadata: {},
});
}
```
Here is an example of a plugin subscribing to a topic.
```typescript jsx
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
env.eventBroker.subscribe([
{
supportsEventTopics: ['publish.example'],
onEvent: async (params: EventParams) => {
env.logger.info(`receieved ${params.topic} event`);
},
},
]);
}
```
### Implementing an `EventSubscriber` class
More complex solutions might need the creation of a class that implements the `EventSubscriber` interface. e.g.
```typescript jsx
import { EventSubscriber } from "./EventSubscriber";
class ExampleSubscriber implements EventSubscriber {
...
supportsEventTopics() {
return ['publish.example']
}
async onEvent(params: EventParams) {
env.logger.info(`receieved ${params.topic} event`)
}
}
```
### Events Backend
The events backend plugin provides a router to handler http events and publish the http requests onto the event
broker.
To configure it add a file [`packages/backend/src/plugins/events.ts`](../../packages/backend/src/plugins/events.ts)
to your Backstage project.
Additionally, add the events plugin to your backend.
@@ -38,54 +117,11 @@ Additionally, add the events plugin to your backend.
// [...]
+ const eventsEnv = useHotMemoize(module, () => createEnv('events'));
// [...]
+ apiRouter.use('/events', await events(eventsEnv, []));
+ apiRouter.use('/events', await events(eventsEnv));
// [...]
```
### With Event-based Entity Providers
In case you use event-based `EntityProviders`,
you may need something like the following:
```diff
// packages/backend/src/index.ts
- apiRouter.use('/events', await events(eventsEnv, []));
+ apiRouter.use('/events', await events(eventsEnv, eventBasedEntityProviders));
```
as well as a file
[`packages/backend/src/plugins/catalogEventBasedProviders.ts`](../../packages/backend/src/plugins/catalogEventBasedProviders.ts)
which contains event-based entity providers.
In case you don't have this dependency added yet:
```bash
# From your Backstage root directory
yarn add --cwd packages/backend @backstage/plugin-events-backend
```
```diff
// packages/backend/src/plugins/catalog.ts
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+import { EntityProvider } from '@backstage/plugin-catalog-node';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createPlugin(
env: PluginEnvironment,
+ providers?: Array<EntityProvider>,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
+ builder.addEntityProvider(providers ?? []);
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
## Configuration
#### Configuration
In order to create HTTP endpoints to receive events for a certain
topic, you need to add them at your configuration:
@@ -115,6 +151,34 @@ in combination with suitable event subscribers.
However, it is not limited to these use cases.
### Event-based Entity Providers
You can implement the `EventSubscriber` interface on an `EntityProviders` to allow it to handle events from other plugins e.g. the event backend plugin
mentioned above.
Assuming you have configured the `eventBroker` into the `PluginEnvironment` you can pass the broker to the entity provider for it to subscribe.
```diff
// packages/backend/src/plugins/catalog.ts
import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
+import { DemoEventBasedEntityProvider } from './DemoEventBasedEntityProvider';
import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
import { Router } from 'express';
import { PluginEnvironment } from '../types';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
const builder = await CatalogBuilder.create(env);
builder.addProcessor(new ScaffolderEntitiesProcessor());
+ const demoProvider = new DemoEventBasedEntityProvider({ logger: env.logger, topics: ['example'], eventBroker: env.eventBroker });
+ builder.addEntityProvider(demoProvider);
const { processingEngine, router } = await builder.build();
await processingEngine.start();
return router;
}
```
## Use Cases
### Custom Event Broker
plugins/events-backend/api-report.md
+12
View File
@@ -5,12 +5,24 @@
```ts
import { Config } from '@backstage/config';
import { EventBroker } from '@backstage/plugin-events-node';
import { EventParams } from '@backstage/plugin-events-node';
import { EventPublisher } from '@backstage/plugin-events-node';
import { EventSubscriber } from '@backstage/plugin-events-node';
import express from 'express';
import { HttpPostIngressOptions } from '@backstage/plugin-events-node';
import { Logger } from 'winston';
// @public
export class DefaultEventBroker implements EventBroker {
constructor(logger: Logger);
// (undocumented)
publish(params: EventParams): Promise<void>;
// (undocumented)
subscribe(
...subscribers: Array<EventSubscriber | Array<EventSubscriber>>
): void;
}
// @public
export class EventsBackend {
constructor(logger: Logger);
plugins/events-backend/src/index.ts
+1
View File
@@ -22,3 +22,4 @@
export { EventsBackend } from './service/EventsBackend';
export { HttpPostIngressEventPublisher } from './service/http';
export { DefaultEventBroker } from './service/DefaultEventBroker';
plugins/events-backend/src/service/InMemoryEventBroker.test.ts → plugins/events-backend/src/service/DefaultEventBroker.test.ts
+4 -4
View File
@@ -17,15 +17,15 @@
import { getVoidLogger } from '@backstage/backend-common';
import { TestEventSubscriber } from '@backstage/plugin-events-backend-test-utils';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import { InMemoryEventBroker } from './InMemoryEventBroker';
import { DefaultEventBroker } from './DefaultEventBroker';
const logger = getVoidLogger();
describe('InMemoryEventBroker', () => {
describe('DefaultEventBroker', () => {
it('passes events to interested subscribers', () => {
const subscriber1 = new TestEventSubscriber('test1', ['topicA', 'topicB']);
const subscriber2 = new TestEventSubscriber('test2', ['topicB', 'topicC']);
const eventBroker = new InMemoryEventBroker(logger);
const eventBroker = new DefaultEventBroker(logger);
eventBroker.subscribe(subscriber1);
eventBroker.subscribe(subscriber2);
@@ -86,7 +86,7 @@ describe('InMemoryEventBroker', () => {
})();
const errorSpy = jest.spyOn(logger, 'error');
const eventBroker = new InMemoryEventBroker(logger);
const eventBroker = new DefaultEventBroker(logger);
eventBroker.subscribe(subscriber1);
await eventBroker.publish({ topic, eventPayload: '1' });
plugins/events-backend/src/service/InMemoryEventBroker.ts → plugins/events-backend/src/service/DefaultEventBroker.ts
+4 -2
View File
@@ -22,12 +22,14 @@ import {
import { Logger } from 'winston';
/**
* In-memory event broker which will pass the event to all registered subscribers
* In process event broker which will pass the event to all registered subscribers
* interested in it.
* Events will not be persisted in any form.
*
* @public
*/
// TODO(pjungermann): add prom metrics? (see plugins/catalog-backend/src/util/metrics.ts, etc.)
export class InMemoryEventBroker implements EventBroker {
export class DefaultEventBroker implements EventBroker {
constructor(private readonly logger: Logger) {}
private readonly subscribers: {
plugins/events-backend/src/service/EventsBackend.ts
+2 -2
View File
@@ -20,7 +20,7 @@ import {
EventSubscriber,
} from '@backstage/plugin-events-node';
import { Logger } from 'winston';
import { InMemoryEventBroker } from './InMemoryEventBroker';
import { DefaultEventBroker } from './DefaultEventBroker';
/**
* A builder that helps wire up all component parts of the event management.
@@ -33,7 +33,7 @@ export class EventsBackend {
private subscribers: EventSubscriber[] = [];
constructor(logger: Logger) {
this.eventBroker = new InMemoryEventBroker(logger);
this.eventBroker = new DefaultEventBroker(logger);
}
setEventBroker(eventBroker: EventBroker): EventsBackend {
plugins/events-backend/src/service/EventsPlugin.ts
+2 -2
View File
@@ -29,7 +29,7 @@ import {
EventSubscriber,
HttpPostIngressOptions,
} from '@backstage/plugin-events-node';
import { InMemoryEventBroker } from './InMemoryEventBroker';
import { DefaultEventBroker } from './DefaultEventBroker';
import Router from 'express-promise-router';
import { HttpPostIngressEventPublisher } from './http';
@@ -113,7 +113,7 @@ export const eventsPlugin = createBackendPlugin({
router.use(eventsRouter);
const eventBroker =
extensionPoint.eventBroker ?? new InMemoryEventBroker(winstonLogger);
extensionPoint.eventBroker ?? new DefaultEventBroker(winstonLogger);
eventBroker.subscribe(extensionPoint.subscribers);
[extensionPoint.publishers, http]