Split to kafka backend plugin

plugins/kafka-backend/.eslintrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};

plugins/kafka-backend/config.d.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export interface Config {
+  kafka?: {
+    clientId: string;
+    brokers: string[];
+  };
+}

plugins/kafka-backend/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@backstage/plugin-kafka-backend",
+  "version": "0.1.1",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/kafka-backend"
+  },
+  "keywords": [
+    "backstage",
+    "kafka"
+  ],
+  "configSchema": "config.d.ts",
+  "scripts": {
+    "start": "backstage-cli backend:dev",
+    "build": "backstage-cli backend:build",
+    "lint": "backstage-cli lint",
+    "test": "backstage-cli test",
+    "prepack": "backstage-cli prepack",
+    "postpack": "backstage-cli postpack",
+    "clean": "backstage-cli clean"
+  },
+  "dependencies": {
+    "@backstage/backend-common": "^0.4.1",
+    "@backstage/catalog-model": "^0.6.0",
+    "@backstage/config": "^0.1.2",
+    "kafkajs": "^1.16.0-beta.6",
+    "winston": "^3.3.3"
+  },
+  "devDependencies": {
+    "@backstage/cli": "^0.4.2",
+    "supertest": "^4.0.2"
+  },
+  "files": [
+    "dist",
+    "schema.d.ts"
+  ]
+}

plugins/kafka-backend/src/index.test.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import '@backstage/backend-common';
+
+describe('test', () => {
+  it('unbreaks the test runner', () => {
+    expect(true).toBeTruthy();
+  });
+});

plugins/kafka-backend/src/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './service/router';
+export * from './types/types';

plugins/kafka-backend/src/run.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import yn from 'yn';
+import { getRootLogger } from '@backstage/backend-common';
+import { startStandaloneServer } from './service/standaloneServer';
+
+const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000;
+const enableCors = yn(process.env.PLUGIN_CORS, { default: false });
+const logger = getRootLogger();
+
+startStandaloneServer({ port, enableCors, logger }).catch(err => {
+  logger.error(err);
+  process.exit(1);
+});
+
+process.on('SIGINT', () => {
+  logger.info('CTRL+C pressed; exiting.');
+  process.exit(0);
+});

plugins/kafka-backend/src/service/KafkaApi.ts

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Kafka, SeekEntry } from 'kafkajs';
+import { Logger } from 'winston';
+
+export type PartitionOffset = {
+  id: number;
+  offset: string;
+};
+
+export type TopicOffset = {
+  topic: string;
+  partitions: PartitionOffset[];
+};
+
+export class KafkaApi {
+  private readonly kafka: Kafka;
+  private readonly logger: Logger;
+
+  constructor(clientId: string, brokers: string[], logger: Logger) {
+    logger.info(
+      `creating kafka client with clientId=${clientId} and brokers=${brokers}`,
+    );
+
+    this.kafka = new Kafka({ clientId, brokers });
+    this.logger = logger;
+  }
+
+  async fetchTopicOffsets(topic: string): Promise<Array<PartitionOffset>> {
+    this.logger.info(`fetching topic offsets for ${topic}`);
+
+    const admin = this.kafka.admin();
+    await admin.connect();
+
+    try {
+      return KafkaApi.toPartitionOffsets(await admin.fetchTopicOffsets(topic));
+    } finally {
+      await admin.disconnect();
+    }
+  }
+
+  async fetchGroupOffsets(groupId: string): Promise<Array<TopicOffset>> {
+    this.logger.info(`fetching consumer group offsets for ${groupId}`);
+
+    const admin = this.kafka.admin();
+    await admin.connect();
+
+    try {
+      let groupOffsets = await admin.fetchOffsets({ groupId });
+
+      return groupOffsets.map(topicOffset => ({
+        topic: topicOffset.topic,
+        partitions: KafkaApi.toPartitionOffsets(topicOffset.partitions),
+      }));
+    } finally {
+      await admin.disconnect();
+    }
+  }
+
+  private static toPartitionOffsets(
+    result: Array<SeekEntry>,
+  ): Array<PartitionOffset> {
+    return result.map(seekEntry => ({
+      id: seekEntry.partition,
+      offset: seekEntry.offset,
+    }));
+  }
+}

plugins/kafka-backend/src/service/router.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import express from 'express';
+import Router from 'express-promise-router';
+import { Logger } from 'winston';
+import { Config } from '@backstage/config';
+import { KafkaApi } from './KafkaApi';
+
+export interface RouterOptions {
+  logger: Logger;
+  config: Config;
+}
+
+export const makeRouter = (
+  logger: Logger,
+  kafkaApi: KafkaApi,
+): express.Router => {
+  const router = Router();
+  router.use(express.json());
+
+  router.get('/topic/:topicId/offsets', async (req, res) => {
+    const topicId = req.params.topicId;
+    try {
+      const response = await kafkaApi.fetchTopicOffsets(topicId);
+      res.send(response);
+    } catch (e) {
+      logger.error(`action=fetchTopicOffsets topicId=${topicId}, error=${e}`);
+      res.status(500).send({ error: e.message });
+    }
+  });
+
+  router.get('/consumer/:consumerId/offsets', async (req, res) => {
+    const consumerId = req.params.consumerId;
+    try {
+      const response = await kafkaApi.fetchGroupOffsets(consumerId);
+      res.send(response);
+    } catch (e) {
+      logger.error(
+        `action=fetchGroupOffsets consumerId=${consumerId}, error=${e}`,
+      );
+      res.status(500).send({ error: e.message });
+    }
+  });
+
+  return router;
+};
+
+export async function createRouter(
+  options: RouterOptions,
+): Promise<express.Router> {
+  const logger = options.logger;
+
+  logger.info('Initializing Kafka backend');
+
+  const clientId = options.config.getString('kafka.clientId');
+  const brokers = options.config.getStringArray('kafka.brokers');
+
+  const kafkaApi = new KafkaApi(clientId, brokers, logger);
+
+  return makeRouter(logger, kafkaApi);
+}

plugins/kafka-backend/src/service/standaloneApplication.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  errorHandler,
+  notFoundHandler,
+  requestLoggingHandler,
+} from '@backstage/backend-common';
+import compression from 'compression';
+import cors from 'cors';
+import express from 'express';
+import helmet from 'helmet';
+import { Logger } from 'winston';
+import { createRouter } from './router';
+import { ConfigReader } from '@backstage/config';
+
+export interface ApplicationOptions {
+  enableCors: boolean;
+  logger: Logger;
+}
+
+export async function createStandaloneApplication(
+  options: ApplicationOptions,
+): Promise<express.Application> {
+  const { enableCors, logger } = options;
+  const config = new ConfigReader({});
+  const app = express();
+
+  app.use(helmet());
+  if (enableCors) {
+    app.use(cors());
+  }
+  app.use(compression());
+  app.use(express.json());
+  app.use(requestLoggingHandler());
+  app.use('/', await createRouter({ logger, config }));
+  app.use(notFoundHandler());
+  app.use(errorHandler());
+
+  return app;
+}

plugins/kafka-backend/src/service/standaloneServer.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Server } from 'http';
+import { Logger } from 'winston';
+import { createStandaloneApplication } from './standaloneApplication';
+
+export interface ServerOptions {
+  port: number;
+  enableCors: boolean;
+  logger: Logger;
+}
+
+export async function startStandaloneServer(
+  options: ServerOptions,
+): Promise<Server> {
+  const logger = options.logger.child({ service: 'kafka-backend' });
+
+  logger.debug('Creating application...');
+  const app = await createStandaloneApplication({
+    enableCors: options.enableCors,
+    logger,
+  });
+
+  logger.debug('Starting application server...');
+  return await new Promise((resolve, reject) => {
+    const server = app.listen(options.port, (err?: Error) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+
+      logger.info(`Listening on port ${options.port}`);
+      resolve(server);
+    });
+  });
+}

plugins/kafka-backend/src/setupTests.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export {};

plugins/kafka-backend/src/types/types.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TopicOffset, PartitionOffset } from '../service/KafkaApi';
+
+export type TopicOffsetsResponse = Array<PartitionOffset>;
+export type ConsumerGroupOffsetsResponse = Array<TopicOffset>;