create stack-overflow backend plugin and move collator there

Signed-off-by: Emma Indal <emma.indahl@gmail.com>
2022-03-04 12:07:14 +01:00
parent fc67e62ac2
commit f80f3a7bc9
9 changed files with 201 additions and 96 deletions
@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};
@@ -0,0 +1,43 @@
+# Stack Overflow
+
+A plugin that provides stack overflow specific functionality that can be used in different ways (e.g. for search) to compose your Backstage App.
+
+## Getting started
+
+Before we begin, make sure:
+
+- You have created your own standalone Backstage app using @backstage/create-app and not using a fork of the backstage repository. If you haven't setup Backstage already, start [here](https://backstage.io/docs/getting-started/).
+
+To use any of the functionality this plugin provides, you need to start by configuring your App with the following config:
+
+```yaml
+stackoverflow:
+  baseUrl: https://api.stackexchange.com/2.2 # alternative: your internal stack overflow instance
+```
+
+## Areas of Responsibility
+
+This stack overflow backend plugin is primarily responsible for the following:
+
+- Provides a `StackOverflowQuestionsCollatorFactory`, which can be used in the search backend to index stack overflow questions to your Backstage Search.
+
+### Index Stack Overflow Questions to search
+
+Before you are able to start index stack overflow questions to search, you need to go through the [search getting started guide](https://backstage.io/docs/features/search/getting-started).
+
+When you have your `packages/backend/src/plugins/search.ts` file ready to make modifications, add the following code snippet to add the `StackOverflowQuestionsCollatorFactory`. Note that you can modify the `requestParams`.
+
+> Note: if your baseUrl is set to the external stack overflow api `https://api.stackexchange.com/2.2`, you can find optional and required params under the official API documentation under [`Usage of /questions GET`](https://api.stackexchange.com/docs/questions)
+
+```ts
+indexBuilder.addCollator({
+  defaultRefreshIntervalSeconds: 600,
+  factory: StackOverflowQuestionsCollatorFactory.fromConfig(config, {
+    requestParams: {
+      tagged: ['backstage'],
+      site: 'stackoverflow',
+      pagesize: 100,
+    },
+  }),
+});
+```
@@ -0,0 +1,45 @@
+{
+  "name": "@backstage/plugin-stack-overflow-backend",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/stack-overflow-backend"
+  },
+  "keywords": [
+    "backstage",
+    "stack-overflow"
+  ],
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean"
+  },
+  "dependencies": {
+    "@backstage/search-common": "^0.2.4",
+    "@backstage/config": "^0.1.15",
+    "qs": "^6.10.3",
+    "cross-fetch": "^3.1.5",
+    "winston": "^3.6.0"
+  },
+  "files": [
+    "dist"
+  ]
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { StackOverflowQuestionsCollatorFactory } from './search';
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  IndexableDocument,
+  DocumentCollatorFactory,
+} from '@backstage/search-common';
+import { Config } from '@backstage/config';
+import { Readable } from 'stream';
+import fetch from 'cross-fetch';
+import qs from 'qs';
+import { Logger } from 'winston';
+
+interface StackOverflowDocument extends IndexableDocument {
+  answers: number;
+  tags: string[];
+}
+
+type StackOverflowQuestionsRequestParams = {
+  [key: string]: string | string[] | number;
+};
+
+type StackOverflowQuestionsCollatorFactoryOptions = {
+  baseUrl?: string;
+  requestParams: StackOverflowQuestionsRequestParams;
+  logger: Logger;
+};
+
+export class StackOverflowQuestionsCollatorFactory
+  implements DocumentCollatorFactory
+{
+  protected requestParams: StackOverflowQuestionsRequestParams;
+  private readonly baseUrl: string | undefined;
+  private readonly logger: Logger;
+  public readonly type: string = 'stack-overflow';
+
+  private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) {
+    this.baseUrl = options.baseUrl;
+    this.requestParams = options.requestParams;
+    this.logger = options.logger;
+  }
+
+  static fromConfig(
+    config: Config,
+    options: StackOverflowQuestionsCollatorFactoryOptions,
+  ) {
+    const baseUrl = config.getString('stackoverflow.baseUrl');
+    return new StackOverflowQuestionsCollatorFactory({ ...options, baseUrl });
+  }
+
+  async getCollator() {
+    return Readable.from(this.execute());
+  }
+
+  async *execute(): AsyncGenerator<StackOverflowDocument> {
+    if (!this.baseUrl) {
+      this.logger.debug(
+        `No stackoverflow.baseUrl configured in your app-config.yaml`,
+      );
+    }
+    const params = this.requestParams
+      ? `?${qs.stringify(this.requestParams, { arrayFormat: 'comma' })}`
+      : '';
+    const res = await fetch(`${this.baseUrl}/questions${params}`);
+    const data = await res.json();
+
+    for (const question of data.items) {
+      yield {
+        title: question.title,
+        location: question.link,
+        text: question.owner.display_name,
+        tags: question.tags,
+        answers: question.answer_count,
+      };
+    }
+  }
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { StackOverflowQuestionsCollatorFactory } from './StackOverflowQuestionsCollatorFactory';