create stack-overflow backend plugin and move collator there

Signed-off-by: Emma Indal <emma.indahl@gmail.com>
2022-03-04 12:07:14 +01:00
parent fc67e62ac2
commit f80f3a7bc9
9 changed files with 201 additions and 96 deletions
@@ -0,0 +1,3 @@
+module.exports = {
+  extends: [require.resolve('@backstage/cli/config/eslint.backend')],
+};
@@ -0,0 +1,43 @@
+# Stack Overflow
+
+A plugin that provides stack overflow specific functionality that can be used in different ways (e.g. for search) to compose your Backstage App.
+
+## Getting started
+
+Before we begin, make sure:
+
+- You have created your own standalone Backstage app using @backstage/create-app and not using a fork of the backstage repository. If you haven't setup Backstage already, start [here](https://backstage.io/docs/getting-started/).
+
+To use any of the functionality this plugin provides, you need to start by configuring your App with the following config:
+
+```yaml
+stackoverflow:
+  baseUrl: https://api.stackexchange.com/2.2 # alternative: your internal stack overflow instance
+```
+
+## Areas of Responsibility
+
+This stack overflow backend plugin is primarily responsible for the following:
+
+- Provides a `StackOverflowQuestionsCollatorFactory`, which can be used in the search backend to index stack overflow questions to your Backstage Search.
+
+### Index Stack Overflow Questions to search
+
+Before you are able to start index stack overflow questions to search, you need to go through the [search getting started guide](https://backstage.io/docs/features/search/getting-started).
+
+When you have your `packages/backend/src/plugins/search.ts` file ready to make modifications, add the following code snippet to add the `StackOverflowQuestionsCollatorFactory`. Note that you can modify the `requestParams`.
+
+> Note: if your baseUrl is set to the external stack overflow api `https://api.stackexchange.com/2.2`, you can find optional and required params under the official API documentation under [`Usage of /questions GET`](https://api.stackexchange.com/docs/questions)
+
+```ts
+indexBuilder.addCollator({
+  defaultRefreshIntervalSeconds: 600,
+  factory: StackOverflowQuestionsCollatorFactory.fromConfig(config, {
+    requestParams: {
+      tagged: ['backstage'],
+      site: 'stackoverflow',
+      pagesize: 100,
+    },
+  }),
+});
+```
@@ -0,0 +1,45 @@
+{
+  "name": "@backstage/plugin-stack-overflow-backend",
+  "version": "0.0.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin"
+  },
+  "homepage": "https://backstage.io",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/stack-overflow-backend"
+  },
+  "keywords": [
+    "backstage",
+    "stack-overflow"
+  ],
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "clean": "backstage-cli package clean"
+  },
+  "dependencies": {
+    "@backstage/search-common": "^0.2.4",
+    "@backstage/config": "^0.1.15",
+    "qs": "^6.10.3",
+    "cross-fetch": "^3.1.5",
+    "winston": "^3.6.0"
+  },
+  "files": [
+    "dist"
+  ]
+}
@@ -14,4 +14,4 @@
 * limitations under the License.
 */

-export { StackOverflowQuestionsCollator } from './StackOverflowQuestionsCollator';
+export { StackOverflowQuestionsCollatorFactory } from './search';
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  IndexableDocument,
+  DocumentCollatorFactory,
+} from '@backstage/search-common';
+import { Config } from '@backstage/config';
+import { Readable } from 'stream';
+import fetch from 'cross-fetch';
+import qs from 'qs';
+import { Logger } from 'winston';
+
+interface StackOverflowDocument extends IndexableDocument {
+  answers: number;
+  tags: string[];
+}
+
+type StackOverflowQuestionsRequestParams = {
+  [key: string]: string | string[] | number;
+};
+
+type StackOverflowQuestionsCollatorFactoryOptions = {
+  baseUrl?: string;
+  requestParams: StackOverflowQuestionsRequestParams;
+  logger: Logger;
+};
+
+export class StackOverflowQuestionsCollatorFactory
+  implements DocumentCollatorFactory
+{
+  protected requestParams: StackOverflowQuestionsRequestParams;
+  private readonly baseUrl: string | undefined;
+  private readonly logger: Logger;
+  public readonly type: string = 'stack-overflow';
+
+  private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) {
+    this.baseUrl = options.baseUrl;
+    this.requestParams = options.requestParams;
+    this.logger = options.logger;
+  }
+
+  static fromConfig(
+    config: Config,
+    options: StackOverflowQuestionsCollatorFactoryOptions,
+  ) {
+    const baseUrl = config.getString('stackoverflow.baseUrl');
+    return new StackOverflowQuestionsCollatorFactory({ ...options, baseUrl });
+  }
+
+  async getCollator() {
+    return Readable.from(this.execute());
+  }
+
+  async *execute(): AsyncGenerator<StackOverflowDocument> {
+    if (!this.baseUrl) {
+      this.logger.debug(
+        `No stackoverflow.baseUrl configured in your app-config.yaml`,
+      );
+    }
+    const params = this.requestParams
+      ? `?${qs.stringify(this.requestParams, { arrayFormat: 'comma' })}`
+      : '';
+    const res = await fetch(`${this.baseUrl}/questions${params}`);
+    const data = await res.json();
+
+    for (const question of data.items) {
+      yield {
+        title: question.title,
+        location: question.link,
+        text: question.owner.display_name,
+        tags: question.tags,
+        answers: question.answer_count,
+      };
+    }
+  }
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { StackOverflowQuestionsCollatorFactory } from './StackOverflowQuestionsCollatorFactory';
@@ -17,30 +17,9 @@ stackoverflow:

 ## Areas of Responsibility

-This search plugin is primarily responsible for the following:
+This stack overflow frontend plugin is primarily responsible for the following:

 - Exposing various stack-overflow related components like `<StackOverflowSearchResultListItem />` which can be used for composing the search page, and `<HomePageStackOverflowQuestions/>` which can be used for composing the homepage.
- Provides a `StackOverflowQuestionsCollator`, which can be used in the search backend to index stack overflow questions to your Backstage Search.
-
-### Index Stack Overflow Questions to search
-
-Before you are able to start index stack overflow questions to search, you need to go through the [search getting started guide](https://backstage.io/docs/features/search/getting-started).
-
-When you have your `packages/backend/src/plugins/search.ts` file ready to make modifications, add the following code snippet to add the `StackOverflowQuestionsCollator`. Note that you can modify the `requestParams`.
-
-```ts
-indexBuilder.addCollator({
-  defaultRefreshIntervalSeconds: 60,
-  collator: new StackOverflowQuestionsCollator({
-    config,
-    requestParams: {
-      tagged: ['backstage'],
-      site: 'stackoverflow',
-      pagesize: 100,
-    },
-  }),
-});
-```

 #### Use specific search result list item for Stack Overflow Question

@@ -13,9 +13,9 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
+
 export {
  stackOverflowPlugin,
  StackOverflowSearchResultListItem,
  HomePageStackOverflowQuestions,
 } from './plugin';
-export { StackOverflowQuestionsCollator } from './search/StackOverflowQuestionsCollator';
@@ -1,72 +0,0 @@
-/*
- * Copyright 2022 The Backstage Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { IndexableDocument, DocumentCollator } from '@backstage/search-common';
-import { Config } from '@backstage/config';
-import fetch from 'cross-fetch';
-import qs from 'qs';
-import {
-  StackOverflowQuestion,
-  StackOverflowQuestionsRequestParams,
-} from '../../types';
-
-interface StackOverflowDocument extends IndexableDocument {
-  answers: number;
-  tags: string[];
-}
-
-export class StackOverflowQuestionsCollator implements DocumentCollator {
-  protected baseUrl: string;
-  protected requestParams: StackOverflowQuestionsRequestParams;
-  public readonly type: string = 'stack-overflow';
-
-  constructor({
-    config,
-    requestParams,
-  }: {
-    config: Config;
-    requestParams: StackOverflowQuestionsRequestParams;
-  }) {
-    this.baseUrl = config.getString('stackoverflow.baseUrl');
-    this.requestParams = requestParams;
-  }
-
-  async execute() {
-    const params = this.requestParams
-      ? `?${qs.stringify(this.requestParams)}`
-      : '';
-    const res = await fetch(`${this.baseUrl}/questions${params}`);
-
-    const data = await res.json();
-    return data.items.map(
-      ({
-        title,
-        link,
-        owner: { display_name },
-        tags,
-        answer_count,
-      }: StackOverflowQuestion): StackOverflowDocument => {
-        return {
-          title: title,
-          location: link,
-          text: display_name,
-          tags: tags,
-          answers: answer_count,
-        };
-      },
-    );
-  }
-}