From 7471e0b9afd42278bdefba8d67ec2770566ac17e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Feb 2021 15:21:55 +0100 Subject: [PATCH 01/52] Initial backend search-indexer plugin (backstage-cli generated, minus route/service stuff). Signed-off-by: Eric Peterson --- .github/CODEOWNERS | 1 + plugins/search-indexer-backend/.eslintrc.js | 3 ++ plugins/search-indexer-backend/README.md | 5 +++ plugins/search-indexer-backend/package.json | 40 +++++++++++++++++++ plugins/search-indexer-backend/src/index.ts | 17 ++++++++ .../search-indexer-backend/src/setupTests.ts | 17 ++++++++ 6 files changed, 83 insertions(+) create mode 100644 plugins/search-indexer-backend/.eslintrc.js create mode 100644 plugins/search-indexer-backend/README.md create mode 100644 plugins/search-indexer-backend/package.json create mode 100644 plugins/search-indexer-backend/src/index.ts create mode 100644 plugins/search-indexer-backend/src/setupTests.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 371fae6e6b..d32072f2c0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,7 @@ /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core +/plugins/search-indexer-backend @backstage/techdocs-core /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core diff --git a/plugins/search-indexer-backend/.eslintrc.js b/plugins/search-indexer-backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-indexer-backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-indexer-backend/README.md b/plugins/search-indexer-backend/README.md new file mode 100644 index 0000000000..bac23fffd0 --- /dev/null +++ b/plugins/search-indexer-backend/README.md @@ -0,0 +1,5 @@ +# search-indexer + +Welcome to the search-indexer backend plugin! + +_This plugin was created through the Backstage CLI_ diff --git a/plugins/search-indexer-backend/package.json b/plugins/search-indexer-backend/package.json new file mode 100644 index 0000000000..f853abc632 --- /dev/null +++ b/plugins/search-indexer-backend/package.json @@ -0,0 +1,40 @@ +{ + "name": "@backstage/plugin-search-indexer-backend", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.5.3", + "@backstage/config": "^0.1.2", + "@types/express": "^4.17.6", + "express": "^4.17.1", + "express-promise-router": "^3.0.3", + "winston": "^3.2.1", + "cross-fetch": "^3.0.6", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.6.0", + "@types/supertest": "^2.0.8", + "supertest": "^4.0.2", + "msw": "^0.21.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts new file mode 100644 index 0000000000..4e230aca20 --- /dev/null +++ b/plugins/search-indexer-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/search-indexer-backend/src/setupTests.ts b/plugins/search-indexer-backend/src/setupTests.ts new file mode 100644 index 0000000000..4e230aca20 --- /dev/null +++ b/plugins/search-indexer-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; From 64e2b34e603e03948a04a900891b6c2a0879f080 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Feb 2021 17:24:06 +0100 Subject: [PATCH 02/52] Initial proposed types/interfaces. Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/types.ts | 97 +++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 plugins/search-indexer-backend/src/types.ts diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-indexer-backend/src/types.ts new file mode 100644 index 0000000000..90529e2101 --- /dev/null +++ b/plugins/search-indexer-backend/src/types.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Base properties that all indexed documents must include, as well as some + * common properties that documents are encouraged to use where appropriate. + */ +export interface IndexableDocument { + /** + * The primary name of the document (e.g. name, title, identifier, etc). + */ + title: string; + + /** + * Free-form text of the document (e.g. description, content, etc). + */ + text: string; + + /** + * The relative or absolute URL of the document (target when a search result + * is clicked). + */ + location: string; + + /** + * The owner of the document (e.g. spec.owner on a catalog entity). + */ + owner?: string; + + /** + * The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). + */ + lifecycle?: string; +} + +/** + * Signature for the callback function that implementors must register to have + * their documents indexed. + */ +export type IndexableDocumentCollator = () => Promise; + +/** + * Signature for the callback function that implementors must register to + * decorate existing documents with additional metadata. + */ +export type IndexableDocumentDecorator = ( + documents: IndexableDocument[], +) => Promise; + +/** + * Parameters required to register a collator. + */ +export interface RegisterCollatorParameters { + /** + * The type of document to be indexed (used to name indices, to configure refresh loop, etc). + */ + type: string; + + /** + * The default interval (in seconds) that the provided collator will be called (can be overridden in config). + */ + defaultRefreshIntervalSeconds: number; + + /** + * The collator function responsible for returning all documents of the given type. + */ + collator: IndexableDocumentCollator; +} + +/** + * Parameters required to register a decorator + */ +export interface RegisterDecoratorParameters { + /** + * The decorator function responsible for appending or modifying documents of the given type(s). + */ + decorator: IndexableDocumentDecorator; + + /** + * An optional array of document types that the given decorator should apply to. If none are provided, + * the decorator will be applied to all types. + */ + types?: string[]; +} From e11d48c1d3f8e255dbf0f33345f43325b83b6f0b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Feb 2021 17:25:13 +0100 Subject: [PATCH 03/52] Initial proposal on how to glue the types/interfaces together. Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/index.ts | 6 +- .../search-indexer-backend/src/registry.ts | 86 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 plugins/search-indexer-backend/src/registry.ts diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts index 4e230aca20..798ad8e1b0 100644 --- a/plugins/search-indexer-backend/src/index.ts +++ b/plugins/search-indexer-backend/src/index.ts @@ -14,4 +14,8 @@ * limitations under the License. */ -export {}; +import { Registry } from './registry'; + +const registry = Registry.getInstance(); +export const registerCollator = registry.addCollator; +export const registerDecorator = registry.addDecorator; diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-indexer-backend/src/registry.ts new file mode 100644 index 0000000000..23d39b12ff --- /dev/null +++ b/plugins/search-indexer-backend/src/registry.ts @@ -0,0 +1,86 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + IndexableDocumentCollator, + IndexableDocumentDecorator, + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; + +interface CollatorRegistryEntry { + collate: IndexableDocumentCollator; + refreshInterval: number; +} + +export class Registry { + private collators: Record; + private decorators: Record; + + private static instance: Registry; + + private constructor() { + this.collators = {}; + this.decorators = {}; + } + + static getInstance(): Registry { + if (!Registry.instance) { + Registry.instance = new Registry(); + } + return Registry.instance; + } + + addCollator({ + type, + collator, + defaultRefreshIntervalSeconds, + }: RegisterCollatorParameters): void { + this.collators[type] = { + refreshInterval: defaultRefreshIntervalSeconds, + collate: collator, + }; + } + + addDecorator({ + types = ['*'], + decorator, + }: RegisterDecoratorParameters): void { + types.forEach(type => { + if (this.decorators.hasOwnProperty(type)) { + this.decorators[type].push(decorator); + } else { + this.decorators[type] = [decorator]; + } + }); + } + + // @todo But like with coordination, timing, error handling, and what have you. + execute() { + Object.keys(this.collators).forEach(async type => { + const decorators: IndexableDocumentDecorator[] = ( + this.decorators['*'] || [] + ).concat(this.decorators[type] || []); + let documents = await this.collators[type].collate(); + + for (let i = 0; i < decorators.length; i++) { + documents = await decorators[i](documents); + } + + // @todo: push documents to a configured search engine. + }); + } +} From 2c58ccaeee6933dc3b3002a4de59f680cbfb7a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Mon, 15 Feb 2021 11:15:32 +0100 Subject: [PATCH 04/52] initial search-backend setup Signed-off-by: Eric Peterson --- plugins/search-backend/.eslintrc.js | 3 + plugins/search-backend/README.md | 13 ++++ plugins/search-backend/package.json | 40 ++++++++++++ plugins/search-backend/src/index.ts | 17 +++++ plugins/search-backend/src/run.ts | 33 ++++++++++ .../search-backend/src/service/router.test.ts | 45 ++++++++++++++ plugins/search-backend/src/service/router.ts | 47 ++++++++++++++ .../src/service/standaloneServer.ts | 62 +++++++++++++++++++ plugins/search-backend/src/setupTests.ts | 17 +++++ 9 files changed, 277 insertions(+) create mode 100644 plugins/search-backend/.eslintrc.js create mode 100644 plugins/search-backend/README.md create mode 100644 plugins/search-backend/package.json create mode 100644 plugins/search-backend/src/index.ts create mode 100644 plugins/search-backend/src/run.ts create mode 100644 plugins/search-backend/src/service/router.test.ts create mode 100644 plugins/search-backend/src/service/router.ts create mode 100644 plugins/search-backend/src/service/standaloneServer.ts create mode 100644 plugins/search-backend/src/setupTests.ts diff --git a/plugins/search-backend/.eslintrc.js b/plugins/search-backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-backend/README.md b/plugins/search-backend/README.md new file mode 100644 index 0000000000..e539afec23 --- /dev/null +++ b/plugins/search-backend/README.md @@ -0,0 +1,13 @@ +# search + +Welcome to the search backend plugin! + +_This plugin was created through the Backstage CLI_ + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/search](http://localhost:3000/search). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json new file mode 100644 index 0000000000..fec85fb8aa --- /dev/null +++ b/plugins/search-backend/package.json @@ -0,0 +1,40 @@ +{ + "name": "@backstage/plugin-search-backend", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.5.3", + "@backstage/config": "^0.1.2", + "@types/express": "^4.17.6", + "express": "^4.17.1", + "express-promise-router": "^3.0.3", + "winston": "^3.2.1", + "cross-fetch": "^3.0.6", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.6.0", + "@types/supertest": "^2.0.8", + "supertest": "^4.0.2", + "msw": "^0.21.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts new file mode 100644 index 0000000000..7612c392a2 --- /dev/null +++ b/plugins/search-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; diff --git a/plugins/search-backend/src/run.ts b/plugins/search-backend/src/run.ts new file mode 100644 index 0000000000..b96989e4b8 --- /dev/null +++ b/plugins/search-backend/src/run.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getRootLogger } from '@backstage/backend-common'; +import yn from 'yn'; +import { startStandaloneServer } from './service/standaloneServer'; + +const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000; +const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); +const logger = getRootLogger(); + +startStandaloneServer({ port, enableCors, logger }).catch(err => { + logger.error(err); + process.exit(1); +}); + +process.on('SIGINT', () => { + logger.info('CTRL+C pressed; exiting.'); + process.exit(0); +}); diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts new file mode 100644 index 0000000000..0aaeafa379 --- /dev/null +++ b/plugins/search-backend/src/service/router.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import request from 'supertest'; + +import { createRouter } from './router'; + +describe('createRouter', () => { + let app: express.Express; + + beforeAll(async () => { + const router = await createRouter({ + logger: getVoidLogger(), + }); + app = express().use(router); + }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('GET /health', () => { + it('returns ok', async () => { + const response = await request(app).get('/health'); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ status: 'ok' }); + }); + }); +}); diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts new file mode 100644 index 0000000000..046d9d27b3 --- /dev/null +++ b/plugins/search-backend/src/service/router.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import Router from 'express-promise-router'; +import { Logger } from 'winston'; + +type RouterOptions = { + logger: Logger; +}; + +export async function createRouter({ + logger, +}: RouterOptions): Promise { + const router = Router(); + + router.get('/query', async (req, res) => { + // Transform req.params into search engine specific query. + const { term } = req.query; + + try { + // Query search engine. + // Transform results into frontend-readable result + res.send({ + term, + }); + } catch (err) { + logger.error(`There was a problem performing the search query.`); + res.status(500).send(`There was a problem performing the search query.`); + } + }); + + return router; +} diff --git a/plugins/search-backend/src/service/standaloneServer.ts b/plugins/search-backend/src/service/standaloneServer.ts new file mode 100644 index 0000000000..e1212ee5a7 --- /dev/null +++ b/plugins/search-backend/src/service/standaloneServer.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceBuilder } from '@backstage/backend-common'; +import { Server } from 'http'; +import { Logger } from 'winston'; +import { createRouter } from './router'; + +export interface ServerOptions { + port: number; + enableCors: boolean; + logger: Logger; +} + +export async function startStandaloneServer( + options: ServerOptions, +): Promise { + const logger = options.logger.child({ service: 'search-backend' }); + logger.debug('Starting application server...'); + const router = await createRouter({ + logger, + }); + + const service = createServiceBuilder(module) + .enableCors({ origin: 'http://localhost:3000' }) + .addRouter('/search', router); + + return await service.start().catch(err => { + logger.error(err); + process.exit(1); + }); +} + +module.hot?.accept(); diff --git a/plugins/search-backend/src/setupTests.ts b/plugins/search-backend/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/plugins/search-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; From 7f4d4022268752a4a218d5b465a4872e134c75ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Mon, 15 Feb 2021 11:51:09 +0100 Subject: [PATCH 05/52] add types Signed-off-by: Eric Peterson --- plugins/search-backend/src/service/router.ts | 42 +++++++++++++------- plugins/search-backend/src/types.ts | 26 ++++++++++++ 2 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 plugins/search-backend/src/types.ts diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 046d9d27b3..02bc553bb9 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -14,9 +14,11 @@ * limitations under the License. */ +import { errorHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; +import { SearchQuery, SearchResultSet } from '../types'; type RouterOptions = { logger: Logger; @@ -27,21 +29,33 @@ export async function createRouter({ }: RouterOptions): Promise { const router = Router(); - router.get('/query', async (req, res) => { - // Transform req.params into search engine specific query. - const { term } = req.query; + router.get( + '/query', + async ( + req: express.Request, + res: express.Response, + ) => { + // @todo Actually transform req.params into search engine specific query. + const { term, filters = {}, pageIndex = 0, pageSize = 30 } = req.query; + logger.info( + `Search requested received: ${term}, ${JSON.stringify( + filters, + )}, ${pageIndex}, ${pageSize}`, + ); - try { - // Query search engine. - // Transform results into frontend-readable result - res.send({ - term, - }); - } catch (err) { - logger.error(`There was a problem performing the search query.`); - res.status(500).send(`There was a problem performing the search query.`); - } - }); + try { + // @todo Actually query search engine. + // @todo And actually transform results into frontend-readable result + res.send({ + results: [], + }); + } catch (err) { + logger.error(`There was a problem performing the search query.`); + throw new Error(`There was a problem performing the search query.`); + } + }, + ); + router.use(errorHandler()); return router; } diff --git a/plugins/search-backend/src/types.ts b/plugins/search-backend/src/types.ts new file mode 100644 index 0000000000..79b72ad80a --- /dev/null +++ b/plugins/search-backend/src/types.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface SearchQuery { + term: string; + filters?: Record; + pageIndex?: number; + pageSize?: number; +} + +export interface SearchResultSet { + results: any[]; // @todo Use IndexableDocument from search-indexer-backend? +} From a93d7b7f57334db9e3fc082551026554d36173a7 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 12 Feb 2021 17:54:04 +0100 Subject: [PATCH 06/52] Be sure to export the IndexableDocument type. Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts index 798ad8e1b0..8988e8f007 100644 --- a/plugins/search-indexer-backend/src/index.ts +++ b/plugins/search-indexer-backend/src/index.ts @@ -19,3 +19,5 @@ import { Registry } from './registry'; const registry = Registry.getInstance(); export const registerCollator = registry.addCollator; export const registerDecorator = registry.addDecorator; + +export type { IndexableDocument } from './types'; From c22f1c33db43ba2da4ea427c4a5c27d156fafa19 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 13:24:35 +0100 Subject: [PATCH 07/52] Update code owners for the various search backend plugins. Signed-off-by: Eric Peterson --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d32072f2c0..4e21603c63 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,7 +11,7 @@ /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core -/plugins/search-indexer-backend @backstage/techdocs-core +/plugins/search-* @backstage/techdocs-core /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core From 04b492d4043e647b2a7bfa6cd25a57fb5583b7fe Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 13:25:05 +0100 Subject: [PATCH 08/52] Use IndexableDocument in search result set. Signed-off-by: Eric Peterson --- plugins/search-backend/package.json | 1 + plugins/search-backend/src/types.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index fec85fb8aa..a4e2b566b8 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -21,6 +21,7 @@ "dependencies": { "@backstage/backend-common": "^0.5.3", "@backstage/config": "^0.1.2", + "@backstage/plugin-search-indexer-backend": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/search-backend/src/types.ts b/plugins/search-backend/src/types.ts index 79b72ad80a..f7e6acf5ee 100644 --- a/plugins/search-backend/src/types.ts +++ b/plugins/search-backend/src/types.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; + export interface SearchQuery { term: string; filters?: Record; @@ -22,5 +24,5 @@ export interface SearchQuery { } export interface SearchResultSet { - results: any[]; // @todo Use IndexableDocument from search-indexer-backend? + results: IndexableDocument[]; } From e0854f10d5fd61060bfe057d00d98a8f834e80c1 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 14:03:21 +0100 Subject: [PATCH 09/52] Set up an unused method to illustrate type usage / communication between search frontend and backend Signed-off-by: Eric Peterson --- plugins/search-backend/src/index.ts | 1 + .../src/service/standaloneServer.ts | 17 +---------------- plugins/search/package.json | 1 + plugins/search/src/apis.ts | 19 ++++++++++++++++++- .../components/SearchResult/SearchResult.tsx | 4 +++- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 7612c392a2..90a67d7605 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -15,3 +15,4 @@ */ export * from './service/router'; +export * from './types'; diff --git a/plugins/search-backend/src/service/standaloneServer.ts b/plugins/search-backend/src/service/standaloneServer.ts index e1212ee5a7..4efc02de6f 100644 --- a/plugins/search-backend/src/service/standaloneServer.ts +++ b/plugins/search-backend/src/service/standaloneServer.ts @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/* - * Copyright 2020 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; @@ -51,7 +36,7 @@ export async function startStandaloneServer( const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) - .addRouter('/search', router); + .addRouter('/api/search', router); return await service.start().catch(err => { logger.error(err); diff --git a/plugins/search/package.json b/plugins/search/package.json index 8003843d8d..6fa80cda54 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -32,6 +32,7 @@ "@backstage/core": "^0.7.2", "@backstage/catalog-model": "^0.7.3", "@backstage/plugin-catalog-react": "^0.1.2", + "@backstage/plugin-search-backend": "^0.1.1", "@backstage/theme": "^0.2.4", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 9d88227a44..65490c3c37 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -15,7 +15,10 @@ */ import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; +import { DiscoveryApi } from '@backstage/core'; import { CatalogApi } from '@backstage/plugin-catalog-react'; +import { SearchQuery, SearchResultSet } from '@backstage/plugin-search-backend'; +import qs from 'qs'; export type Result = { name: string; @@ -30,9 +33,11 @@ export type SearchResults = Array; class SearchApi { private catalogApi: CatalogApi; + private discoveryApi: DiscoveryApi; - constructor(catalogApi: CatalogApi) { + constructor(catalogApi: CatalogApi, discoveryApi: DiscoveryApi) { this.catalogApi = catalogApi; + this.discoveryApi = discoveryApi; } private async entities() { @@ -56,6 +61,18 @@ class SearchApi { public getSearchResult(): Promise { return this.entities(); } + + // @todo Productionalize as we implement search milestones. + public async _alphaPerformSearch( + query: SearchQuery, + ): Promise { + const queryString = qs.stringify(query); + const url = `${await this.discoveryApi.getBaseUrl( + 'search/query', + )}?${queryString}`; + const response = await fetch(url); + return response.json(); + } } export default SearchApi; diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index 7f4d31a6df..141b7684cd 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -14,6 +14,7 @@ * limitations under the License. */ import { + discoveryApiRef, EmptyState, Link, Progress, @@ -117,6 +118,7 @@ const TableHeader = ({ export const SearchResult = ({ searchQuery }: SearchResultProps) => { const catalogApi = useApi(catalogApiRef); + const discoveryApi = useApi(discoveryApiRef); const [showFilters, toggleFilters] = useState(false); const [selectedFilters, setSelectedFilters] = useState({ @@ -126,7 +128,7 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { const [filteredResults, setFilteredResults] = useState([]); - const searchApi = new SearchApi(catalogApi); + const searchApi = new SearchApi(catalogApi, discoveryApi); const { loading, error, value: results } = useAsync(() => { return searchApi.getSearchResult(); From b7b4cc319c78134a2474854756c31e798f9a8b9e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 16:07:34 +0100 Subject: [PATCH 10/52] Wire up search backend to app backend testbed. Signed-off-by: Eric Peterson --- packages/backend/package.json | 1 + packages/backend/src/index.ts | 3 +++ packages/backend/src/plugins/search.ts | 23 +++++++++++++++++++ .../src/service/standaloneServer.ts | 2 +- 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 packages/backend/src/plugins/search.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 98a69f6ff6..74a86598d3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -40,6 +40,7 @@ "@backstage/plugin-proxy-backend": "^0.2.6", "@backstage/plugin-rollbar-backend": "^0.1.8", "@backstage/plugin-scaffolder-backend": "^0.9.2", + "@backstage/plugin-search-backend": "^0.1.1", "@backstage/plugin-techdocs-backend": "^0.6.5", "@backstage/plugin-todo-backend": "^0.1.1", "@gitbeaker/node": "^28.0.2", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 23739596e8..c32c7e6895 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -42,6 +42,7 @@ import kafka from './plugins/kafka'; import rollbar from './plugins/rollbar'; import scaffolder from './plugins/scaffolder'; import proxy from './plugins/proxy'; +import search from './plugins/search'; import techdocs from './plugins/techdocs'; import todo from './plugins/todo'; import graphql from './plugins/graphql'; @@ -77,6 +78,7 @@ async function main() { const authEnv = useHotMemoize(module, () => createEnv('auth')); const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); + const searchEnv = useHotMemoize(module, () => createEnv('search')); const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const todoEnv = useHotMemoize(module, () => createEnv('todo')); const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); @@ -89,6 +91,7 @@ async function main() { apiRouter.use('/rollbar', await rollbar(rollbarEnv)); apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); apiRouter.use('/auth', await auth(authEnv)); + apiRouter.use('/search', await search(searchEnv)); apiRouter.use('/techdocs', await techdocs(techdocsEnv)); apiRouter.use('/todo', await todo(todoEnv)); apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts new file mode 100644 index 0000000000..07a784f654 --- /dev/null +++ b/packages/backend/src/plugins/search.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createRouter } from '@backstage/plugin-search-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ logger }: PluginEnvironment) { + return await createRouter({ + logger, + }); +} diff --git a/plugins/search-backend/src/service/standaloneServer.ts b/plugins/search-backend/src/service/standaloneServer.ts index 4efc02de6f..b0c53a6adc 100644 --- a/plugins/search-backend/src/service/standaloneServer.ts +++ b/plugins/search-backend/src/service/standaloneServer.ts @@ -36,7 +36,7 @@ export async function startStandaloneServer( const service = createServiceBuilder(module) .enableCors({ origin: 'http://localhost:3000' }) - .addRouter('/api/search', router); + .addRouter('/search', router); return await service.start().catch(err => { logger.error(err); From 199f66a565024426a36810ba1210bc8b12dd9863 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 16:08:47 +0100 Subject: [PATCH 11/52] Add feature flag + app-based search page. Signed-off-by: Eric Peterson --- packages/app/src/App.tsx | 6 ++ .../app/src/components/search/SearchPage.tsx | 67 +++++++++++++++++++ plugins/search/src/plugin.ts | 3 +- 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 packages/app/src/components/search/SearchPage.tsx diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 7acde03953..99a6f8096f 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -17,6 +17,7 @@ import { AlertDisplay, createApp, + LocalStorageFeatureFlags, FlatRoutes, OAuthRequestDialog, SignInPage, @@ -50,6 +51,7 @@ import { Navigate, Route } from 'react-router'; import { apis } from './apis'; import { EntityPage } from './components/catalog/EntityPage'; import Root from './components/Root'; +import { SearchPage } from './components/search/SearchPage'; import { providers } from './identityProviders'; import * as plugins from './plugins'; @@ -87,6 +89,7 @@ const app = createApp({ const AppProvider = app.getProvider(); const AppRouter = app.getRouter(); +const featureFlags = new LocalStorageFeatureFlags(); const routes = ( @@ -100,6 +103,9 @@ const routes = ( } /> } /> + {featureFlags.isActive('use-search-platform') && ( + } /> + )} } /> } /> { + const [queryString, setQueryString] = useQueryParamState('query'); + const [searchQuery, setSearchQuery] = useState(queryString ?? ''); + + const handleSearch = (event: React.ChangeEvent) => { + event.preventDefault(); + setSearchQuery(event.target.value); + }; + + useEffect(() => setSearchQuery(queryString ?? ''), [queryString]); + + useDebounce( + () => { + setQueryString(searchQuery); + }, + 200, + [searchQuery], + ); + + const handleClearSearchBar = () => { + setSearchQuery(''); + }; + + return ( + +
+ + + + + + + + + + + + ); +}; diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index 37503b7751..81d4489655 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -27,8 +27,9 @@ export const rootRouteRef = createRouteRef({ export const searchPlugin = createPlugin({ id: 'search', - register({ router }) { + register({ router, featureFlags }) { router.addRoute(rootRouteRef, SearchPageComponent); + featureFlags.register('use-search-platform'); }, routes: { root: rootRouteRef, From 3e2e3ffe4f220f6895564b9c5a47536f3ac986d9 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 16:09:33 +0100 Subject: [PATCH 12/52] Document development process for search platform. Signed-off-by: Eric Peterson --- plugins/search/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/search/README.md b/plugins/search/README.md index fb4b047811..a95f6a21ed 100644 --- a/plugins/search/README.md +++ b/plugins/search/README.md @@ -2,8 +2,22 @@ **This plugin is still under development.** -You can follow the progress under the Global search in Backstage [milestone](https://github.com/backstage/backstage/milestone/21) or reach out to us in the #search Discord channel. +You can follow the progress and contribute at the Backstage [Search Project Board](https://github.com/backstage/backstage/projects/6) or reach out to us in the #search Discord channel. ## Getting started Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin. + +### Working on the search platform + +Currently, the search experience available in Backstage is 100% on the front-end, and only looks at the Software Catalog. We are actively developing [a more complete search platform](https://backstage.io/docs/features/search/search-overview), which will replace the current experience when ready. + +In order to work on the new search platform, you will need to be aware of the following: + +- **The use-search-platform feature flag**: The new search platform will move the primary search page to the App-level, out of the search plugin. You'll need to enable this feature flag to work on the net platform. +- **App SearchPage Component**: You'll find a new search page under `/packages/app/src/components/search`. When ready, we'll add an equivalent page to the `@backstage/create-app` template. +- **Backend**: Don't forget, a lot of functionality will be made available in backend plugins: + - `@backstage/plugin-search-indexer-backend`, which is responsible for the search index management + - `@backstage/plugin-search-backend`, which is responsible for query processing + +As you work, be sure not to break the existing, frontend-only search page. From 3e9d8427b6d36558cf4b9eecb394abab14eea591 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 17:36:45 +0100 Subject: [PATCH 13/52] Illustrate potential software catalog indexing. Signed-off-by: Eric Peterson --- packages/backend/package.json | 1 + packages/backend/src/index.ts | 2 + packages/backend/src/plugins/catalog.ts | 8 +++ plugins/catalog-backend/package.json | 1 + plugins/catalog-backend/src/index.ts | 1 + .../catalog-backend/src/search/Collator.ts | 51 +++++++++++++++++++ plugins/catalog-backend/src/search/index.ts | 17 +++++++ plugins/search-indexer-backend/src/index.ts | 18 +++++-- 8 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 plugins/catalog-backend/src/search/Collator.ts create mode 100644 plugins/catalog-backend/src/search/index.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 74a86598d3..4e999b1531 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -41,6 +41,7 @@ "@backstage/plugin-rollbar-backend": "^0.1.8", "@backstage/plugin-scaffolder-backend": "^0.9.2", "@backstage/plugin-search-backend": "^0.1.1", + "@backstage/plugin-search-indexer-backend": "^0.1.1", "@backstage/plugin-techdocs-backend": "^0.6.5", "@backstage/plugin-todo-backend": "^0.1.1", "@gitbeaker/node": "^28.0.2", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index c32c7e6895..fededf0f0a 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -110,6 +110,8 @@ async function main() { console.log(err); process.exit(1); }); + + // @todo Start Search Refresh Loop Here? } module.hot?.accept(); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index eaca42717c..86375203ab 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,8 +19,10 @@ import { CatalogBuilder, createRouter, runPeriodically, + SearchCollatorFactory, } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; +import { registerCollator } from '@backstage/plugin-search-indexer-backend'; import { PluginEnvironment } from '../types'; export default async function createPlugin( @@ -39,6 +41,12 @@ export default async function createPlugin( runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), ); + registerCollator({ + type: 'software-catalog', + defaultRefreshIntervalSeconds: 600, + collator: SearchCollatorFactory(entitiesCatalog), + }); + return await createRouter({ entitiesCatalog, locationsCatalog, diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index bad9e3c809..c99221b05f 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -35,6 +35,7 @@ "@backstage/config": "^0.1.4", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", + "@backstage/plugin-search-indexer-backend": "^0.1.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index f621930d10..51df909d05 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -17,5 +17,6 @@ export * from './catalog'; export * from './database'; export * from './ingestion'; +export * from './search'; export * from './service'; export * from './util'; diff --git a/plugins/catalog-backend/src/search/Collator.ts b/plugins/catalog-backend/src/search/Collator.ts new file mode 100644 index 0000000000..691e99eb3a --- /dev/null +++ b/plugins/catalog-backend/src/search/Collator.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + IndexableDocument, + IndexableDocumentCollator, +} from '@backstage/plugin-search-indexer-backend'; +import { EntitiesCatalog } from '../catalog'; +import { EntityFilters } from '../service/EntityFilters'; + +export interface CatalogEntityDocument extends IndexableDocument { + componentType: string; +} + +export const SearchCollatorFactory = ( + entitiesCatalog: EntitiesCatalog, +): IndexableDocumentCollator => { + return async (): Promise => { + const filter = EntityFilters.ofQuery({ kind: 'Component' }); + const entities = await entitiesCatalog.entities(filter); + return entities.map( + (entity): CatalogEntityDocument => { + return { + title: entity.metadata.name, + location: `/catalog/${ + entity.metadata.namespace || 'default' + }/component/${entity.metadata.name}`, + text: entity.metadata.description || '', + componentType: entity.spec?.type?.toString() || 'other', + ...(entity.spec?.owner && { owner: entity.spec.owner.toString() }), + ...(entity.metadata.lifecycle && { + lifecycle: entity.metadata.lifecycle.toString(), + }), + }; + }, + ); + }; +}; diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts new file mode 100644 index 0000000000..568fafaed5 --- /dev/null +++ b/plugins/catalog-backend/src/search/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './Collator'; diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts index 8988e8f007..13d1a298b7 100644 --- a/plugins/search-indexer-backend/src/index.ts +++ b/plugins/search-indexer-backend/src/index.ts @@ -15,9 +15,21 @@ */ import { Registry } from './registry'; +import { + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; const registry = Registry.getInstance(); -export const registerCollator = registry.addCollator; -export const registerDecorator = registry.addDecorator; +export const registerCollator = (params: RegisterCollatorParameters) => { + registry.addCollator(params); +}; +export const registerDecorator = (params: RegisterDecoratorParameters) => { + registry.addDecorator(params); +}; -export type { IndexableDocument } from './types'; +export type { + IndexableDocument, + IndexableDocumentCollator, + IndexableDocumentDecorator, +} from './types'; From ff91e9b72034f1355fe42f120eebf40a0a6d9766 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 17:38:53 +0100 Subject: [PATCH 14/52] Better formatting for "optional" Co-authored-by: Emma Indal Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-indexer-backend/src/types.ts index 90529e2101..c4db1ee565 100644 --- a/plugins/search-indexer-backend/src/types.ts +++ b/plugins/search-indexer-backend/src/types.ts @@ -90,7 +90,7 @@ export interface RegisterDecoratorParameters { decorator: IndexableDocumentDecorator; /** - * An optional array of document types that the given decorator should apply to. If none are provided, + * (Optional) An array of document types that the given decorator should apply to. If none are provided, * the decorator will be applied to all types. */ types?: string[]; From 77b2e143a5022b77d6da90f1830efc10e9079c7d Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 17:39:01 +0100 Subject: [PATCH 15/52] Better formatting for "optional" Co-authored-by: Emma Indal Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-indexer-backend/src/types.ts index c4db1ee565..f9c2c0a263 100644 --- a/plugins/search-indexer-backend/src/types.ts +++ b/plugins/search-indexer-backend/src/types.ts @@ -41,7 +41,7 @@ export interface IndexableDocument { owner?: string; /** - * The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). + * (Optional) The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). */ lifecycle?: string; } From 7c0d234cb5e46e60fb6ec06b36c2f32fcf276674 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 15 Feb 2021 17:39:14 +0100 Subject: [PATCH 16/52] Better formatting for "optional" Co-authored-by: Emma Indal Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-indexer-backend/src/types.ts index f9c2c0a263..63a10be2ea 100644 --- a/plugins/search-indexer-backend/src/types.ts +++ b/plugins/search-indexer-backend/src/types.ts @@ -36,7 +36,7 @@ export interface IndexableDocument { location: string; /** - * The owner of the document (e.g. spec.owner on a catalog entity). + * (Optional) The owner of the document (e.g. spec.owner on a catalog entity). */ owner?: string; From a8513b65dbd587bc1485ff8572808f407080b386 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:11:23 +0100 Subject: [PATCH 17/52] Fix readme title Co-authored-by: Adam Harvey Signed-off-by: Eric Peterson --- plugins/search-backend/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend/README.md b/plugins/search-backend/README.md index e539afec23..a5e1c1cfd8 100644 --- a/plugins/search-backend/README.md +++ b/plugins/search-backend/README.md @@ -1,4 +1,4 @@ -# search +# search-backend Welcome to the search backend plugin! From a9c1a720163f17a0cc5a57d17a4a4feeeb4cb3b3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:11:47 +0100 Subject: [PATCH 18/52] (C)* Co-authored-by: Adam Harvey Signed-off-by: Eric Peterson --- plugins/search-backend/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 90a67d7605..8d30303db2 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 0c04830e8b1fde3bbcf646992a211a3961399215 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:11:58 +0100 Subject: [PATCH 19/52] (C)* Co-authored-by: Adam Harvey Signed-off-by: Eric Peterson --- plugins/search-backend/src/run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend/src/run.ts b/plugins/search-backend/src/run.ts index b96989e4b8..a59d90d09a 100644 --- a/plugins/search-backend/src/run.ts +++ b/plugins/search-backend/src/run.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 Spotify AB + * Copyright 2021 Spotify AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 70bec421bfa74c25a93cd40e9396601ccbdd0e3a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:12:15 +0100 Subject: [PATCH 20/52] Link to discord! Co-authored-by: Adam Harvey Signed-off-by: Eric Peterson --- plugins/search/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search/README.md b/plugins/search/README.md index a95f6a21ed..62e8be1f1e 100644 --- a/plugins/search/README.md +++ b/plugins/search/README.md @@ -2,7 +2,7 @@ **This plugin is still under development.** -You can follow the progress and contribute at the Backstage [Search Project Board](https://github.com/backstage/backstage/projects/6) or reach out to us in the #search Discord channel. +You can follow the progress and contribute at the Backstage [Search Project Board](https://github.com/backstage/backstage/projects/6) or reach out to us in the [`#search` Discord channel](https://discord.com/channels/687207715902193673/770283289327566848). ## Getting started From c3030112470dbcf4595acaefa47723f18e83e945 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:15:37 +0100 Subject: [PATCH 21/52] Own (more of) it! Signed-off-by: Eric Peterson --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4e21603c63..c9efd18a91 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,6 +12,7 @@ /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core /plugins/search-* @backstage/techdocs-core +/packages/app/src/components/search/* @backstage/techdocs-core /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core From 1c136c7da8dd480f48ca69d1e5467727eab9b77a Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 10:37:02 +0100 Subject: [PATCH 22/52] Signed-off-by: Eric Peterson --- .../app/src/components/search/SearchPage.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/app/src/components/search/SearchPage.tsx b/packages/app/src/components/search/SearchPage.tsx index db6dcac2eb..1a0c1b8210 100644 --- a/packages/app/src/components/search/SearchPage.tsx +++ b/packages/app/src/components/search/SearchPage.tsx @@ -13,7 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Content, Header, Page, useQueryParamState } from '@backstage/core'; +import { + Content, + Header, + Lifecycle, + Page, + useQueryParamState, +} from '@backstage/core'; import { SearchBar, SearchResult } from '@backstage/plugin-search'; import { Grid } from '@material-ui/core'; import React, { useEffect, useState } from 'react'; @@ -47,7 +53,13 @@ export const SearchPage = () => { return ( -
+
+ Search + + } + /> From d78542d03793c5cd088b04ec4b7e76fdde30c76e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 11:10:13 +0100 Subject: [PATCH 23/52] Clean up READMEs Signed-off-by: Eric Peterson --- plugins/search-backend/README.md | 18 ++++++++++-------- plugins/search-indexer-backend/README.md | 20 ++++++++++++++++++-- plugins/search/README.md | 8 ++++---- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/plugins/search-backend/README.md b/plugins/search-backend/README.md index a5e1c1cfd8..678639d36b 100644 --- a/plugins/search-backend/README.md +++ b/plugins/search-backend/README.md @@ -1,13 +1,15 @@ # search-backend -Welcome to the search backend plugin! +This plugin is part of a suite of plugins that comprise the Backstage search +platform, which is still very much under development. This plugin specifically +is responsible for: -_This plugin was created through the Backstage CLI_ +- Exposing a JSON API for querying a search engine +- Types related to interacting with that API -## Getting started +Documentation on how to develop and improve the search platform is currently +centralized in the `search` plugin README.md. -Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/search](http://localhost:3000/search). - -You can also serve the plugin in isolation by running `yarn start` in the plugin directory. -This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. -It is only meant for local development, and the setup for it can be found inside the [/dev](/dev) directory. +For a better overview of how the search platform is put together, check the +[Backstage Search Architecture](https://backstage.io/docs/features/search/architecture) +documentation. diff --git a/plugins/search-indexer-backend/README.md b/plugins/search-indexer-backend/README.md index bac23fffd0..e065806052 100644 --- a/plugins/search-indexer-backend/README.md +++ b/plugins/search-indexer-backend/README.md @@ -1,5 +1,21 @@ # search-indexer -Welcome to the search-indexer backend plugin! +This plugin is part of a suite of plugins that comprise the Backstage search +platform, which is still very much under development. This plugin specifically +is responsible for: -_This plugin was created through the Backstage CLI_ +- Allowing other backend plugins to register the fact that they have documents + that they'd like to be indexed by a search engine (known as `collators`) +- Allowing other backend plugins to register the fact that they have metadata + that they'd like to augment existing documents in the search index with + (known as `decorators`) +- A scheduler that, at configurable intervals, compiles documents to be indexed + and passes them to a search engine for indexing +- Types for all of the above + +Documentation on how to develop and improve the search platform is currently +centralized in the `search` plugin README.md. + +For a better overview of how the search platform is put together, check the +[Backstage Search Architecture](https://backstage.io/docs/features/search/architecture) +documentation. diff --git a/plugins/search/README.md b/plugins/search/README.md index 62e8be1f1e..92e7eda15a 100644 --- a/plugins/search/README.md +++ b/plugins/search/README.md @@ -6,15 +6,15 @@ You can follow the progress and contribute at the Backstage [Search Project Boar ## Getting started -Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin. +Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search) to check out the plugin. ### Working on the search platform -Currently, the search experience available in Backstage is 100% on the front-end, and only looks at the Software Catalog. We are actively developing [a more complete search platform](https://backstage.io/docs/features/search/search-overview), which will replace the current experience when ready. +The above search experience is 100% client-side and only looks at the Software Catalog. We are actively developing [a more complete search platform](https://backstage.io/docs/features/search/search-overview), which will replace the current experience when ready. -In order to work on the new search platform, you will need to be aware of the following: +In order to work on this new search platform, you will need to be aware of the following: -- **The use-search-platform feature flag**: The new search platform will move the primary search page to the App-level, out of the search plugin. You'll need to enable this feature flag to work on the net platform. +- **The use-search-platform feature flag**: The new search platform will move the primary search page to the App-level, out of the search plugin. You'll need to enable this feature flag to work on the new platform. - **App SearchPage Component**: You'll find a new search page under `/packages/app/src/components/search`. When ready, we'll add an equivalent page to the `@backstage/create-app` template. - **Backend**: Don't forget, a lot of functionality will be made available in backend plugins: - `@backstage/plugin-search-indexer-backend`, which is responsible for the search index management From e49bc5f4cf0b75ce5fbdad1496d7466584e68a57 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 11:12:33 +0100 Subject: [PATCH 24/52] Avoid [object] in title bar by moving to subtitle. Signed-off-by: Eric Peterson --- packages/app/src/components/search/SearchPage.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/app/src/components/search/SearchPage.tsx b/packages/app/src/components/search/SearchPage.tsx index 1a0c1b8210..d1d03d48ea 100644 --- a/packages/app/src/components/search/SearchPage.tsx +++ b/packages/app/src/components/search/SearchPage.tsx @@ -53,13 +53,7 @@ export const SearchPage = () => { return ( -
- Search - - } - /> +
} /> From 1e82240ec30f2a00e8dc088f2e17448a6d07c484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Tue, 16 Feb 2021 14:17:00 +0100 Subject: [PATCH 25/52] update deps Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/package.json | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/plugins/search-indexer-backend/package.json b/plugins/search-indexer-backend/package.json index f853abc632..7b010442d0 100644 --- a/plugins/search-indexer-backend/package.json +++ b/plugins/search-indexer-backend/package.json @@ -20,19 +20,10 @@ }, "dependencies": { "@backstage/backend-common": "^0.5.3", - "@backstage/config": "^0.1.2", - "@types/express": "^4.17.6", - "express": "^4.17.1", - "express-promise-router": "^3.0.3", - "winston": "^3.2.1", - "cross-fetch": "^3.0.6", - "yn": "^4.0.0" + "@backstage/config": "^0.1.2" }, "devDependencies": { - "@backstage/cli": "^0.6.0", - "@types/supertest": "^2.0.8", - "supertest": "^4.0.2", - "msw": "^0.21.2" + "@backstage/cli": "^0.6.0" }, "files": [ "dist" From ca74761fce180c7dc3d2ff4222429104e50a8aa4 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 15:13:43 +0100 Subject: [PATCH 26/52] Add a simple sanity test to the search-backend plugin. Signed-off-by: Eric Peterson --- plugins/search-backend/src/service/router.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts index 0aaeafa379..ba7032eb93 100644 --- a/plugins/search-backend/src/service/router.test.ts +++ b/plugins/search-backend/src/service/router.test.ts @@ -34,12 +34,12 @@ describe('createRouter', () => { jest.resetAllMocks(); }); - describe('GET /health', () => { - it('returns ok', async () => { - const response = await request(app).get('/health'); + describe('GET /query', () => { + it('returns empty results array', async () => { + const response = await request(app).get('/query'); expect(response.status).toEqual(200); - expect(response.body).toEqual({ status: 'ok' }); + expect(response.body).toMatchObject({ results: [] }); }); }); }); From 98cd0442263cb77deddecc2d0e2d5d326b6fcba4 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 16 Feb 2021 15:14:11 +0100 Subject: [PATCH 27/52] Add tests exercising the API of the search-indexer-backend plugin. Signed-off-by: Eric Peterson --- .../search-indexer-backend/src/index.test.ts | 121 ++++++++++++++++++ .../search-indexer-backend/src/registry.ts | 32 +++-- 2 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 plugins/search-indexer-backend/src/index.test.ts diff --git a/plugins/search-indexer-backend/src/index.test.ts b/plugins/search-indexer-backend/src/index.test.ts new file mode 100644 index 0000000000..21db1657b3 --- /dev/null +++ b/plugins/search-indexer-backend/src/index.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { registerCollator, registerDecorator } from './'; +import { Registry } from './registry'; + +describe('external api', () => { + afterEach(() => { + Registry.getInstance()._reset(); + }); + + describe('registerCollator', () => { + it('registers a collator', async () => { + const collatorSpy = jest.fn(async () => []); + + // Register a collator. + registerCollator({ + type: 'anything', + defaultRefreshIntervalSeconds: 600, + collator: collatorSpy, + }); + + // Execute the registry and ensure the collator was invoked. + await Registry.getInstance().execute(); + expect(collatorSpy).toHaveBeenCalled(); + }); + }); + + describe('registerDecorator', () => { + it('registers a decorator', async () => { + const mockCollator = jest.fn(async () => []); + const decoratorSpy = jest.fn(async docs => docs); + + // Register a collator. + registerCollator({ + type: 'anything', + defaultRefreshIntervalSeconds: 600, + collator: mockCollator, + }); + + // Register a decorator. + registerDecorator({ + decorator: decoratorSpy, + }); + + // Execute the registry and ensure the decorator was invoked. + await Registry.getInstance().execute(); + expect(decoratorSpy).toHaveBeenCalled(); + }); + + it('registers a type-specific decorator', async () => { + const expectedType = 'an-expected-type'; + const docFixture = { + title: 'Test', + text: 'Test text.', + location: '/test/location', + }; + const mockCollator = jest.fn(async () => [docFixture]); + const decoratorSpy = jest.fn(async docs => docs); + + // Register a collator. + registerCollator({ + type: expectedType, + defaultRefreshIntervalSeconds: 600, + collator: mockCollator, + }); + + // Register a decorator for the same type. + registerDecorator({ + types: [expectedType], + decorator: decoratorSpy, + }); + + // Execute the registry and ensure the decorator was invoked. + await Registry.getInstance().execute(); + expect(decoratorSpy).toHaveBeenCalled(); + expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); + }); + + it('registers a type-specific decorator that should not be called', async () => { + const expectedType = 'an-expected-type'; + const docFixture = { + title: 'Test', + text: 'Test text.', + location: '/test/location', + }; + const mockCollator = jest.fn(async () => [docFixture]); + const decoratorSpy = jest.fn(async docs => docs); + + // Register a collator. + registerCollator({ + type: expectedType, + defaultRefreshIntervalSeconds: 600, + collator: mockCollator, + }); + + // Register a decorator for a different type. + registerDecorator({ + types: ['not-the-expected-type'], + decorator: decoratorSpy, + }); + + // Execute the registry and ensure the decorator was not invoked. + await Registry.getInstance().execute(); + expect(decoratorSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-indexer-backend/src/registry.ts index 23d39b12ff..8b71a87e57 100644 --- a/plugins/search-indexer-backend/src/registry.ts +++ b/plugins/search-indexer-backend/src/registry.ts @@ -69,18 +69,30 @@ export class Registry { } // @todo But like with coordination, timing, error handling, and what have you. - execute() { - Object.keys(this.collators).forEach(async type => { - const decorators: IndexableDocumentDecorator[] = ( - this.decorators['*'] || [] - ).concat(this.decorators[type] || []); - let documents = await this.collators[type].collate(); + async execute(): Promise { + return new Promise(resolve => { + Object.keys(this.collators).forEach(async type => { + const decorators: IndexableDocumentDecorator[] = ( + this.decorators['*'] || [] + ).concat(this.decorators[type] || []); + let documents = await this.collators[type].collate(); - for (let i = 0; i < decorators.length; i++) { - documents = await decorators[i](documents); - } + for (let i = 0; i < decorators.length; i++) { + documents = await decorators[i](documents); + } - // @todo: push documents to a configured search engine. + // @todo: push documents to a configured search engine. + resolve(undefined); + }); }); } + + /** + * Utility method for tests. Do not use otherwise. + * @private + */ + _reset() { + this.collators = {}; + this.decorators = {}; + } } From 92cd651fa42c335db003f9112ff19b79720059b5 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 19 Feb 2021 17:33:01 +0100 Subject: [PATCH 28/52] Ditch the feature flag. Signed-off-by: Eric Peterson --- packages/app/src/App.tsx | 6 +----- plugins/search/README.md | 2 +- plugins/search/src/plugin.ts | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 99a6f8096f..7bf503daed 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -17,7 +17,6 @@ import { AlertDisplay, createApp, - LocalStorageFeatureFlags, FlatRoutes, OAuthRequestDialog, SignInPage, @@ -89,7 +88,6 @@ const app = createApp({ const AppProvider = app.getProvider(); const AppRouter = app.getRouter(); -const featureFlags = new LocalStorageFeatureFlags(); const routes = ( @@ -103,9 +101,7 @@ const routes = ( } /> } /> - {featureFlags.isActive('use-search-platform') && ( - } /> - )} + } /> } /> } /> Date: Fri, 19 Feb 2021 17:39:20 +0100 Subject: [PATCH 29/52] Clean up error handling. Signed-off-by: Eric Peterson --- plugins/search-backend/src/service/router.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 02bc553bb9..185a92831a 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { errorHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; @@ -38,7 +37,7 @@ export async function createRouter({ // @todo Actually transform req.params into search engine specific query. const { term, filters = {}, pageIndex = 0, pageSize = 30 } = req.query; logger.info( - `Search requested received: ${term}, ${JSON.stringify( + `Search request received: ${term}, ${JSON.stringify( filters, )}, ${pageIndex}, ${pageSize}`, ); @@ -50,12 +49,10 @@ export async function createRouter({ results: [], }); } catch (err) { - logger.error(`There was a problem performing the search query.`); throw new Error(`There was a problem performing the search query.`); } }, ); - router.use(errorHandler()); return router; } From 2c759b452a593c6f48fb457f7acee56f0dcddb9b Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 19 Feb 2021 18:27:15 +0100 Subject: [PATCH 30/52] Clean up query types, ensure we've got room to expand on the result items. Signed-off-by: Eric Peterson --- plugins/search-backend/src/types.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/search-backend/src/types.ts b/plugins/search-backend/src/types.ts index f7e6acf5ee..dbb088e117 100644 --- a/plugins/search-backend/src/types.ts +++ b/plugins/search-backend/src/types.ts @@ -14,15 +14,20 @@ * limitations under the License. */ +import { JsonObject } from '@backstage/config'; import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; export interface SearchQuery { term: string; - filters?: Record; + filters?: JsonObject; pageIndex?: number; pageSize?: number; } -export interface SearchResultSet { - results: IndexableDocument[]; +export interface SearchResult { + document: IndexableDocument; +} + +export interface SearchResultSet { + results: SearchResult[]; } From 925f5847debd90849c093093d5fe9e49687d1d9e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 25 Feb 2021 11:29:12 +0100 Subject: [PATCH 31/52] Fix app. Signed-off-by: Eric Peterson --- packages/app/src/App.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 7bf503daed..0e874e64e0 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -50,7 +50,7 @@ import { Navigate, Route } from 'react-router'; import { apis } from './apis'; import { EntityPage } from './components/catalog/EntityPage'; import Root from './components/Root'; -import { SearchPage } from './components/search/SearchPage'; +import { SearchPage as SearchPageNext } from './components/search/SearchPage'; import { providers } from './identityProviders'; import * as plugins from './plugins'; @@ -101,7 +101,6 @@ const routes = ( } /> } /> - } /> } /> } /> } /> } /> } /> + } /> } /> Date: Thu, 25 Feb 2021 14:45:48 +0100 Subject: [PATCH 32/52] standard TODO syntax Signed-off-by: Eric Peterson --- packages/app/src/components/search/SearchPage.tsx | 2 +- packages/backend/src/index.ts | 2 +- plugins/search-backend/src/service/router.ts | 6 +++--- plugins/search-indexer-backend/src/registry.ts | 4 ++-- plugins/search/src/apis.ts | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/app/src/components/search/SearchPage.tsx b/packages/app/src/components/search/SearchPage.tsx index d1d03d48ea..c2f0c90208 100644 --- a/packages/app/src/components/search/SearchPage.tsx +++ b/packages/app/src/components/search/SearchPage.tsx @@ -25,7 +25,7 @@ import { Grid } from '@material-ui/core'; import React, { useEffect, useState } from 'react'; import { useDebounce } from 'react-use'; -// @todo Simplify / remove as much state handling here. The goal is for this to +// TODO: Simplify / remove as much state handling here. The goal is for this to // more or less use the publicly exposed components from the search plugin and // not much else. export const SearchPage = () => { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index fededf0f0a..8feb060de2 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -111,7 +111,7 @@ async function main() { process.exit(1); }); - // @todo Start Search Refresh Loop Here? + // TODO: Start Search Refresh Loop Here? } module.hot?.accept(); diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 185a92831a..47b7354b22 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -34,7 +34,7 @@ export async function createRouter({ req: express.Request, res: express.Response, ) => { - // @todo Actually transform req.params into search engine specific query. + // TODO: Actually transform req.params into search engine specific query. const { term, filters = {}, pageIndex = 0, pageSize = 30 } = req.query; logger.info( `Search request received: ${term}, ${JSON.stringify( @@ -43,8 +43,8 @@ export async function createRouter({ ); try { - // @todo Actually query search engine. - // @todo And actually transform results into frontend-readable result + // TODO: Actually query search engine. + // TODO: And actually transform results into frontend-readable result res.send({ results: [], }); diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-indexer-backend/src/registry.ts index 8b71a87e57..609bb7b30f 100644 --- a/plugins/search-indexer-backend/src/registry.ts +++ b/plugins/search-indexer-backend/src/registry.ts @@ -68,7 +68,7 @@ export class Registry { }); } - // @todo But like with coordination, timing, error handling, and what have you. + // TODO: But like with coordination, timing, error handling, and what have you. async execute(): Promise { return new Promise(resolve => { Object.keys(this.collators).forEach(async type => { @@ -81,7 +81,7 @@ export class Registry { documents = await decorators[i](documents); } - // @todo: push documents to a configured search engine. + // TODO: push documents to a configured search engine. resolve(undefined); }); }); diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 65490c3c37..6b739427d7 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -62,7 +62,7 @@ class SearchApi { return this.entities(); } - // @todo Productionalize as we implement search milestones. + // TODO: Productionalize as we implement search milestones. public async _alphaPerformSearch( query: SearchQuery, ): Promise { From 8ac1f0d2ccd47488231878fb68c1b2ab86ff8ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Thu, 25 Feb 2021 14:53:53 +0100 Subject: [PATCH 33/52] removed filter on component Signed-off-by: Eric Peterson --- plugins/catalog-backend/src/search/Collator.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/catalog-backend/src/search/Collator.ts b/plugins/catalog-backend/src/search/Collator.ts index 691e99eb3a..e7d5483230 100644 --- a/plugins/catalog-backend/src/search/Collator.ts +++ b/plugins/catalog-backend/src/search/Collator.ts @@ -19,7 +19,6 @@ import { IndexableDocumentCollator, } from '@backstage/plugin-search-indexer-backend'; import { EntitiesCatalog } from '../catalog'; -import { EntityFilters } from '../service/EntityFilters'; export interface CatalogEntityDocument extends IndexableDocument { componentType: string; @@ -29,8 +28,7 @@ export const SearchCollatorFactory = ( entitiesCatalog: EntitiesCatalog, ): IndexableDocumentCollator => { return async (): Promise => { - const filter = EntityFilters.ofQuery({ kind: 'Component' }); - const entities = await entitiesCatalog.entities(filter); + const entities = await entitiesCatalog.entities(); return entities.map( (entity): CatalogEntityDocument => { return { From 355d720362e0b9da1c9a058d45a9bac0e6c75e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Thu, 25 Feb 2021 14:59:47 +0100 Subject: [PATCH 34/52] use Promise.all Signed-off-by: Eric Peterson --- plugins/search-indexer-backend/src/registry.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-indexer-backend/src/registry.ts index 609bb7b30f..1e5363401a 100644 --- a/plugins/search-indexer-backend/src/registry.ts +++ b/plugins/search-indexer-backend/src/registry.ts @@ -69,22 +69,21 @@ export class Registry { } // TODO: But like with coordination, timing, error handling, and what have you. - async execute(): Promise { - return new Promise(resolve => { - Object.keys(this.collators).forEach(async type => { + async execute() { + return Promise.all( + Object.keys(this.collators).map(async type => { const decorators: IndexableDocumentDecorator[] = ( this.decorators['*'] || [] ).concat(this.decorators[type] || []); - let documents = await this.collators[type].collate(); + let documents = await this.collators[type].collate(); for (let i = 0; i < decorators.length; i++) { documents = await decorators[i](documents); } // TODO: push documents to a configured search engine. - resolve(undefined); - }); - }); + }), + ); } /** From 779a164507d25d5907f3a4f5c62bf40a68229a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Fri, 26 Feb 2021 14:05:45 +0100 Subject: [PATCH 35/52] use Utility API Signed-off-by: Eric Peterson --- plugins/search/src/apis.ts | 12 ++++++++---- .../src/components/SearchResult/SearchResult.tsx | 10 +++------- plugins/search/src/index.ts | 2 ++ plugins/search/src/plugin.ts | 13 +++++++++++++ 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 6b739427d7..a222cde5a9 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -14,12 +14,18 @@ * limitations under the License. */ +import { createApiRef, DiscoveryApi } from '@backstage/core'; import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; -import { DiscoveryApi } from '@backstage/core'; + import { CatalogApi } from '@backstage/plugin-catalog-react'; import { SearchQuery, SearchResultSet } from '@backstage/plugin-search-backend'; import qs from 'qs'; +export const searchApiRef = createApiRef({ + id: 'plugin.search.queryservice', + description: 'Used to make requests against the search API', +}); + export type Result = { name: string; description: string | undefined; @@ -31,7 +37,7 @@ export type Result = { export type SearchResults = Array; -class SearchApi { +export class SearchApi { private catalogApi: CatalogApi; private discoveryApi: DiscoveryApi; @@ -74,5 +80,3 @@ class SearchApi { return response.json(); } } - -export default SearchApi; diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index 141b7684cd..0aa209e6fe 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -14,7 +14,6 @@ * limitations under the License. */ import { - discoveryApiRef, EmptyState, Link, Progress, @@ -22,12 +21,12 @@ import { TableColumn, useApi, } from '@backstage/core'; -import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { Divider, Grid, makeStyles, Typography } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; import React, { useEffect, useState } from 'react'; import { useAsync } from 'react-use'; -import SearchApi, { Result, SearchResults } from '../../apis'; +import { Result, SearchResults, searchApiRef } from '../../apis'; + import { Filters, FiltersButton, FiltersState } from '../Filters'; const useStyles = makeStyles(theme => ({ @@ -117,8 +116,7 @@ const TableHeader = ({ }; export const SearchResult = ({ searchQuery }: SearchResultProps) => { - const catalogApi = useApi(catalogApiRef); - const discoveryApi = useApi(discoveryApiRef); + const searchApi = useApi(searchApiRef); const [showFilters, toggleFilters] = useState(false); const [selectedFilters, setSelectedFilters] = useState({ @@ -128,8 +126,6 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { const [filteredResults, setFilteredResults] = useState([]); - const searchApi = new SearchApi(catalogApi, discoveryApi); - const { loading, error, value: results } = useAsync(() => { return searchApi.getSearchResult(); }, []); diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts index 572d75bea5..268de25a00 100644 --- a/plugins/search/src/index.ts +++ b/plugins/search/src/index.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +// TODO: export searchApiRef from ./apis once interface is stable and settled. export { searchPlugin, searchPlugin as plugin, SearchPage } from './plugin'; export { Filters, diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index 37503b7751..c94602868e 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -14,10 +14,14 @@ * limitations under the License. */ import { + createApiFactory, createPlugin, createRouteRef, createRoutableExtension, + discoveryApiRef, } from '@backstage/core'; +import { SearchApi, searchApiRef } from './apis'; +import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { SearchPage as SearchPageComponent } from './components/SearchPage'; export const rootRouteRef = createRouteRef({ @@ -27,6 +31,15 @@ export const rootRouteRef = createRouteRef({ export const searchPlugin = createPlugin({ id: 'search', + apis: [ + createApiFactory({ + api: searchApiRef, + deps: { catalogApi: catalogApiRef, discoveryApi: discoveryApiRef }, + factory: ({ catalogApi, discoveryApi }) => { + return new SearchApi(catalogApi, discoveryApi); + }, + }), + ], register({ router }) { router.addRoute(rootRouteRef, SearchPageComponent); }, From f9b203e07e1170cb6280dacd694de51f1d004926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Fri, 26 Feb 2021 14:36:19 +0100 Subject: [PATCH 36/52] cursor pagination Signed-off-by: Eric Peterson --- plugins/search-backend/src/service/router.ts | 4 ++-- plugins/search-backend/src/types.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 47b7354b22..7dab206b6a 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -35,11 +35,11 @@ export async function createRouter({ res: express.Response, ) => { // TODO: Actually transform req.params into search engine specific query. - const { term, filters = {}, pageIndex = 0, pageSize = 30 } = req.query; + const { term, filters = {}, pageCursor = '' } = req.query; logger.info( `Search request received: ${term}, ${JSON.stringify( filters, - )}, ${pageIndex}, ${pageSize}`, + )}, ${pageCursor}`, ); try { diff --git a/plugins/search-backend/src/types.ts b/plugins/search-backend/src/types.ts index dbb088e117..119ecf35f3 100644 --- a/plugins/search-backend/src/types.ts +++ b/plugins/search-backend/src/types.ts @@ -20,8 +20,7 @@ import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; export interface SearchQuery { term: string; filters?: JsonObject; - pageIndex?: number; - pageSize?: number; + pageCursor: string; } export interface SearchResult { From 608649b13cb0b320f1ccf02f60a077e02dfb42bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Fri, 26 Feb 2021 14:55:01 +0100 Subject: [PATCH 37/52] add search-common package Signed-off-by: Eric Peterson --- packages/search-common/.eslintrc.js | 3 ++ packages/search-common/README.md | 3 ++ packages/search-common/package.json | 56 +++++++++++++++++++++++++++++ packages/search-common/src/index.ts | 16 +++++++++ packages/search-common/src/types.ts | 31 ++++++++++++++++ 5 files changed, 109 insertions(+) create mode 100644 packages/search-common/.eslintrc.js create mode 100644 packages/search-common/README.md create mode 100644 packages/search-common/package.json create mode 100644 packages/search-common/src/index.ts create mode 100644 packages/search-common/src/types.ts diff --git a/packages/search-common/.eslintrc.js b/packages/search-common/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/search-common/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/search-common/README.md b/packages/search-common/README.md new file mode 100644 index 0000000000..66828cc7af --- /dev/null +++ b/packages/search-common/README.md @@ -0,0 +1,3 @@ +# @backstage/search-common + +Common functionalities for Search, to be shared between various search-enabled plugins. diff --git a/packages/search-common/package.json b/packages/search-common/package.json new file mode 100644 index 0000000000..6336a13449 --- /dev/null +++ b/packages/search-common/package.json @@ -0,0 +1,56 @@ +{ + "name": "@backstage/search-common", + "description": "Common functionalities for Search, to be shared between various search-enabled plugins", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/search-common" + }, + "keywords": [ + "backstage", + "search" + ], + "license": "Apache-2.0", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli build --outputs cjs,types", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "bugs": { + "url": "https://github.com/backstage/backstage/issues" + }, + "dependencies": { + "@backstage/config": "^0.1.3", + "@backstage/plugin-search-indexer-backend": "^0.1.1" + }, + "devDependencies": { + "@backstage/cli": "^0.6.2", + "@types/fs-extra": "^9.0.5", + "@types/git-url-parse": "^9.0.0", + "@types/js-yaml": "^3.12.5", + "@types/mime-types": "^2.1.0", + "@types/mock-fs": "^4.13.0", + "@types/recursive-readdir": "^2.2.0" + }, + "jest": { + "roots": [ + ".." + ] + } +} diff --git a/packages/search-common/src/index.ts b/packages/search-common/src/index.ts new file mode 100644 index 0000000000..e6432477f2 --- /dev/null +++ b/packages/search-common/src/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './types'; diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts new file mode 100644 index 0000000000..1a05e85f69 --- /dev/null +++ b/packages/search-common/src/types.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { JsonObject } from '@backstage/config'; +import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; + +export interface SearchQuery { + term: string; + filters?: JsonObject; + pageCursor: string; +} + +export interface SearchResult { + document: IndexableDocument; +} + +export interface SearchResultSet { + results: SearchResult[]; +} From 4abd922fb53ef7cb0008c99880618378c829d905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Fri, 26 Feb 2021 14:56:00 +0100 Subject: [PATCH 38/52] moved types from search-backend to search-common Signed-off-by: Eric Peterson --- plugins/search-backend/package.json | 1 + plugins/search-backend/src/index.ts | 1 - plugins/search-backend/src/types.ts | 32 ----------------------------- plugins/search/package.json | 1 + plugins/search/src/apis.ts | 2 +- 5 files changed, 3 insertions(+), 34 deletions(-) delete mode 100644 plugins/search-backend/src/types.ts diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index a4e2b566b8..7717b4c7ee 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -22,6 +22,7 @@ "@backstage/backend-common": "^0.5.3", "@backstage/config": "^0.1.2", "@backstage/plugin-search-indexer-backend": "^0.1.1", + "@backstage/search-common": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts index 8d30303db2..38e2cdf4cb 100644 --- a/plugins/search-backend/src/index.ts +++ b/plugins/search-backend/src/index.ts @@ -15,4 +15,3 @@ */ export * from './service/router'; -export * from './types'; diff --git a/plugins/search-backend/src/types.ts b/plugins/search-backend/src/types.ts deleted file mode 100644 index 119ecf35f3..0000000000 --- a/plugins/search-backend/src/types.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2021 Spotify AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { JsonObject } from '@backstage/config'; -import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; - -export interface SearchQuery { - term: string; - filters?: JsonObject; - pageCursor: string; -} - -export interface SearchResult { - document: IndexableDocument; -} - -export interface SearchResultSet { - results: SearchResult[]; -} diff --git a/plugins/search/package.json b/plugins/search/package.json index 6fa80cda54..d88ed9b241 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -33,6 +33,7 @@ "@backstage/catalog-model": "^0.7.3", "@backstage/plugin-catalog-react": "^0.1.2", "@backstage/plugin-search-backend": "^0.1.1", + "@backstage/search-common": "^0.1.1", "@backstage/theme": "^0.2.4", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index a222cde5a9..dc6c12e249 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -18,7 +18,7 @@ import { createApiRef, DiscoveryApi } from '@backstage/core'; import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { CatalogApi } from '@backstage/plugin-catalog-react'; -import { SearchQuery, SearchResultSet } from '@backstage/plugin-search-backend'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; import qs from 'qs'; export const searchApiRef = createApiRef({ From 5e70312d67babc7b443064c771de4118cb6dd4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Fri, 26 Feb 2021 15:17:37 +0100 Subject: [PATCH 39/52] unwind dependencies Signed-off-by: Eric Peterson --- packages/search-common/package.json | 11 +---- packages/search-common/src/types.ts | 47 +++++++++++++++++- plugins/catalog-backend/package.json | 1 + .../catalog-backend/src/search/Collator.ts | 2 +- plugins/search-backend/package.json | 6 +-- plugins/search-backend/src/service/router.ts | 2 +- plugins/search-indexer-backend/package.json | 3 +- plugins/search-indexer-backend/src/index.ts | 6 --- .../search-indexer-backend/src/registry.ts | 2 + plugins/search-indexer-backend/src/types.ts | 49 ++----------------- 10 files changed, 59 insertions(+), 70 deletions(-) diff --git a/packages/search-common/package.json b/packages/search-common/package.json index 6336a13449..18e3557a49 100644 --- a/packages/search-common/package.json +++ b/packages/search-common/package.json @@ -36,17 +36,10 @@ "url": "https://github.com/backstage/backstage/issues" }, "dependencies": { - "@backstage/config": "^0.1.3", - "@backstage/plugin-search-indexer-backend": "^0.1.1" + "@backstage/config": "^0.1.3" }, "devDependencies": { - "@backstage/cli": "^0.6.2", - "@types/fs-extra": "^9.0.5", - "@types/git-url-parse": "^9.0.0", - "@types/js-yaml": "^3.12.5", - "@types/mime-types": "^2.1.0", - "@types/mock-fs": "^4.13.0", - "@types/recursive-readdir": "^2.2.0" + "@backstage/cli": "^0.6.2" }, "jest": { "roots": [ diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 1a05e85f69..3a47187945 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -14,7 +14,6 @@ * limitations under the License. */ import { JsonObject } from '@backstage/config'; -import { IndexableDocument } from '@backstage/plugin-search-indexer-backend'; export interface SearchQuery { term: string; @@ -29,3 +28,49 @@ export interface SearchResult { export interface SearchResultSet { results: SearchResult[]; } + +/** + * Base properties that all indexed documents must include, as well as some + * common properties that documents are encouraged to use where appropriate. + */ +export interface IndexableDocument { + /** + * The primary name of the document (e.g. name, title, identifier, etc). + */ + title: string; + + /** + * Free-form text of the document (e.g. description, content, etc). + */ + text: string; + + /** + * The relative or absolute URL of the document (target when a search result + * is clicked). + */ + location: string; + + /** + * (Optional) The owner of the document (e.g. spec.owner on a catalog entity). + */ + owner?: string; + + /** + * (Optional) The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). + */ + lifecycle?: string; +} + +/** + * Signature for the callback function that implementors must register to have + * their documents indexed. + */ +export type IndexableDocumentCollator = () => Promise; + +/** + * Signature for the callback function that implementors must register to + * decorate existing documents with additional metadata. + */ +export type IndexableDocumentDecorator = ( + documents: IndexableDocument[], +) => Promise; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index c99221b05f..f178e05f68 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -36,6 +36,7 @@ "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", "@backstage/plugin-search-indexer-backend": "^0.1.1", + "@backstage/search-common": "^0.1.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/search/Collator.ts b/plugins/catalog-backend/src/search/Collator.ts index e7d5483230..1513060384 100644 --- a/plugins/catalog-backend/src/search/Collator.ts +++ b/plugins/catalog-backend/src/search/Collator.ts @@ -17,7 +17,7 @@ import { IndexableDocument, IndexableDocumentCollator, -} from '@backstage/plugin-search-indexer-backend'; +} from '@backstage/search-common'; import { EntitiesCatalog } from '../catalog'; export interface CatalogEntityDocument extends IndexableDocument { diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json index 7717b4c7ee..9a1150de8a 100644 --- a/plugins/search-backend/package.json +++ b/plugins/search-backend/package.json @@ -20,21 +20,17 @@ }, "dependencies": { "@backstage/backend-common": "^0.5.3", - "@backstage/config": "^0.1.2", - "@backstage/plugin-search-indexer-backend": "^0.1.1", "@backstage/search-common": "^0.1.1", "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^3.0.3", "winston": "^3.2.1", - "cross-fetch": "^3.0.6", "yn": "^4.0.0" }, "devDependencies": { "@backstage/cli": "^0.6.0", "@types/supertest": "^2.0.8", - "supertest": "^4.0.2", - "msw": "^0.21.2" + "supertest": "^4.0.2" }, "files": [ "dist" diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 7dab206b6a..1a0c67bdb8 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -17,7 +17,7 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { SearchQuery, SearchResultSet } from '../types'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; type RouterOptions = { logger: Logger; diff --git a/plugins/search-indexer-backend/package.json b/plugins/search-indexer-backend/package.json index 7b010442d0..656fb2dca5 100644 --- a/plugins/search-indexer-backend/package.json +++ b/plugins/search-indexer-backend/package.json @@ -19,8 +19,7 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/backend-common": "^0.5.3", - "@backstage/config": "^0.1.2" + "@backstage/search-common": "^0.1.1" }, "devDependencies": { "@backstage/cli": "^0.6.0" diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts index 13d1a298b7..cde6e268b1 100644 --- a/plugins/search-indexer-backend/src/index.ts +++ b/plugins/search-indexer-backend/src/index.ts @@ -27,9 +27,3 @@ export const registerCollator = (params: RegisterCollatorParameters) => { export const registerDecorator = (params: RegisterDecoratorParameters) => { registry.addDecorator(params); }; - -export type { - IndexableDocument, - IndexableDocumentCollator, - IndexableDocumentDecorator, -} from './types'; diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-indexer-backend/src/registry.ts index 1e5363401a..6125e8f6a3 100644 --- a/plugins/search-indexer-backend/src/registry.ts +++ b/plugins/search-indexer-backend/src/registry.ts @@ -17,6 +17,8 @@ import { IndexableDocumentCollator, IndexableDocumentDecorator, +} from '@backstage/search-common'; +import { RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-indexer-backend/src/types.ts index 63a10be2ea..08ba475df8 100644 --- a/plugins/search-indexer-backend/src/types.ts +++ b/plugins/search-indexer-backend/src/types.ts @@ -14,51 +14,10 @@ * limitations under the License. */ -/** - * Base properties that all indexed documents must include, as well as some - * common properties that documents are encouraged to use where appropriate. - */ -export interface IndexableDocument { - /** - * The primary name of the document (e.g. name, title, identifier, etc). - */ - title: string; - - /** - * Free-form text of the document (e.g. description, content, etc). - */ - text: string; - - /** - * The relative or absolute URL of the document (target when a search result - * is clicked). - */ - location: string; - - /** - * (Optional) The owner of the document (e.g. spec.owner on a catalog entity). - */ - owner?: string; - - /** - * (Optional) The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). - */ - lifecycle?: string; -} - -/** - * Signature for the callback function that implementors must register to have - * their documents indexed. - */ -export type IndexableDocumentCollator = () => Promise; - -/** - * Signature for the callback function that implementors must register to - * decorate existing documents with additional metadata. - */ -export type IndexableDocumentDecorator = ( - documents: IndexableDocument[], -) => Promise; +import { + IndexableDocumentCollator, + IndexableDocumentDecorator, +} from '@backstage/search-common'; /** * Parameters required to register a collator. From 6d4919a61204ceb255d0c69cd964cf8b5d7ac3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Mon, 1 Mar 2021 13:24:56 +0100 Subject: [PATCH 40/52] make SearchApi an interface Signed-off-by: Eric Peterson --- plugins/search/src/apis.ts | 23 +++++++++++++---------- plugins/search/src/plugin.ts | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index dc6c12e249..b15be349cf 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -37,13 +37,18 @@ export type Result = { export type SearchResults = Array; -export class SearchApi { - private catalogApi: CatalogApi; - private discoveryApi: DiscoveryApi; +export interface SearchApi { + getSearchResult(): Promise; + _alphaPerformSearch(query: SearchQuery): Promise; +} - constructor(catalogApi: CatalogApi, discoveryApi: DiscoveryApi) { - this.catalogApi = catalogApi; - this.discoveryApi = discoveryApi; +export class SearchClient implements SearchApi { + private readonly catalogApi: CatalogApi; + private readonly discoveryApi: DiscoveryApi; + + constructor(options: { catalogApi: CatalogApi; discoveryApi: DiscoveryApi }) { + this.catalogApi = options.catalogApi; + this.discoveryApi = options.discoveryApi; } private async entities() { @@ -64,14 +69,12 @@ export class SearchApi { })); } - public getSearchResult(): Promise { + getSearchResult(): Promise { return this.entities(); } // TODO: Productionalize as we implement search milestones. - public async _alphaPerformSearch( - query: SearchQuery, - ): Promise { + async _alphaPerformSearch(query: SearchQuery): Promise { const queryString = qs.stringify(query); const url = `${await this.discoveryApi.getBaseUrl( 'search/query', diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index c94602868e..e21ad934a0 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -20,7 +20,7 @@ import { createRoutableExtension, discoveryApiRef, } from '@backstage/core'; -import { SearchApi, searchApiRef } from './apis'; +import { SearchClient, searchApiRef } from './apis'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { SearchPage as SearchPageComponent } from './components/SearchPage'; @@ -36,7 +36,7 @@ export const searchPlugin = createPlugin({ api: searchApiRef, deps: { catalogApi: catalogApiRef, discoveryApi: discoveryApiRef }, factory: ({ catalogApi, discoveryApi }) => { - return new SearchApi(catalogApi, discoveryApi); + return new SearchClient({ catalogApi, discoveryApi }); }, }), ], From 00576924011c64cd044e6fca3069226035d6e3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Wed, 17 Mar 2021 09:59:12 +0100 Subject: [PATCH 41/52] move collation to search plugin Signed-off-by: Eric Peterson --- packages/backend/src/index.ts | 2 -- packages/backend/src/plugins/catalog.ts | 8 -------- packages/backend/src/plugins/search.ts | 15 +++++++++++++++ plugins/search-indexer-backend/src/index.ts | 3 +++ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 8feb060de2..c32c7e6895 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -110,8 +110,6 @@ async function main() { console.log(err); process.exit(1); }); - - // TODO: Start Search Refresh Loop Here? } module.hot?.accept(); diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 86375203ab..eaca42717c 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -19,10 +19,8 @@ import { CatalogBuilder, createRouter, runPeriodically, - SearchCollatorFactory, } from '@backstage/plugin-catalog-backend'; import { Router } from 'express'; -import { registerCollator } from '@backstage/plugin-search-indexer-backend'; import { PluginEnvironment } from '../types'; export default async function createPlugin( @@ -41,12 +39,6 @@ export default async function createPlugin( runPeriodically(() => higherOrderOperation.refreshAllLocations(), 100000), ); - registerCollator({ - type: 'software-catalog', - defaultRefreshIntervalSeconds: 600, - collator: SearchCollatorFactory(entitiesCatalog), - }); - return await createRouter({ entitiesCatalog, locationsCatalog, diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 07a784f654..8619c1b6ca 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -14,9 +14,24 @@ * limitations under the License. */ import { createRouter } from '@backstage/plugin-search-backend'; +import { + collateDocuments, + // registerCollator, +} from '@backstage/plugin-search-indexer-backend'; import { PluginEnvironment } from '../types'; +// import { SearchCollatorFactory } from '@backstage/plugin-catalog-backend'; export default async function createPlugin({ logger }: PluginEnvironment) { + // TODO: Within this PR, update to use REST API instead of Catalog Builder. + /* registerCollator({ + type: 'software-catalog', + defaultRefreshIntervalSeconds: 600, + collator: SearchCollatorFactory(entitiesCatalog), + });*/ + + // TODO: Make this a more proper refresh loop. + collateDocuments(); + return await createRouter({ logger, }); diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-indexer-backend/src/index.ts index cde6e268b1..f016068db1 100644 --- a/plugins/search-indexer-backend/src/index.ts +++ b/plugins/search-indexer-backend/src/index.ts @@ -27,3 +27,6 @@ export const registerCollator = (params: RegisterCollatorParameters) => { export const registerDecorator = (params: RegisterDecoratorParameters) => { registry.addDecorator(params); }; +export const collateDocuments = () => { + registry.execute(); +}; From 124b93bff45f9438fa3ab43666f74c5467fcd5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Wed, 17 Mar 2021 10:12:44 +0100 Subject: [PATCH 42/52] rename plugin-search-indexer to plugin-search-backend-node Signed-off-by: Eric Peterson --- docs/assets/search/architecture.drawio.svg | 12 ++++++------ packages/backend/package.json | 2 +- packages/backend/src/plugins/search.ts | 2 +- .../.eslintrc.js | 0 .../README.md | 2 +- .../package.json | 2 +- .../src/index.test.ts | 0 .../src/index.ts | 0 .../src/registry.ts | 0 .../src/setupTests.ts | 0 .../src/types.ts | 0 plugins/search/README.md | 2 +- 12 files changed, 11 insertions(+), 11 deletions(-) rename plugins/{search-indexer-backend => search-backend-node}/.eslintrc.js (100%) rename plugins/{search-indexer-backend => search-backend-node}/README.md (97%) rename plugins/{search-indexer-backend => search-backend-node}/package.json (92%) rename plugins/{search-indexer-backend => search-backend-node}/src/index.test.ts (100%) rename plugins/{search-indexer-backend => search-backend-node}/src/index.ts (100%) rename plugins/{search-indexer-backend => search-backend-node}/src/registry.ts (100%) rename plugins/{search-indexer-backend => search-backend-node}/src/setupTests.ts (100%) rename plugins/{search-indexer-backend => search-backend-node}/src/types.ts (100%) diff --git a/docs/assets/search/architecture.drawio.svg b/docs/assets/search/architecture.drawio.svg index a624a375e1..1acad6d74b 100644 --- a/docs/assets/search/architecture.drawio.svg +++ b/docs/assets/search/architecture.drawio.svg @@ -1,4 +1,4 @@ - + @@ -268,7 +268,7 @@ -
+
Pass Search @@ -282,7 +282,7 @@
- + Pass Search... @@ -368,7 +368,7 @@ - + @@ -529,7 +529,7 @@ - + @@ -538,7 +538,7 @@
@backstage/
- plugin-search-indexer + plugin-search-backend-node
diff --git a/packages/backend/package.json b/packages/backend/package.json index 4e999b1531..eab3d512fe 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -41,7 +41,7 @@ "@backstage/plugin-rollbar-backend": "^0.1.8", "@backstage/plugin-scaffolder-backend": "^0.9.2", "@backstage/plugin-search-backend": "^0.1.1", - "@backstage/plugin-search-indexer-backend": "^0.1.1", + "@backstage/plugin-search-backend-node": "^0.1.1", "@backstage/plugin-techdocs-backend": "^0.6.5", "@backstage/plugin-todo-backend": "^0.1.1", "@gitbeaker/node": "^28.0.2", diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 8619c1b6ca..d1268e4f59 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -17,7 +17,7 @@ import { createRouter } from '@backstage/plugin-search-backend'; import { collateDocuments, // registerCollator, -} from '@backstage/plugin-search-indexer-backend'; +} from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; // import { SearchCollatorFactory } from '@backstage/plugin-catalog-backend'; diff --git a/plugins/search-indexer-backend/.eslintrc.js b/plugins/search-backend-node/.eslintrc.js similarity index 100% rename from plugins/search-indexer-backend/.eslintrc.js rename to plugins/search-backend-node/.eslintrc.js diff --git a/plugins/search-indexer-backend/README.md b/plugins/search-backend-node/README.md similarity index 97% rename from plugins/search-indexer-backend/README.md rename to plugins/search-backend-node/README.md index e065806052..126e8b9988 100644 --- a/plugins/search-indexer-backend/README.md +++ b/plugins/search-backend-node/README.md @@ -1,4 +1,4 @@ -# search-indexer +# search-backend-node This plugin is part of a suite of plugins that comprise the Backstage search platform, which is still very much under development. This plugin specifically diff --git a/plugins/search-indexer-backend/package.json b/plugins/search-backend-node/package.json similarity index 92% rename from plugins/search-indexer-backend/package.json rename to plugins/search-backend-node/package.json index 656fb2dca5..381c6baf7d 100644 --- a/plugins/search-indexer-backend/package.json +++ b/plugins/search-backend-node/package.json @@ -1,5 +1,5 @@ { - "name": "@backstage/plugin-search-indexer-backend", + "name": "@backstage/plugin-search-backend-node", "version": "0.1.1", "main": "src/index.ts", "types": "src/index.ts", diff --git a/plugins/search-indexer-backend/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts similarity index 100% rename from plugins/search-indexer-backend/src/index.test.ts rename to plugins/search-backend-node/src/index.test.ts diff --git a/plugins/search-indexer-backend/src/index.ts b/plugins/search-backend-node/src/index.ts similarity index 100% rename from plugins/search-indexer-backend/src/index.ts rename to plugins/search-backend-node/src/index.ts diff --git a/plugins/search-indexer-backend/src/registry.ts b/plugins/search-backend-node/src/registry.ts similarity index 100% rename from plugins/search-indexer-backend/src/registry.ts rename to plugins/search-backend-node/src/registry.ts diff --git a/plugins/search-indexer-backend/src/setupTests.ts b/plugins/search-backend-node/src/setupTests.ts similarity index 100% rename from plugins/search-indexer-backend/src/setupTests.ts rename to plugins/search-backend-node/src/setupTests.ts diff --git a/plugins/search-indexer-backend/src/types.ts b/plugins/search-backend-node/src/types.ts similarity index 100% rename from plugins/search-indexer-backend/src/types.ts rename to plugins/search-backend-node/src/types.ts diff --git a/plugins/search/README.md b/plugins/search/README.md index 498cc47586..d73995dcf5 100644 --- a/plugins/search/README.md +++ b/plugins/search/README.md @@ -17,7 +17,7 @@ In order to work on this new search platform, you will need to be aware of the f - **In-development app search route**: The new search platform will move the primary search page to the App-level, out of the search plugin. For now, to ensure the old experience is still easy to test, you can work on the new platform at `/search-next` instead of `/search`. - **App SearchPage Component**: You'll find a new search page under `/packages/app/src/components/search`. When ready, we'll add an equivalent page to the `@backstage/create-app` template. - **Backend**: Don't forget, a lot of functionality will be made available in backend plugins: - - `@backstage/plugin-search-indexer-backend`, which is responsible for the search index management + - `@backstage/plugin-search-backend-node`, which is responsible for the search index management - `@backstage/plugin-search-backend`, which is responsible for query processing As you work, be sure not to break the existing, frontend-only search page. From 51fc8de48c66ddb35ac0df10e739defe6224f9a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Wed, 17 Mar 2021 10:53:52 +0100 Subject: [PATCH 43/52] Tweak Collator/Decorator to be classes instead of methods Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 4 +- packages/search-common/src/types.ts | 18 +++--- .../catalog-backend/src/search/Collator.ts | 23 ++++---- plugins/search-backend-node/src/index.test.ts | 58 ++++++++++++++----- plugins/search-backend-node/src/registry.ts | 15 ++--- plugins/search-backend-node/src/types.ts | 13 ++--- 6 files changed, 78 insertions(+), 53 deletions(-) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index d1268e4f59..bde67360c7 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -19,14 +19,14 @@ import { // registerCollator, } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; -// import { SearchCollatorFactory } from '@backstage/plugin-catalog-backend'; +// import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; export default async function createPlugin({ logger }: PluginEnvironment) { // TODO: Within this PR, update to use REST API instead of Catalog Builder. /* registerCollator({ type: 'software-catalog', defaultRefreshIntervalSeconds: 600, - collator: SearchCollatorFactory(entitiesCatalog), + collator: new DefaultCatalogCollator(entitiesCatalog), });*/ // TODO: Make this a more proper refresh loop. diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 3a47187945..ad8dcf9fc6 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -62,15 +62,17 @@ export interface IndexableDocument { } /** - * Signature for the callback function that implementors must register to have - * their documents indexed. + * Interface that must be implemented in order to expose new documents to + * search. */ -export type IndexableDocumentCollator = () => Promise; +export interface DocumentCollator { + execute(): Promise; +} /** - * Signature for the callback function that implementors must register to - * decorate existing documents with additional metadata. + * Interface that must be implemented in order to decorate existing documents with + * additional metadata. */ -export type IndexableDocumentDecorator = ( - documents: IndexableDocument[], -) => Promise; +export interface DocumentDecorator { + execute(documents: IndexableDocument[]): Promise; +} diff --git a/plugins/catalog-backend/src/search/Collator.ts b/plugins/catalog-backend/src/search/Collator.ts index 1513060384..625cc09a6b 100644 --- a/plugins/catalog-backend/src/search/Collator.ts +++ b/plugins/catalog-backend/src/search/Collator.ts @@ -14,21 +14,22 @@ * limitations under the License. */ -import { - IndexableDocument, - IndexableDocumentCollator, -} from '@backstage/search-common'; +import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; import { EntitiesCatalog } from '../catalog'; export interface CatalogEntityDocument extends IndexableDocument { componentType: string; } -export const SearchCollatorFactory = ( - entitiesCatalog: EntitiesCatalog, -): IndexableDocumentCollator => { - return async (): Promise => { - const entities = await entitiesCatalog.entities(); +export class DefaultCatalogCollator implements DocumentCollator { + protected entitiesCatalog: EntitiesCatalog; + + constructor(entitiesCatalog: EntitiesCatalog) { + this.entitiesCatalog = entitiesCatalog; + } + + async execute() { + const entities = await this.entitiesCatalog.entities(); return entities.map( (entity): CatalogEntityDocument => { return { @@ -45,5 +46,5 @@ export const SearchCollatorFactory = ( }; }, ); - }; -}; + } +} diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts index 21db1657b3..01a4d9a486 100644 --- a/plugins/search-backend-node/src/index.test.ts +++ b/plugins/search-backend-node/src/index.test.ts @@ -14,23 +14,48 @@ * limitations under the License. */ +import { + DocumentCollator, + DocumentDecorator, + IndexableDocument, +} from '@backstage/search-common'; import { registerCollator, registerDecorator } from './'; import { Registry } from './registry'; -describe('external api', () => { +class TestDocumentCollator implements DocumentCollator { + async execute() { + return []; + } +} + +class TestDocumentDecorator implements DocumentDecorator { + async execute(documents: IndexableDocument[]) { + return documents; + } +} + +describe('search indexer external api', () => { + let testCollator: DocumentCollator; + let testDecorator: DocumentDecorator; + + beforeEach(() => { + testCollator = new TestDocumentCollator(); + testDecorator = new TestDocumentDecorator(); + }); + afterEach(() => { Registry.getInstance()._reset(); }); describe('registerCollator', () => { it('registers a collator', async () => { - const collatorSpy = jest.fn(async () => []); + const collatorSpy = jest.spyOn(testCollator, 'execute'); // Register a collator. registerCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, - collator: collatorSpy, + collator: testCollator, }); // Execute the registry and ensure the collator was invoked. @@ -41,19 +66,18 @@ describe('external api', () => { describe('registerDecorator', () => { it('registers a decorator', async () => { - const mockCollator = jest.fn(async () => []); - const decoratorSpy = jest.fn(async docs => docs); + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. registerCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, - collator: mockCollator, + collator: testCollator, }); // Register a decorator. registerDecorator({ - decorator: decoratorSpy, + decorator: testDecorator, }); // Execute the registry and ensure the decorator was invoked. @@ -68,20 +92,22 @@ describe('external api', () => { text: 'Test text.', location: '/test/location', }; - const mockCollator = jest.fn(async () => [docFixture]); - const decoratorSpy = jest.fn(async docs => docs); + jest + .spyOn(testCollator, 'execute') + .mockImplementation(async () => [docFixture]); + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. registerCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, - collator: mockCollator, + collator: testCollator, }); // Register a decorator for the same type. registerDecorator({ types: [expectedType], - decorator: decoratorSpy, + decorator: testDecorator, }); // Execute the registry and ensure the decorator was invoked. @@ -97,20 +123,22 @@ describe('external api', () => { text: 'Test text.', location: '/test/location', }; - const mockCollator = jest.fn(async () => [docFixture]); - const decoratorSpy = jest.fn(async docs => docs); + jest + .spyOn(testCollator, 'execute') + .mockImplementation(async () => [docFixture]); + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. registerCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, - collator: mockCollator, + collator: testCollator, }); // Register a decorator for a different type. registerDecorator({ types: ['not-the-expected-type'], - decorator: decoratorSpy, + decorator: testDecorator, }); // Execute the registry and ensure the decorator was not invoked. diff --git a/plugins/search-backend-node/src/registry.ts b/plugins/search-backend-node/src/registry.ts index 6125e8f6a3..3c17ae5569 100644 --- a/plugins/search-backend-node/src/registry.ts +++ b/plugins/search-backend-node/src/registry.ts @@ -14,23 +14,20 @@ * limitations under the License. */ -import { - IndexableDocumentCollator, - IndexableDocumentDecorator, -} from '@backstage/search-common'; +import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; import { RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; interface CollatorRegistryEntry { - collate: IndexableDocumentCollator; + collate: DocumentCollator; refreshInterval: number; } export class Registry { private collators: Record; - private decorators: Record; + private decorators: Record; private static instance: Registry; @@ -74,13 +71,13 @@ export class Registry { async execute() { return Promise.all( Object.keys(this.collators).map(async type => { - const decorators: IndexableDocumentDecorator[] = ( + const decorators: DocumentDecorator[] = ( this.decorators['*'] || [] ).concat(this.decorators[type] || []); - let documents = await this.collators[type].collate(); + let documents = await this.collators[type].collate.execute(); for (let i = 0; i < decorators.length; i++) { - documents = await decorators[i](documents); + documents = await decorators[i].execute(documents); } // TODO: push documents to a configured search engine. diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index 08ba475df8..e55faccfb4 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -14,10 +14,7 @@ * limitations under the License. */ -import { - IndexableDocumentCollator, - IndexableDocumentDecorator, -} from '@backstage/search-common'; +import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; /** * Parameters required to register a collator. @@ -34,9 +31,9 @@ export interface RegisterCollatorParameters { defaultRefreshIntervalSeconds: number; /** - * The collator function responsible for returning all documents of the given type. + * The collator class responsible for returning all documents of the given type. */ - collator: IndexableDocumentCollator; + collator: DocumentCollator; } /** @@ -44,9 +41,9 @@ export interface RegisterCollatorParameters { */ export interface RegisterDecoratorParameters { /** - * The decorator function responsible for appending or modifying documents of the given type(s). + * The decorator class responsible for appending or modifying documents of the given type(s). */ - decorator: IndexableDocumentDecorator; + decorator: DocumentDecorator; /** * (Optional) An array of document types that the given decorator should apply to. If none are provided, From 5a045906c2949fe099682e7df2b649950559848d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Wed, 17 Mar 2021 11:06:26 +0100 Subject: [PATCH 44/52] remove singleton pattern from indexer Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 10 +++---- plugins/search-backend-node/src/index.test.ts | 29 +++++++++---------- plugins/search-backend-node/src/index.ts | 17 +---------- plugins/search-backend-node/src/registry.ts | 20 +------------ 4 files changed, 19 insertions(+), 57 deletions(-) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index bde67360c7..fa97ead727 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -14,23 +14,21 @@ * limitations under the License. */ import { createRouter } from '@backstage/plugin-search-backend'; -import { - collateDocuments, - // registerCollator, -} from '@backstage/plugin-search-backend-node'; +import { Registry } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; // import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; export default async function createPlugin({ logger }: PluginEnvironment) { + const indexRegistry = new Registry(); // TODO: Within this PR, update to use REST API instead of Catalog Builder. - /* registerCollator({ + /* indexRegistry.addCollator({ type: 'software-catalog', defaultRefreshIntervalSeconds: 600, collator: new DefaultCatalogCollator(entitiesCatalog), });*/ // TODO: Make this a more proper refresh loop. - collateDocuments(); + indexRegistry.execute(); return await createRouter({ logger, diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts index 01a4d9a486..22a67af20b 100644 --- a/plugins/search-backend-node/src/index.test.ts +++ b/plugins/search-backend-node/src/index.test.ts @@ -19,7 +19,6 @@ import { DocumentDecorator, IndexableDocument, } from '@backstage/search-common'; -import { registerCollator, registerDecorator } from './'; import { Registry } from './registry'; class TestDocumentCollator implements DocumentCollator { @@ -35,31 +34,29 @@ class TestDocumentDecorator implements DocumentDecorator { } describe('search indexer external api', () => { + let testRegistry: Registry; let testCollator: DocumentCollator; let testDecorator: DocumentDecorator; beforeEach(() => { + testRegistry = new Registry(); testCollator = new TestDocumentCollator(); testDecorator = new TestDocumentDecorator(); }); - afterEach(() => { - Registry.getInstance()._reset(); - }); - describe('registerCollator', () => { it('registers a collator', async () => { const collatorSpy = jest.spyOn(testCollator, 'execute'); // Register a collator. - registerCollator({ + testRegistry.addCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, collator: testCollator, }); // Execute the registry and ensure the collator was invoked. - await Registry.getInstance().execute(); + await testRegistry.execute(); expect(collatorSpy).toHaveBeenCalled(); }); }); @@ -69,19 +66,19 @@ describe('search indexer external api', () => { const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. - registerCollator({ + testRegistry.addCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, collator: testCollator, }); // Register a decorator. - registerDecorator({ + testRegistry.addDecorator({ decorator: testDecorator, }); // Execute the registry and ensure the decorator was invoked. - await Registry.getInstance().execute(); + await testRegistry.execute(); expect(decoratorSpy).toHaveBeenCalled(); }); @@ -98,20 +95,20 @@ describe('search indexer external api', () => { const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. - registerCollator({ + testRegistry.addCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, collator: testCollator, }); // Register a decorator for the same type. - registerDecorator({ + testRegistry.addDecorator({ types: [expectedType], decorator: testDecorator, }); // Execute the registry and ensure the decorator was invoked. - await Registry.getInstance().execute(); + await testRegistry.execute(); expect(decoratorSpy).toHaveBeenCalled(); expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); @@ -129,20 +126,20 @@ describe('search indexer external api', () => { const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Register a collator. - registerCollator({ + testRegistry.addCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, collator: testCollator, }); // Register a decorator for a different type. - registerDecorator({ + testRegistry.addDecorator({ types: ['not-the-expected-type'], decorator: testDecorator, }); // Execute the registry and ensure the decorator was not invoked. - await Registry.getInstance().execute(); + await testRegistry.execute(); expect(decoratorSpy).not.toHaveBeenCalled(); }); }); diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index f016068db1..af6811e202 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -14,19 +14,4 @@ * limitations under the License. */ -import { Registry } from './registry'; -import { - RegisterCollatorParameters, - RegisterDecoratorParameters, -} from './types'; - -const registry = Registry.getInstance(); -export const registerCollator = (params: RegisterCollatorParameters) => { - registry.addCollator(params); -}; -export const registerDecorator = (params: RegisterDecoratorParameters) => { - registry.addDecorator(params); -}; -export const collateDocuments = () => { - registry.execute(); -}; +export { Registry } from './registry'; diff --git a/plugins/search-backend-node/src/registry.ts b/plugins/search-backend-node/src/registry.ts index 3c17ae5569..d62e9d1090 100644 --- a/plugins/search-backend-node/src/registry.ts +++ b/plugins/search-backend-node/src/registry.ts @@ -29,20 +29,11 @@ export class Registry { private collators: Record; private decorators: Record; - private static instance: Registry; - - private constructor() { + constructor() { this.collators = {}; this.decorators = {}; } - static getInstance(): Registry { - if (!Registry.instance) { - Registry.instance = new Registry(); - } - return Registry.instance; - } - addCollator({ type, collator, @@ -84,13 +75,4 @@ export class Registry { }), ); } - - /** - * Utility method for tests. Do not use otherwise. - * @private - */ - _reset() { - this.collators = {}; - this.decorators = {}; - } } From e450be8227c3882356175f4b76a470ceaf6826c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20N=C3=A4sman?= Date: Wed, 17 Mar 2021 14:05:14 +0100 Subject: [PATCH 45/52] move search-next page from app to plugin-search Signed-off-by: Eric Peterson --- packages/app/src/App.tsx | 8 +++++--- .../SearchPageNext/SearchPageNext.tsx | 10 ++++------ .../src/components/SearchPageNext/index.tsx | 17 +++++++++++++++++ plugins/search/src/components/index.tsx | 1 + plugins/search/src/index.ts | 7 ++++++- plugins/search/src/plugin.ts | 16 ++++++++++++++++ 6 files changed, 49 insertions(+), 10 deletions(-) rename packages/app/src/components/search/SearchPage.tsx => plugins/search/src/components/SearchPageNext/SearchPageNext.tsx (86%) create mode 100644 plugins/search/src/components/SearchPageNext/index.tsx diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 0e874e64e0..f5ad0337ee 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -39,7 +39,7 @@ import { GraphiQLPage } from '@backstage/plugin-graphiql'; import { LighthousePage } from '@backstage/plugin-lighthouse'; import { NewRelicPage } from '@backstage/plugin-newrelic'; import { ScaffolderPage, scaffolderPlugin } from '@backstage/plugin-scaffolder'; -import { SearchPage } from '@backstage/plugin-search'; +import { SearchPage, SearchPageNext } from '@backstage/plugin-search'; import { TechRadarPage } from '@backstage/plugin-tech-radar'; import { TechdocsPage } from '@backstage/plugin-techdocs'; import { UserSettingsPage } from '@backstage/plugin-user-settings'; @@ -50,7 +50,6 @@ import { Navigate, Route } from 'react-router'; import { apis } from './apis'; import { EntityPage } from './components/catalog/EntityPage'; import Root from './components/Root'; -import { SearchPage as SearchPageNext } from './components/search/SearchPage'; import { providers } from './identityProviders'; import * as plugins from './plugins'; @@ -113,7 +112,10 @@ const routes = ( } /> } /> } /> - } /> + } + /> } /> { +export const SearchPageNext = () => { const [queryString, setQueryString] = useQueryParamState('query'); const [searchQuery, setSearchQuery] = useState(queryString ?? ''); diff --git a/plugins/search/src/components/SearchPageNext/index.tsx b/plugins/search/src/components/SearchPageNext/index.tsx new file mode 100644 index 0000000000..464ba28750 --- /dev/null +++ b/plugins/search/src/components/SearchPageNext/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchPageNext } from './SearchPageNext'; diff --git a/plugins/search/src/components/index.tsx b/plugins/search/src/components/index.tsx index ac47860dc2..f571c61373 100644 --- a/plugins/search/src/components/index.tsx +++ b/plugins/search/src/components/index.tsx @@ -17,5 +17,6 @@ export * from './Filters'; export * from './SearchBar'; export * from './SearchPage'; +export * from './SearchPageNext'; export * from './SearchResult'; export * from './SidebarSearch'; diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts index 268de25a00..519d7fcf66 100644 --- a/plugins/search/src/index.ts +++ b/plugins/search/src/index.ts @@ -15,7 +15,12 @@ */ // TODO: export searchApiRef from ./apis once interface is stable and settled. -export { searchPlugin, searchPlugin as plugin, SearchPage } from './plugin'; +export { + searchPlugin, + searchPlugin as plugin, + SearchPage, + SearchPageNext, +} from './plugin'; export { Filters, FiltersButton, diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index e21ad934a0..e3e953e07d 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -23,12 +23,18 @@ import { import { SearchClient, searchApiRef } from './apis'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { SearchPage as SearchPageComponent } from './components/SearchPage'; +import { SearchPageNext as SearchPageNextComponent } from './components/SearchPageNext'; export const rootRouteRef = createRouteRef({ path: '/search', title: 'search', }); +export const rootNextRouteRef = createRouteRef({ + path: '/search-next', + title: 'search', +}); + export const searchPlugin = createPlugin({ id: 'search', apis: [ @@ -42,9 +48,11 @@ export const searchPlugin = createPlugin({ ], register({ router }) { router.addRoute(rootRouteRef, SearchPageComponent); + router.addRoute(rootNextRouteRef, SearchPageNextComponent); }, routes: { root: rootRouteRef, + nextRoot: rootNextRouteRef, }, }); @@ -54,3 +62,11 @@ export const SearchPage = searchPlugin.provide( mountPoint: rootRouteRef, }), ); + +export const SearchPageNext = searchPlugin.provide( + createRoutableExtension({ + component: () => + import('./components/SearchPageNext').then(m => m.SearchPageNext), + mountPoint: rootNextRouteRef, + }), +); From 9d214a88cea0f9c70ae963f33741ac085a670f1e Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 09:42:49 +0100 Subject: [PATCH 46/52] Update codeowners to remove non-existent files Signed-off-by: Eric Peterson --- .github/CODEOWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c9efd18a91..4e21603c63 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,7 +12,6 @@ /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core /plugins/search-* @backstage/techdocs-core -/packages/app/src/components/search/* @backstage/techdocs-core /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core From 3560405cab35a7e8781dc19b9dd7314d53d61e50 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 10:06:05 +0100 Subject: [PATCH 47/52] Call a builder a builder. Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 6 +- .../src/{registry.ts => IndexBuilder.ts} | 24 +++++-- plugins/search-backend-node/src/index.test.ts | 64 +++++++++---------- plugins/search-backend-node/src/index.ts | 2 +- 4 files changed, 55 insertions(+), 41 deletions(-) rename plugins/search-backend-node/src/{registry.ts => IndexBuilder.ts} (75%) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index fa97ead727..ad5111aaf3 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -14,12 +14,12 @@ * limitations under the License. */ import { createRouter } from '@backstage/plugin-search-backend'; -import { Registry } from '@backstage/plugin-search-backend-node'; +import { IndexBuilder } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; // import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; export default async function createPlugin({ logger }: PluginEnvironment) { - const indexRegistry = new Registry(); + const indexBuilder = new IndexBuilder(); // TODO: Within this PR, update to use REST API instead of Catalog Builder. /* indexRegistry.addCollator({ type: 'software-catalog', @@ -28,7 +28,7 @@ export default async function createPlugin({ logger }: PluginEnvironment) { });*/ // TODO: Make this a more proper refresh loop. - indexRegistry.execute(); + indexBuilder.build(); return await createRouter({ logger, diff --git a/plugins/search-backend-node/src/registry.ts b/plugins/search-backend-node/src/IndexBuilder.ts similarity index 75% rename from plugins/search-backend-node/src/registry.ts rename to plugins/search-backend-node/src/IndexBuilder.ts index d62e9d1090..2ec3cdc951 100644 --- a/plugins/search-backend-node/src/registry.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -20,13 +20,13 @@ import { RegisterDecoratorParameters, } from './types'; -interface CollatorRegistryEntry { +interface CollatorEnvelope { collate: DocumentCollator; refreshInterval: number; } -export class Registry { - private collators: Record; +export class IndexBuilder { + private collators: Record; private decorators: Record; constructor() { @@ -34,6 +34,10 @@ export class Registry { this.decorators = {}; } + /** + * Makes the index builder aware of a collator that should be executed at the + * given refresh interval. + */ addCollator({ type, collator, @@ -45,6 +49,11 @@ export class Registry { }; } + /** + * Makes the index builder aware of a decorator. If no types are provided, it + * will be applied to documents from all known collators, otherwise it will + * only be applied to documents of the given types. + */ addDecorator({ types = ['*'], decorator, @@ -58,8 +67,13 @@ export class Registry { }); } - // TODO: But like with coordination, timing, error handling, and what have you. - async execute() { + /** + * Starts the process of executing collators and decorators and building the + * search index. + * + * TODO: But like with coordination, timing, error handling, and what have you. + */ + async build() { return Promise.all( Object.keys(this.collators).map(async type => { const decorators: DocumentDecorator[] = ( diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts index 22a67af20b..1d64e80b79 100644 --- a/plugins/search-backend-node/src/index.test.ts +++ b/plugins/search-backend-node/src/index.test.ts @@ -19,7 +19,7 @@ import { DocumentDecorator, IndexableDocument, } from '@backstage/search-common'; -import { Registry } from './registry'; +import { IndexBuilder } from './IndexBuilder'; class TestDocumentCollator implements DocumentCollator { async execute() { @@ -33,56 +33,56 @@ class TestDocumentDecorator implements DocumentDecorator { } } -describe('search indexer external api', () => { - let testRegistry: Registry; +describe('IndexBuilder', () => { + let testIndexBuilder: IndexBuilder; let testCollator: DocumentCollator; let testDecorator: DocumentDecorator; beforeEach(() => { - testRegistry = new Registry(); + testIndexBuilder = new IndexBuilder(); testCollator = new TestDocumentCollator(); testDecorator = new TestDocumentDecorator(); }); - describe('registerCollator', () => { - it('registers a collator', async () => { + describe('addCollator', () => { + it('adds a collator', async () => { const collatorSpy = jest.spyOn(testCollator, 'execute'); - // Register a collator. - testRegistry.addCollator({ + // Add a collator. + testIndexBuilder.addCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, collator: testCollator, }); - // Execute the registry and ensure the collator was invoked. - await testRegistry.execute(); + // Build the index and ensure the collator was invoked. + await testIndexBuilder.build(); expect(collatorSpy).toHaveBeenCalled(); }); }); - describe('registerDecorator', () => { - it('registers a decorator', async () => { + describe('addDecorator', () => { + it('adds a decorator', async () => { const decoratorSpy = jest.spyOn(testDecorator, 'execute'); - // Register a collator. - testRegistry.addCollator({ + // Add a collator. + testIndexBuilder.addCollator({ type: 'anything', defaultRefreshIntervalSeconds: 600, collator: testCollator, }); - // Register a decorator. - testRegistry.addDecorator({ + // Add a decorator. + testIndexBuilder.addDecorator({ decorator: testDecorator, }); - // Execute the registry and ensure the decorator was invoked. - await testRegistry.execute(); + // Build the index and ensure the decorator was invoked. + await testIndexBuilder.build(); expect(decoratorSpy).toHaveBeenCalled(); }); - it('registers a type-specific decorator', async () => { + it('adds a type-specific decorator', async () => { const expectedType = 'an-expected-type'; const docFixture = { title: 'Test', @@ -94,26 +94,26 @@ describe('search indexer external api', () => { .mockImplementation(async () => [docFixture]); const decoratorSpy = jest.spyOn(testDecorator, 'execute'); - // Register a collator. - testRegistry.addCollator({ + // Add a collator. + testIndexBuilder.addCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, collator: testCollator, }); - // Register a decorator for the same type. - testRegistry.addDecorator({ + // Add a decorator for the same type. + testIndexBuilder.addDecorator({ types: [expectedType], decorator: testDecorator, }); - // Execute the registry and ensure the decorator was invoked. - await testRegistry.execute(); + // Build the index and ensure the decorator was invoked. + await testIndexBuilder.build(); expect(decoratorSpy).toHaveBeenCalled(); expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); - it('registers a type-specific decorator that should not be called', async () => { + it('adds a type-specific decorator that should not be called', async () => { const expectedType = 'an-expected-type'; const docFixture = { title: 'Test', @@ -125,21 +125,21 @@ describe('search indexer external api', () => { .mockImplementation(async () => [docFixture]); const decoratorSpy = jest.spyOn(testDecorator, 'execute'); - // Register a collator. - testRegistry.addCollator({ + // Add a collator. + testIndexBuilder.addCollator({ type: expectedType, defaultRefreshIntervalSeconds: 600, collator: testCollator, }); - // Register a decorator for a different type. - testRegistry.addDecorator({ + // Add a decorator for a different type. + testIndexBuilder.addDecorator({ types: ['not-the-expected-type'], decorator: testDecorator, }); - // Execute the registry and ensure the decorator was not invoked. - await testRegistry.execute(); + // Build the index and ensure the decorator was not invoked. + await testIndexBuilder.build(); expect(decoratorSpy).not.toHaveBeenCalled(); }); }); diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index af6811e202..924dfbc9dc 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { Registry } from './registry'; +export { IndexBuilder } from './IndexBuilder'; From 44ec2571b65de281dd68c669562c1a67dc0976ce Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 13:28:04 +0100 Subject: [PATCH 48/52] Use Catalog REST API instead of assuming local process. Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 23 ++++++++++++------- packages/search-common/src/types.ts | 10 -------- ...{Collator.ts => DefaultCatalogCollator.ts} | 23 +++++++++++-------- plugins/catalog-backend/src/search/index.ts | 2 +- 4 files changed, 30 insertions(+), 28 deletions(-) rename plugins/catalog-backend/src/search/{Collator.ts => DefaultCatalogCollator.ts} (66%) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index ad5111aaf3..671f4ea55e 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -13,22 +13,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { useHotCleanup } from '@backstage/backend-common'; import { createRouter } from '@backstage/plugin-search-backend'; import { IndexBuilder } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; -// import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; -export default async function createPlugin({ logger }: PluginEnvironment) { +export default async function createPlugin({ + logger, + discovery, +}: PluginEnvironment) { const indexBuilder = new IndexBuilder(); - // TODO: Within this PR, update to use REST API instead of Catalog Builder. - /* indexRegistry.addCollator({ + + indexBuilder.addCollator({ type: 'software-catalog', defaultRefreshIntervalSeconds: 600, - collator: new DefaultCatalogCollator(entitiesCatalog), - });*/ + collator: new DefaultCatalogCollator(discovery), + }); - // TODO: Make this a more proper refresh loop. - indexBuilder.build(); + // TODO: Move refresh loop logic into the builder. + const timerId = setInterval(() => { + indexBuilder.build(); + }, 60000); + useHotCleanup(module, () => clearInterval(timerId)); return await createRouter({ logger, diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index ad8dcf9fc6..ad784c3b36 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -49,16 +49,6 @@ export interface IndexableDocument { * is clicked). */ location: string; - - /** - * (Optional) The owner of the document (e.g. spec.owner on a catalog entity). - */ - owner?: string; - - /** - * (Optional) The lifecycle of the document (e.g. spec.lifecycle on a catalog entity). - */ - lifecycle?: string; } /** diff --git a/plugins/catalog-backend/src/search/Collator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts similarity index 66% rename from plugins/catalog-backend/src/search/Collator.ts rename to plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 625cc09a6b..72bf43f8f9 100644 --- a/plugins/catalog-backend/src/search/Collator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -14,35 +14,40 @@ * limitations under the License. */ +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; -import { EntitiesCatalog } from '../catalog'; +import fetch from 'cross-fetch'; export interface CatalogEntityDocument extends IndexableDocument { componentType: string; + namespace: string; + kind: string; } export class DefaultCatalogCollator implements DocumentCollator { - protected entitiesCatalog: EntitiesCatalog; + protected discovery: PluginEndpointDiscovery; - constructor(entitiesCatalog: EntitiesCatalog) { - this.entitiesCatalog = entitiesCatalog; + constructor(discovery: PluginEndpointDiscovery) { + this.discovery = discovery; } async execute() { - const entities = await this.entitiesCatalog.entities(); + const baseUrl = await this.discovery.getBaseUrl('catalog'); + const res = await fetch(`${baseUrl}/entities`); + const entities: Entity[] = await res.json(); return entities.map( (entity): CatalogEntityDocument => { return { title: entity.metadata.name, + // TODO: Use a config-based template approach for entity location. location: `/catalog/${ entity.metadata.namespace || 'default' }/component/${entity.metadata.name}`, text: entity.metadata.description || '', componentType: entity.spec?.type?.toString() || 'other', - ...(entity.spec?.owner && { owner: entity.spec.owner.toString() }), - ...(entity.metadata.lifecycle && { - lifecycle: entity.metadata.lifecycle.toString(), - }), + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, }; }, ); diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 568fafaed5..aed16aad76 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export * from './Collator'; +export * from './DefaultCatalogCollator'; From fb2ff7dfb0fa25a2c12525fbea138668e7ad9aa9 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 13:43:11 +0100 Subject: [PATCH 49/52] Provide useful logging in the IndexBuilder. Signed-off-by: Eric Peterson --- packages/backend/src/plugins/search.ts | 2 +- plugins/search-backend-node/package.json | 3 ++- .../search-backend-node/src/IndexBuilder.ts | 23 ++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 671f4ea55e..7bd473756d 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -23,7 +23,7 @@ export default async function createPlugin({ logger, discovery, }: PluginEnvironment) { - const indexBuilder = new IndexBuilder(); + const indexBuilder = new IndexBuilder({ logger }); indexBuilder.addCollator({ type: 'software-catalog', diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index 381c6baf7d..045cfb130c 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -19,7 +19,8 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/search-common": "^0.1.1" + "@backstage/search-common": "^0.1.1", + "winston": "^3.2.1" }, "devDependencies": { "@backstage/cli": "^0.6.0" diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 2ec3cdc951..43bf84a59d 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -15,6 +15,7 @@ */ import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; +import { Logger } from 'winston'; import { RegisterCollatorParameters, RegisterDecoratorParameters, @@ -25,13 +26,19 @@ interface CollatorEnvelope { refreshInterval: number; } +type IndexBuilderOptions = { + logger: Logger; +}; + export class IndexBuilder { private collators: Record; private decorators: Record; + private logger: Logger; - constructor() { + constructor({ logger }: IndexBuilderOptions) { this.collators = {}; this.decorators = {}; + this.logger = logger; } /** @@ -43,6 +50,9 @@ export class IndexBuilder { collator, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void { + this.logger.info( + `Added ${collator.constructor.name} collator for type ${type}`, + ); this.collators[type] = { refreshInterval: defaultRefreshIntervalSeconds, collate: collator, @@ -58,6 +68,11 @@ export class IndexBuilder { types = ['*'], decorator, }: RegisterDecoratorParameters): void { + this.logger.info( + `Added decorator ${decorator.constructor.name} to types ${types.join( + ', ', + )}`, + ); types.forEach(type => { if (this.decorators.hasOwnProperty(type)) { this.decorators[type].push(decorator); @@ -80,8 +95,14 @@ export class IndexBuilder { this.decorators['*'] || [] ).concat(this.decorators[type] || []); + this.logger.info( + `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, + ); let documents = await this.collators[type].collate.execute(); for (let i = 0; i < decorators.length; i++) { + this.logger.info( + `Decorating ${type} documents via ${decorators[i].constructor.name}`, + ); documents = await decorators[i].execute(documents); } From 900f230291023071ce77692f2d485763bb41b7bc Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 14:03:05 +0100 Subject: [PATCH 50/52] Fix bad rebase Signed-off-by: Eric Peterson --- plugins/catalog-backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index f178e05f68..08a8e21cc7 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -35,7 +35,7 @@ "@backstage/config": "^0.1.4", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", - "@backstage/plugin-search-indexer-backend": "^0.1.1", + "@backstage/plugin-search-backend-node": "^0.1.1", "@backstage/search-common": "^0.1.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", From c491df119f7fddfdb98b6ed85596e29311fd42f9 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Thu, 18 Mar 2021 15:23:28 +0100 Subject: [PATCH 51/52] Use void logger in test. --- plugins/search-backend-node/package.json | 1 + plugins/search-backend-node/src/index.test.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index 045cfb130c..12b3741c13 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -23,6 +23,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@backstage/backend-common": "0.5.6", "@backstage/cli": "^0.6.0" }, "files": [ diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts index 1d64e80b79..0941f23540 100644 --- a/plugins/search-backend-node/src/index.test.ts +++ b/plugins/search-backend-node/src/index.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { DocumentCollator, DocumentDecorator, @@ -39,7 +40,7 @@ describe('IndexBuilder', () => { let testDecorator: DocumentDecorator; beforeEach(() => { - testIndexBuilder = new IndexBuilder(); + testIndexBuilder = new IndexBuilder({ logger: getVoidLogger() }); testCollator = new TestDocumentCollator(); testDecorator = new TestDocumentDecorator(); }); From cb85673c71579d5ee4b2aa4cd97e9b3bc7bffb02 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Fri, 19 Mar 2021 11:36:56 +0100 Subject: [PATCH 52/52] Satisfy the test requirement. Signed-off-by: Eric Peterson --- packages/search-common/src/index.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 packages/search-common/src/index.test.ts diff --git a/packages/search-common/src/index.test.ts b/packages/search-common/src/index.test.ts new file mode 100644 index 0000000000..9e32ffa412 --- /dev/null +++ b/packages/search-common/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('search-common', () => { + // TODO: Test real things once they exist. + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +});