diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 371fae6e6b..4e21603c63 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,7 @@ /plugins/cost-insights @backstage/silver-lining /plugins/cloudbuild @trivago/ebarrios /plugins/search @backstage/techdocs-core +/plugins/search-* @backstage/techdocs-core /plugins/techdocs @backstage/techdocs-core /plugins/techdocs-backend @backstage/techdocs-core /packages/techdocs-common @backstage/techdocs-core diff --git a/docs/assets/search/architecture.drawio.svg b/docs/assets/search/architecture.drawio.svg index a624a375e1..1acad6d74b 100644 --- a/docs/assets/search/architecture.drawio.svg +++ b/docs/assets/search/architecture.drawio.svg @@ -1,4 +1,4 @@ - + @@ -268,7 +268,7 @@ -
+
Pass Search @@ -282,7 +282,7 @@
- + Pass Search... @@ -368,7 +368,7 @@ - + @@ -529,7 +529,7 @@ - + @@ -538,7 +538,7 @@
@backstage/
- plugin-search-indexer + plugin-search-backend-node
diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 7acde03953..f5ad0337ee 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -39,7 +39,7 @@ import { GraphiQLPage } from '@backstage/plugin-graphiql'; import { LighthousePage } from '@backstage/plugin-lighthouse'; import { NewRelicPage } from '@backstage/plugin-newrelic'; import { ScaffolderPage, scaffolderPlugin } from '@backstage/plugin-scaffolder'; -import { SearchPage } from '@backstage/plugin-search'; +import { SearchPage, SearchPageNext } from '@backstage/plugin-search'; import { TechRadarPage } from '@backstage/plugin-tech-radar'; import { TechdocsPage } from '@backstage/plugin-techdocs'; import { UserSettingsPage } from '@backstage/plugin-user-settings'; @@ -112,6 +112,10 @@ const routes = ( } /> } /> } /> + } + /> } /> createEnv('auth')); const proxyEnv = useHotMemoize(module, () => createEnv('proxy')); const rollbarEnv = useHotMemoize(module, () => createEnv('rollbar')); + const searchEnv = useHotMemoize(module, () => createEnv('search')); const techdocsEnv = useHotMemoize(module, () => createEnv('techdocs')); const todoEnv = useHotMemoize(module, () => createEnv('todo')); const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); @@ -91,6 +93,7 @@ async function main() { apiRouter.use('/rollbar', await rollbar(rollbarEnv)); apiRouter.use('/scaffolder', await scaffolder(scaffolderEnv)); apiRouter.use('/auth', await auth(authEnv)); + apiRouter.use('/search', await search(searchEnv)); apiRouter.use('/techdocs', await techdocs(techdocsEnv)); apiRouter.use('/todo', await todo(todoEnv)); apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts new file mode 100644 index 0000000000..7bd473756d --- /dev/null +++ b/packages/backend/src/plugins/search.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useHotCleanup } from '@backstage/backend-common'; +import { createRouter } from '@backstage/plugin-search-backend'; +import { IndexBuilder } from '@backstage/plugin-search-backend-node'; +import { PluginEnvironment } from '../types'; +import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; + +export default async function createPlugin({ + logger, + discovery, +}: PluginEnvironment) { + const indexBuilder = new IndexBuilder({ logger }); + + indexBuilder.addCollator({ + type: 'software-catalog', + defaultRefreshIntervalSeconds: 600, + collator: new DefaultCatalogCollator(discovery), + }); + + // TODO: Move refresh loop logic into the builder. + const timerId = setInterval(() => { + indexBuilder.build(); + }, 60000); + useHotCleanup(module, () => clearInterval(timerId)); + + return await createRouter({ + logger, + }); +} diff --git a/packages/search-common/.eslintrc.js b/packages/search-common/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/search-common/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/search-common/README.md b/packages/search-common/README.md new file mode 100644 index 0000000000..66828cc7af --- /dev/null +++ b/packages/search-common/README.md @@ -0,0 +1,3 @@ +# @backstage/search-common + +Common functionalities for Search, to be shared between various search-enabled plugins. diff --git a/packages/search-common/package.json b/packages/search-common/package.json new file mode 100644 index 0000000000..18e3557a49 --- /dev/null +++ b/packages/search-common/package.json @@ -0,0 +1,49 @@ +{ + "name": "@backstage/search-common", + "description": "Common functionalities for Search, to be shared between various search-enabled plugins", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/search-common" + }, + "keywords": [ + "backstage", + "search" + ], + "license": "Apache-2.0", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli build --outputs cjs,types", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "bugs": { + "url": "https://github.com/backstage/backstage/issues" + }, + "dependencies": { + "@backstage/config": "^0.1.3" + }, + "devDependencies": { + "@backstage/cli": "^0.6.2" + }, + "jest": { + "roots": [ + ".." + ] + } +} diff --git a/packages/search-common/src/index.test.ts b/packages/search-common/src/index.test.ts new file mode 100644 index 0000000000..9e32ffa412 --- /dev/null +++ b/packages/search-common/src/index.test.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as anything from './'; + +describe('search-common', () => { + // TODO: Test real things once they exist. + it('should exist', () => { + expect(anything).toBeTruthy(); + }); +}); diff --git a/packages/search-common/src/index.ts b/packages/search-common/src/index.ts new file mode 100644 index 0000000000..e6432477f2 --- /dev/null +++ b/packages/search-common/src/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './types'; diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts new file mode 100644 index 0000000000..ad784c3b36 --- /dev/null +++ b/packages/search-common/src/types.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { JsonObject } from '@backstage/config'; + +export interface SearchQuery { + term: string; + filters?: JsonObject; + pageCursor: string; +} + +export interface SearchResult { + document: IndexableDocument; +} + +export interface SearchResultSet { + results: SearchResult[]; +} + +/** + * Base properties that all indexed documents must include, as well as some + * common properties that documents are encouraged to use where appropriate. + */ +export interface IndexableDocument { + /** + * The primary name of the document (e.g. name, title, identifier, etc). + */ + title: string; + + /** + * Free-form text of the document (e.g. description, content, etc). + */ + text: string; + + /** + * The relative or absolute URL of the document (target when a search result + * is clicked). + */ + location: string; +} + +/** + * Interface that must be implemented in order to expose new documents to + * search. + */ +export interface DocumentCollator { + execute(): Promise; +} + +/** + * Interface that must be implemented in order to decorate existing documents with + * additional metadata. + */ +export interface DocumentDecorator { + execute(documents: IndexableDocument[]): Promise; +} diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index ce146a8a5e..479fc7ba4d 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -35,6 +35,8 @@ "@backstage/config": "^0.1.4", "@backstage/errors": "^0.1.1", "@backstage/integration": "^0.5.1", + "@backstage/plugin-search-backend-node": "^0.1.1", + "@backstage/search-common": "^0.1.1", "@octokit/graphql": "^4.5.8", "@types/express": "^4.17.6", "@types/ldapjs": "^1.0.9", diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index f621930d10..51df909d05 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -17,5 +17,6 @@ export * from './catalog'; export * from './database'; export * from './ingestion'; +export * from './search'; export * from './service'; export * from './util'; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts new file mode 100644 index 0000000000..72bf43f8f9 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch'; + +export interface CatalogEntityDocument extends IndexableDocument { + componentType: string; + namespace: string; + kind: string; +} + +export class DefaultCatalogCollator implements DocumentCollator { + protected discovery: PluginEndpointDiscovery; + + constructor(discovery: PluginEndpointDiscovery) { + this.discovery = discovery; + } + + async execute() { + const baseUrl = await this.discovery.getBaseUrl('catalog'); + const res = await fetch(`${baseUrl}/entities`); + const entities: Entity[] = await res.json(); + return entities.map( + (entity): CatalogEntityDocument => { + return { + title: entity.metadata.name, + // TODO: Use a config-based template approach for entity location. + location: `/catalog/${ + entity.metadata.namespace || 'default' + }/component/${entity.metadata.name}`, + text: entity.metadata.description || '', + componentType: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + }; + }, + ); + } +} diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts new file mode 100644 index 0000000000..aed16aad76 --- /dev/null +++ b/plugins/catalog-backend/src/search/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './DefaultCatalogCollator'; diff --git a/plugins/search-backend-node/.eslintrc.js b/plugins/search-backend-node/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-backend-node/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-backend-node/README.md b/plugins/search-backend-node/README.md new file mode 100644 index 0000000000..126e8b9988 --- /dev/null +++ b/plugins/search-backend-node/README.md @@ -0,0 +1,21 @@ +# search-backend-node + +This plugin is part of a suite of plugins that comprise the Backstage search +platform, which is still very much under development. This plugin specifically +is responsible for: + +- Allowing other backend plugins to register the fact that they have documents + that they'd like to be indexed by a search engine (known as `collators`) +- Allowing other backend plugins to register the fact that they have metadata + that they'd like to augment existing documents in the search index with + (known as `decorators`) +- A scheduler that, at configurable intervals, compiles documents to be indexed + and passes them to a search engine for indexing +- Types for all of the above + +Documentation on how to develop and improve the search platform is currently +centralized in the `search` plugin README.md. + +For a better overview of how the search platform is put together, check the +[Backstage Search Architecture](https://backstage.io/docs/features/search/architecture) +documentation. diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json new file mode 100644 index 0000000000..12b3741c13 --- /dev/null +++ b/plugins/search-backend-node/package.json @@ -0,0 +1,32 @@ +{ + "name": "@backstage/plugin-search-backend-node", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/search-common": "^0.1.1", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/backend-common": "0.5.6", + "@backstage/cli": "^0.6.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts new file mode 100644 index 0000000000..43bf84a59d --- /dev/null +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -0,0 +1,113 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; +import { Logger } from 'winston'; +import { + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; + +interface CollatorEnvelope { + collate: DocumentCollator; + refreshInterval: number; +} + +type IndexBuilderOptions = { + logger: Logger; +}; + +export class IndexBuilder { + private collators: Record; + private decorators: Record; + private logger: Logger; + + constructor({ logger }: IndexBuilderOptions) { + this.collators = {}; + this.decorators = {}; + this.logger = logger; + } + + /** + * Makes the index builder aware of a collator that should be executed at the + * given refresh interval. + */ + addCollator({ + type, + collator, + defaultRefreshIntervalSeconds, + }: RegisterCollatorParameters): void { + this.logger.info( + `Added ${collator.constructor.name} collator for type ${type}`, + ); + this.collators[type] = { + refreshInterval: defaultRefreshIntervalSeconds, + collate: collator, + }; + } + + /** + * Makes the index builder aware of a decorator. If no types are provided, it + * will be applied to documents from all known collators, otherwise it will + * only be applied to documents of the given types. + */ + addDecorator({ + types = ['*'], + decorator, + }: RegisterDecoratorParameters): void { + this.logger.info( + `Added decorator ${decorator.constructor.name} to types ${types.join( + ', ', + )}`, + ); + types.forEach(type => { + if (this.decorators.hasOwnProperty(type)) { + this.decorators[type].push(decorator); + } else { + this.decorators[type] = [decorator]; + } + }); + } + + /** + * Starts the process of executing collators and decorators and building the + * search index. + * + * TODO: But like with coordination, timing, error handling, and what have you. + */ + async build() { + return Promise.all( + Object.keys(this.collators).map(async type => { + const decorators: DocumentDecorator[] = ( + this.decorators['*'] || [] + ).concat(this.decorators[type] || []); + + this.logger.info( + `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, + ); + let documents = await this.collators[type].collate.execute(); + for (let i = 0; i < decorators.length; i++) { + this.logger.info( + `Decorating ${type} documents via ${decorators[i].constructor.name}`, + ); + documents = await decorators[i].execute(documents); + } + + // TODO: push documents to a configured search engine. + }), + ); + } +} diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/index.test.ts new file mode 100644 index 0000000000..0941f23540 --- /dev/null +++ b/plugins/search-backend-node/src/index.test.ts @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + DocumentCollator, + DocumentDecorator, + IndexableDocument, +} from '@backstage/search-common'; +import { IndexBuilder } from './IndexBuilder'; + +class TestDocumentCollator implements DocumentCollator { + async execute() { + return []; + } +} + +class TestDocumentDecorator implements DocumentDecorator { + async execute(documents: IndexableDocument[]) { + return documents; + } +} + +describe('IndexBuilder', () => { + let testIndexBuilder: IndexBuilder; + let testCollator: DocumentCollator; + let testDecorator: DocumentDecorator; + + beforeEach(() => { + testIndexBuilder = new IndexBuilder({ logger: getVoidLogger() }); + testCollator = new TestDocumentCollator(); + testDecorator = new TestDocumentDecorator(); + }); + + describe('addCollator', () => { + it('adds a collator', async () => { + const collatorSpy = jest.spyOn(testCollator, 'execute'); + + // Add a collator. + testIndexBuilder.addCollator({ + type: 'anything', + defaultRefreshIntervalSeconds: 600, + collator: testCollator, + }); + + // Build the index and ensure the collator was invoked. + await testIndexBuilder.build(); + expect(collatorSpy).toHaveBeenCalled(); + }); + }); + + describe('addDecorator', () => { + it('adds a decorator', async () => { + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + + // Add a collator. + testIndexBuilder.addCollator({ + type: 'anything', + defaultRefreshIntervalSeconds: 600, + collator: testCollator, + }); + + // Add a decorator. + testIndexBuilder.addDecorator({ + decorator: testDecorator, + }); + + // Build the index and ensure the decorator was invoked. + await testIndexBuilder.build(); + expect(decoratorSpy).toHaveBeenCalled(); + }); + + it('adds a type-specific decorator', async () => { + const expectedType = 'an-expected-type'; + const docFixture = { + title: 'Test', + text: 'Test text.', + location: '/test/location', + }; + jest + .spyOn(testCollator, 'execute') + .mockImplementation(async () => [docFixture]); + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + + // Add a collator. + testIndexBuilder.addCollator({ + type: expectedType, + defaultRefreshIntervalSeconds: 600, + collator: testCollator, + }); + + // Add a decorator for the same type. + testIndexBuilder.addDecorator({ + types: [expectedType], + decorator: testDecorator, + }); + + // Build the index and ensure the decorator was invoked. + await testIndexBuilder.build(); + expect(decoratorSpy).toHaveBeenCalled(); + expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); + }); + + it('adds a type-specific decorator that should not be called', async () => { + const expectedType = 'an-expected-type'; + const docFixture = { + title: 'Test', + text: 'Test text.', + location: '/test/location', + }; + jest + .spyOn(testCollator, 'execute') + .mockImplementation(async () => [docFixture]); + const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + + // Add a collator. + testIndexBuilder.addCollator({ + type: expectedType, + defaultRefreshIntervalSeconds: 600, + collator: testCollator, + }); + + // Add a decorator for a different type. + testIndexBuilder.addDecorator({ + types: ['not-the-expected-type'], + decorator: testDecorator, + }); + + // Build the index and ensure the decorator was not invoked. + await testIndexBuilder.build(); + expect(decoratorSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts new file mode 100644 index 0000000000..924dfbc9dc --- /dev/null +++ b/plugins/search-backend-node/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { IndexBuilder } from './IndexBuilder'; diff --git a/plugins/search-backend-node/src/setupTests.ts b/plugins/search-backend-node/src/setupTests.ts new file mode 100644 index 0000000000..4e230aca20 --- /dev/null +++ b/plugins/search-backend-node/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts new file mode 100644 index 0000000000..e55faccfb4 --- /dev/null +++ b/plugins/search-backend-node/src/types.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; + +/** + * Parameters required to register a collator. + */ +export interface RegisterCollatorParameters { + /** + * The type of document to be indexed (used to name indices, to configure refresh loop, etc). + */ + type: string; + + /** + * The default interval (in seconds) that the provided collator will be called (can be overridden in config). + */ + defaultRefreshIntervalSeconds: number; + + /** + * The collator class responsible for returning all documents of the given type. + */ + collator: DocumentCollator; +} + +/** + * Parameters required to register a decorator + */ +export interface RegisterDecoratorParameters { + /** + * The decorator class responsible for appending or modifying documents of the given type(s). + */ + decorator: DocumentDecorator; + + /** + * (Optional) An array of document types that the given decorator should apply to. If none are provided, + * the decorator will be applied to all types. + */ + types?: string[]; +} diff --git a/plugins/search-backend/.eslintrc.js b/plugins/search-backend/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/plugins/search-backend/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/plugins/search-backend/README.md b/plugins/search-backend/README.md new file mode 100644 index 0000000000..678639d36b --- /dev/null +++ b/plugins/search-backend/README.md @@ -0,0 +1,15 @@ +# search-backend + +This plugin is part of a suite of plugins that comprise the Backstage search +platform, which is still very much under development. This plugin specifically +is responsible for: + +- Exposing a JSON API for querying a search engine +- Types related to interacting with that API + +Documentation on how to develop and improve the search platform is currently +centralized in the `search` plugin README.md. + +For a better overview of how the search platform is put together, check the +[Backstage Search Architecture](https://backstage.io/docs/features/search/architecture) +documentation. diff --git a/plugins/search-backend/package.json b/plugins/search-backend/package.json new file mode 100644 index 0000000000..9a1150de8a --- /dev/null +++ b/plugins/search-backend/package.json @@ -0,0 +1,38 @@ +{ + "name": "@backstage/plugin-search-backend", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "scripts": { + "start": "backstage-cli backend:dev", + "build": "backstage-cli backend:build", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "dependencies": { + "@backstage/backend-common": "^0.5.3", + "@backstage/search-common": "^0.1.1", + "@types/express": "^4.17.6", + "express": "^4.17.1", + "express-promise-router": "^3.0.3", + "winston": "^3.2.1", + "yn": "^4.0.0" + }, + "devDependencies": { + "@backstage/cli": "^0.6.0", + "@types/supertest": "^2.0.8", + "supertest": "^4.0.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/search-backend/src/index.ts b/plugins/search-backend/src/index.ts new file mode 100644 index 0000000000..38e2cdf4cb --- /dev/null +++ b/plugins/search-backend/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './service/router'; diff --git a/plugins/search-backend/src/run.ts b/plugins/search-backend/src/run.ts new file mode 100644 index 0000000000..a59d90d09a --- /dev/null +++ b/plugins/search-backend/src/run.ts @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getRootLogger } from '@backstage/backend-common'; +import yn from 'yn'; +import { startStandaloneServer } from './service/standaloneServer'; + +const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 7000; +const enableCors = yn(process.env.PLUGIN_CORS, { default: false }); +const logger = getRootLogger(); + +startStandaloneServer({ port, enableCors, logger }).catch(err => { + logger.error(err); + process.exit(1); +}); + +process.on('SIGINT', () => { + logger.info('CTRL+C pressed; exiting.'); + process.exit(0); +}); diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts new file mode 100644 index 0000000000..ba7032eb93 --- /dev/null +++ b/plugins/search-backend/src/service/router.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import express from 'express'; +import request from 'supertest'; + +import { createRouter } from './router'; + +describe('createRouter', () => { + let app: express.Express; + + beforeAll(async () => { + const router = await createRouter({ + logger: getVoidLogger(), + }); + app = express().use(router); + }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('GET /query', () => { + it('returns empty results array', async () => { + const response = await request(app).get('/query'); + + expect(response.status).toEqual(200); + expect(response.body).toMatchObject({ results: [] }); + }); + }); +}); diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts new file mode 100644 index 0000000000..1a0c67bdb8 --- /dev/null +++ b/plugins/search-backend/src/service/router.ts @@ -0,0 +1,58 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import Router from 'express-promise-router'; +import { Logger } from 'winston'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; + +type RouterOptions = { + logger: Logger; +}; + +export async function createRouter({ + logger, +}: RouterOptions): Promise { + const router = Router(); + + router.get( + '/query', + async ( + req: express.Request, + res: express.Response, + ) => { + // TODO: Actually transform req.params into search engine specific query. + const { term, filters = {}, pageCursor = '' } = req.query; + logger.info( + `Search request received: ${term}, ${JSON.stringify( + filters, + )}, ${pageCursor}`, + ); + + try { + // TODO: Actually query search engine. + // TODO: And actually transform results into frontend-readable result + res.send({ + results: [], + }); + } catch (err) { + throw new Error(`There was a problem performing the search query.`); + } + }, + ); + + return router; +} diff --git a/plugins/search-backend/src/service/standaloneServer.ts b/plugins/search-backend/src/service/standaloneServer.ts new file mode 100644 index 0000000000..b0c53a6adc --- /dev/null +++ b/plugins/search-backend/src/service/standaloneServer.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createServiceBuilder } from '@backstage/backend-common'; +import { Server } from 'http'; +import { Logger } from 'winston'; +import { createRouter } from './router'; + +export interface ServerOptions { + port: number; + enableCors: boolean; + logger: Logger; +} + +export async function startStandaloneServer( + options: ServerOptions, +): Promise { + const logger = options.logger.child({ service: 'search-backend' }); + logger.debug('Starting application server...'); + const router = await createRouter({ + logger, + }); + + const service = createServiceBuilder(module) + .enableCors({ origin: 'http://localhost:3000' }) + .addRouter('/search', router); + + return await service.start().catch(err => { + logger.error(err); + process.exit(1); + }); +} + +module.hot?.accept(); diff --git a/plugins/search-backend/src/setupTests.ts b/plugins/search-backend/src/setupTests.ts new file mode 100644 index 0000000000..ba33cf996b --- /dev/null +++ b/plugins/search-backend/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export {}; diff --git a/plugins/search/README.md b/plugins/search/README.md index fb4b047811..d73995dcf5 100644 --- a/plugins/search/README.md +++ b/plugins/search/README.md @@ -2,8 +2,22 @@ **This plugin is still under development.** -You can follow the progress under the Global search in Backstage [milestone](https://github.com/backstage/backstage/milestone/21) or reach out to us in the #search Discord channel. +You can follow the progress and contribute at the Backstage [Search Project Board](https://github.com/backstage/backstage/projects/6) or reach out to us in the [`#search` Discord channel](https://discord.com/channels/687207715902193673/770283289327566848). ## Getting started -Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search)to check out the plugin. +Run `yarn start` in the root directory, and then navigate to [/search](http://localhost:3000/search) to check out the plugin. + +### Working on the search platform + +The above search experience is 100% client-side and only looks at the Software Catalog. We are actively developing [a more complete search platform](https://backstage.io/docs/features/search/search-overview), which will replace the current experience when ready. + +In order to work on this new search platform, you will need to be aware of the following: + +- **In-development app search route**: The new search platform will move the primary search page to the App-level, out of the search plugin. For now, to ensure the old experience is still easy to test, you can work on the new platform at `/search-next` instead of `/search`. +- **App SearchPage Component**: You'll find a new search page under `/packages/app/src/components/search`. When ready, we'll add an equivalent page to the `@backstage/create-app` template. +- **Backend**: Don't forget, a lot of functionality will be made available in backend plugins: + - `@backstage/plugin-search-backend-node`, which is responsible for the search index management + - `@backstage/plugin-search-backend`, which is responsible for query processing + +As you work, be sure not to break the existing, frontend-only search page. diff --git a/plugins/search/package.json b/plugins/search/package.json index 8003843d8d..d88ed9b241 100644 --- a/plugins/search/package.json +++ b/plugins/search/package.json @@ -32,6 +32,8 @@ "@backstage/core": "^0.7.2", "@backstage/catalog-model": "^0.7.3", "@backstage/plugin-catalog-react": "^0.1.2", + "@backstage/plugin-search-backend": "^0.1.1", + "@backstage/search-common": "^0.1.1", "@backstage/theme": "^0.2.4", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/search/src/apis.ts b/plugins/search/src/apis.ts index 9d88227a44..b15be349cf 100644 --- a/plugins/search/src/apis.ts +++ b/plugins/search/src/apis.ts @@ -14,8 +14,17 @@ * limitations under the License. */ +import { createApiRef, DiscoveryApi } from '@backstage/core'; import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; + import { CatalogApi } from '@backstage/plugin-catalog-react'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; +import qs from 'qs'; + +export const searchApiRef = createApiRef({ + id: 'plugin.search.queryservice', + description: 'Used to make requests against the search API', +}); export type Result = { name: string; @@ -28,11 +37,18 @@ export type Result = { export type SearchResults = Array; -class SearchApi { - private catalogApi: CatalogApi; +export interface SearchApi { + getSearchResult(): Promise; + _alphaPerformSearch(query: SearchQuery): Promise; +} - constructor(catalogApi: CatalogApi) { - this.catalogApi = catalogApi; +export class SearchClient implements SearchApi { + private readonly catalogApi: CatalogApi; + private readonly discoveryApi: DiscoveryApi; + + constructor(options: { catalogApi: CatalogApi; discoveryApi: DiscoveryApi }) { + this.catalogApi = options.catalogApi; + this.discoveryApi = options.discoveryApi; } private async entities() { @@ -53,9 +69,17 @@ class SearchApi { })); } - public getSearchResult(): Promise { + getSearchResult(): Promise { return this.entities(); } -} -export default SearchApi; + // TODO: Productionalize as we implement search milestones. + async _alphaPerformSearch(query: SearchQuery): Promise { + const queryString = qs.stringify(query); + const url = `${await this.discoveryApi.getBaseUrl( + 'search/query', + )}?${queryString}`; + const response = await fetch(url); + return response.json(); + } +} diff --git a/plugins/search/src/components/SearchPageNext/SearchPageNext.tsx b/plugins/search/src/components/SearchPageNext/SearchPageNext.tsx new file mode 100644 index 0000000000..a6655e37ac --- /dev/null +++ b/plugins/search/src/components/SearchPageNext/SearchPageNext.tsx @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Content, + Header, + Lifecycle, + Page, + useQueryParamState, +} from '@backstage/core'; +import { Grid } from '@material-ui/core'; +import React, { useEffect, useState } from 'react'; +import { useDebounce } from 'react-use'; +import { SearchBar } from '../SearchBar'; +import { SearchResult } from '../SearchResult'; + +export const SearchPageNext = () => { + const [queryString, setQueryString] = useQueryParamState('query'); + const [searchQuery, setSearchQuery] = useState(queryString ?? ''); + + const handleSearch = (event: React.ChangeEvent) => { + event.preventDefault(); + setSearchQuery(event.target.value); + }; + + useEffect(() => setSearchQuery(queryString ?? ''), [queryString]); + + useDebounce( + () => { + setQueryString(searchQuery); + }, + 200, + [searchQuery], + ); + + const handleClearSearchBar = () => { + setSearchQuery(''); + }; + + return ( + +
} /> + + + + + + + + + + + + ); +}; diff --git a/plugins/search/src/components/SearchPageNext/index.tsx b/plugins/search/src/components/SearchPageNext/index.tsx new file mode 100644 index 0000000000..464ba28750 --- /dev/null +++ b/plugins/search/src/components/SearchPageNext/index.tsx @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { SearchPageNext } from './SearchPageNext'; diff --git a/plugins/search/src/components/SearchResult/SearchResult.tsx b/plugins/search/src/components/SearchResult/SearchResult.tsx index 7f4d31a6df..0aa209e6fe 100644 --- a/plugins/search/src/components/SearchResult/SearchResult.tsx +++ b/plugins/search/src/components/SearchResult/SearchResult.tsx @@ -21,12 +21,12 @@ import { TableColumn, useApi, } from '@backstage/core'; -import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { Divider, Grid, makeStyles, Typography } from '@material-ui/core'; import { Alert } from '@material-ui/lab'; import React, { useEffect, useState } from 'react'; import { useAsync } from 'react-use'; -import SearchApi, { Result, SearchResults } from '../../apis'; +import { Result, SearchResults, searchApiRef } from '../../apis'; + import { Filters, FiltersButton, FiltersState } from '../Filters'; const useStyles = makeStyles(theme => ({ @@ -116,7 +116,7 @@ const TableHeader = ({ }; export const SearchResult = ({ searchQuery }: SearchResultProps) => { - const catalogApi = useApi(catalogApiRef); + const searchApi = useApi(searchApiRef); const [showFilters, toggleFilters] = useState(false); const [selectedFilters, setSelectedFilters] = useState({ @@ -126,8 +126,6 @@ export const SearchResult = ({ searchQuery }: SearchResultProps) => { const [filteredResults, setFilteredResults] = useState([]); - const searchApi = new SearchApi(catalogApi); - const { loading, error, value: results } = useAsync(() => { return searchApi.getSearchResult(); }, []); diff --git a/plugins/search/src/components/index.tsx b/plugins/search/src/components/index.tsx index ac47860dc2..f571c61373 100644 --- a/plugins/search/src/components/index.tsx +++ b/plugins/search/src/components/index.tsx @@ -17,5 +17,6 @@ export * from './Filters'; export * from './SearchBar'; export * from './SearchPage'; +export * from './SearchPageNext'; export * from './SearchResult'; export * from './SidebarSearch'; diff --git a/plugins/search/src/index.ts b/plugins/search/src/index.ts index 572d75bea5..519d7fcf66 100644 --- a/plugins/search/src/index.ts +++ b/plugins/search/src/index.ts @@ -13,7 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { searchPlugin, searchPlugin as plugin, SearchPage } from './plugin'; + +// TODO: export searchApiRef from ./apis once interface is stable and settled. +export { + searchPlugin, + searchPlugin as plugin, + SearchPage, + SearchPageNext, +} from './plugin'; export { Filters, FiltersButton, diff --git a/plugins/search/src/plugin.ts b/plugins/search/src/plugin.ts index 37503b7751..e3e953e07d 100644 --- a/plugins/search/src/plugin.ts +++ b/plugins/search/src/plugin.ts @@ -14,24 +14,45 @@ * limitations under the License. */ import { + createApiFactory, createPlugin, createRouteRef, createRoutableExtension, + discoveryApiRef, } from '@backstage/core'; +import { SearchClient, searchApiRef } from './apis'; +import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { SearchPage as SearchPageComponent } from './components/SearchPage'; +import { SearchPageNext as SearchPageNextComponent } from './components/SearchPageNext'; export const rootRouteRef = createRouteRef({ path: '/search', title: 'search', }); +export const rootNextRouteRef = createRouteRef({ + path: '/search-next', + title: 'search', +}); + export const searchPlugin = createPlugin({ id: 'search', + apis: [ + createApiFactory({ + api: searchApiRef, + deps: { catalogApi: catalogApiRef, discoveryApi: discoveryApiRef }, + factory: ({ catalogApi, discoveryApi }) => { + return new SearchClient({ catalogApi, discoveryApi }); + }, + }), + ], register({ router }) { router.addRoute(rootRouteRef, SearchPageComponent); + router.addRoute(rootNextRouteRef, SearchPageNextComponent); }, routes: { root: rootRouteRef, + nextRoot: rootNextRouteRef, }, }); @@ -41,3 +62,11 @@ export const SearchPage = searchPlugin.provide( mountPoint: rootRouteRef, }), ); + +export const SearchPageNext = searchPlugin.provide( + createRoutableExtension({ + component: () => + import('./components/SearchPageNext').then(m => m.SearchPageNext), + mountPoint: rootNextRouteRef, + }), +);