diff --git a/.changeset/beige-snails-drum.md b/.changeset/beige-snails-drum.md new file mode 100644 index 0000000000..3476c9383b --- /dev/null +++ b/.changeset/beige-snails-drum.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Move `@types/json-schema` to be a dev dependency diff --git a/.changeset/brown-dryers-serve.md b/.changeset/brown-dryers-serve.md new file mode 100644 index 0000000000..40c02538b8 --- /dev/null +++ b/.changeset/brown-dryers-serve.md @@ -0,0 +1,10 @@ +--- +'@backstage/catalog-client': minor +--- + +**BREAKING**: Removed the old deprecated request/response types: + +- `CatalogEntitiesRequest` - please use `GetEntitiesRequest` instead +- `CatalogEntityAncestorsRequest` - please use `GetEntityAncestorsRequest` instead +- `CatalogEntityAncestorsResponse` - please use `GetEntityAncestorsResponse` instead +- `CatalogListResponse` - please use `GetEntitiesResponse` instead diff --git a/.changeset/dull-months-knock.md b/.changeset/dull-months-knock.md new file mode 100644 index 0000000000..3b957518c2 --- /dev/null +++ b/.changeset/dull-months-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Added a new interface that allows for customization of when to build techdocs diff --git a/.changeset/fluffy-trees-occur.md b/.changeset/fluffy-trees-occur.md new file mode 100644 index 0000000000..6e41a61069 --- /dev/null +++ b/.changeset/fluffy-trees-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add config option `groupExpand` to allow expanding a single relationship diff --git a/.changeset/gentle-icons-vanish.md b/.changeset/gentle-icons-vanish.md new file mode 100644 index 0000000000..27ea78863b --- /dev/null +++ b/.changeset/gentle-icons-vanish.md @@ -0,0 +1,20 @@ +--- +'@backstage/plugin-api-docs': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-badges-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `getEntityByRef` instead of `getEntityByName` in the catalog client diff --git a/.changeset/gorgeous-boats-hide.md b/.changeset/gorgeous-boats-hide.md new file mode 100644 index 0000000000..5c60240433 --- /dev/null +++ b/.changeset/gorgeous-boats-hide.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-org': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-techdocs': patch +--- + +- **DEPRECATION**: Deprecated `formatEntityRefTitle` in favor of the new `humanizeEntityRef` method instead. Please migrate to using the new method instead. diff --git a/.changeset/honest-students-clean.md b/.changeset/honest-students-clean.md new file mode 100644 index 0000000000..ca8db28343 --- /dev/null +++ b/.changeset/honest-students-clean.md @@ -0,0 +1,7 @@ +--- +'@backstage/integration': minor +'@backstage/backend-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +ensure `apiBaseUrl` being set for Bitbucket integrations, replace hardcoded defaults diff --git a/.changeset/large-dancers-learn.md b/.changeset/large-dancers-learn.md new file mode 100644 index 0000000000..9f25990924 --- /dev/null +++ b/.changeset/large-dancers-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +- **BREAKING**: Removed the `FavouriteTemplate` export in favor of the `FavoriteEntity` from `@backstage/plugin-catalog-react`. Please migrate any usages to that component instead if you are creating your own `TemplateCard` page. diff --git a/.changeset/nasty-pets-join.md b/.changeset/nasty-pets-join.md new file mode 100644 index 0000000000..1ff07274cf --- /dev/null +++ b/.changeset/nasty-pets-join.md @@ -0,0 +1,9 @@ +--- +'@backstage/catalog-model': patch +--- + +**DEPRECATION**: Deprecated the `EntityName` type, and added the better-named `CompoundEntityRef` to replace it. + +**DEPRECATION**: Deprecated the `getEntityName` function, and added the better-named `getCompoundEntityRef` to replace it. + +Please switch over to using the new symbols, as the old ones may be removed in a future release. diff --git a/.changeset/olive-glasses-approve.md b/.changeset/olive-glasses-approve.md new file mode 100644 index 0000000000..e74b7ef485 --- /dev/null +++ b/.changeset/olive-glasses-approve.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Export FetchUrlReader to facilitate more flexible configuration of the backend. diff --git a/.changeset/orange-crews-explain.md b/.changeset/orange-crews-explain.md new file mode 100644 index 0000000000..4b831392eb --- /dev/null +++ b/.changeset/orange-crews-explain.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': minor +--- + +**BREAKING**: Removed the deprecated `app.` template variables from the `index.html` templating. These should be replaced by using `config.getString("app.")` instead. diff --git a/.changeset/polite-houses-wink.md b/.changeset/polite-houses-wink.md new file mode 100644 index 0000000000..3e177f5383 --- /dev/null +++ b/.changeset/polite-houses-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +- **BREAKING**: The `isOwnerOf` function has been marked as `@alpha` and is now only available via the `@backstage/plugin-catalog-react/alpha` import. The limitations of this function with regards to only supporting direct relations have also been documented. diff --git a/.changeset/popular-items-tan.md b/.changeset/popular-items-tan.md new file mode 100644 index 0000000000..b91773a5d8 --- /dev/null +++ b/.changeset/popular-items-tan.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-permission-node': patch +--- + +Export some utility functions for parsing PermissionCriteria + +`isAndCriteria`, `isOrCriteria`, `isNotCriteria` are now exported. diff --git a/.changeset/quick-mugs-pay.md b/.changeset/quick-mugs-pay.md new file mode 100644 index 0000000000..f2d3a00eb8 --- /dev/null +++ b/.changeset/quick-mugs-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add documentation for config options `userGroupMemberSearch` and `groupSearch` diff --git a/.changeset/red-chefs-beam.md b/.changeset/red-chefs-beam.md new file mode 100644 index 0000000000..5cadccb0aa --- /dev/null +++ b/.changeset/red-chefs-beam.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-airbrake': patch +'@backstage/plugin-airbrake-backend': patch +--- + +The Airbrake plugin installation instructions have been updated to work better and conform to how the frontend and backend plugins are supposed to be integrated into a Backstage instance. diff --git a/.changeset/rotten-windows-worry.md b/.changeset/rotten-windows-worry.md new file mode 100644 index 0000000000..a1ab353ce3 --- /dev/null +++ b/.changeset/rotten-windows-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +**DEPRECATION**: Deprecated `getEntityByName` from `CatalogApi` and added `getEntityByRef` instead, which accepts both string and compound ref forms. diff --git a/.changeset/search-blankly-have-a-nice-life.md b/.changeset/search-blankly-have-a-nice-life.md new file mode 100644 index 0000000000..63dc6051bd --- /dev/null +++ b/.changeset/search-blankly-have-a-nice-life.md @@ -0,0 +1,18 @@ +--- +'@backstage/plugin-search-backend-node': minor +'@backstage/search-common': minor +--- + +**BREAKING** + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +The concepts of `Collator` and `Decorator` have been replaced with readable and +transform object streams (respectively), as well as factory classes to +instantiate them. Accordingly, the `SearchEngine.index()` method has also been +replaced with a `getIndexer()` factory method that resolves to a writable +object stream. + +Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-done-me-no-favor.md b/.changeset/search-done-me-no-favor.md new file mode 100644 index 0000000000..0b4e24c4a8 --- /dev/null +++ b/.changeset/search-done-me-no-favor.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-pg': minor +--- + +**BREAKING** + +The `PgSearchEngine` implements the new stream-based indexing process expected +by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-just-smile-politely.md b/.changeset/search-just-smile-politely.md new file mode 100644 index 0000000000..a7831155da --- /dev/null +++ b/.changeset/search-just-smile-politely.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +A `DefaultTechDocsCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultTechDocsCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for necessary changes to your search backend plugin configuration. diff --git a/.changeset/search-like-a-bank-teller.md b/.changeset/search-like-a-bank-teller.md new file mode 100644 index 0000000000..a9b0330eb2 --- /dev/null +++ b/.changeset/search-like-a-bank-teller.md @@ -0,0 +1,43 @@ +--- +'@backstage/create-app': patch +--- + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +To take advantage of this, upgrade to the latest version of +`@backstage/plugin-search-backend-node`, as well as any backend plugins whose +collators you are using. Then, make the following changes to your +`/packages/backend/src/plugins/search.ts` file: + +```diff +-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; ++import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; ++import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + +// ... + + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultCatalogCollator.fromConfig(config, { discovery }), ++ factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultTechDocsCollator.fromConfig(config, { ++ factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); +``` + +If you've written custom collators, decorators, or search engines in your +Backstage backend instance, you will need to re-implement them as readable, +transform, and writable streams respectively (including factory classes for +instantiating them). [A how-to guide for refactoring](https://backstage.io/docs/features/search/how-to-guides#rewriting-alpha-style-collators-for-beta) +existing implementations is available. diff --git a/.changeset/search-selfless-cold-composed.md b/.changeset/search-selfless-cold-composed.md new file mode 100644 index 0000000000..7e7dafd421 --- /dev/null +++ b/.changeset/search-selfless-cold-composed.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +**BREAKING** + +The `ElasticSearchSearchEngine` implements the new stream-based indexing +process expected by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-throw-me-a-right.md b/.changeset/search-throw-me-a-right.md new file mode 100644 index 0000000000..32f4e82fe4 --- /dev/null +++ b/.changeset/search-throw-me-a-right.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +A `DefaultCatalogCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultCatalogCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for necessary changes to your search backend plugin configuration. diff --git a/.changeset/tame-lions-know.md b/.changeset/tame-lions-know.md new file mode 100644 index 0000000000..7f304b22ef --- /dev/null +++ b/.changeset/tame-lions-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +**BREAKING**: Removed the deprecated `GithubAuth.normalizeScopes` method. diff --git a/.changeset/tasty-poems-raise.md b/.changeset/tasty-poems-raise.md new file mode 100644 index 0000000000..fbd81423b6 --- /dev/null +++ b/.changeset/tasty-poems-raise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': minor +--- + +**BREAKING**: The `TokenFactory.issueToken` used by custom sign-in resolvers now ensures that the sub claim given is a full entity reference of the format `:/`. Any existing custom sign-in resolver functions that do not supply a full entity reference must be updated. diff --git a/.changeset/tidy-jokes-dream.md b/.changeset/tidy-jokes-dream.md new file mode 100644 index 0000000000..aa9c84fc4b --- /dev/null +++ b/.changeset/tidy-jokes-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add `userExpand` config option to allow expanding a single relationship diff --git a/.changeset/tiny-jobs-hunt.md b/.changeset/tiny-jobs-hunt.md new file mode 100644 index 0000000000..8cf9db20b8 --- /dev/null +++ b/.changeset/tiny-jobs-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Modify techdocs builder to automatically append techdocs-core plugin to mkdocs.yaml file if it is missing. Adds an optional configuration item if this plugin needs to be omitted. diff --git a/.changeset/tricky-students-promise.md b/.changeset/tricky-students-promise.md new file mode 100644 index 0000000000..b9c6150a4a --- /dev/null +++ b/.changeset/tricky-students-promise.md @@ -0,0 +1,22 @@ +--- +'@backstage/catalog-client': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-tech-insights': patch +'@backstage/plugin-tech-insights-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `CompoundEntityRef` instead of `EntityName`, and `getCompoundEntityRef` instead of `getEntityName`, from `@backstage/catalog-model`. diff --git a/.changeset/wild-dolphins-lick.md b/.changeset/wild-dolphins-lick.md new file mode 100644 index 0000000000..cdd5827b00 --- /dev/null +++ b/.changeset/wild-dolphins-lick.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATION**: Deprecated the `RefreshIntervalFunction` and `createRandomRefreshInterval` in favour of the `ProcessingIntervalFunction` and `createRandomProcessingInterval` type and method respectively. Please migrate to use the new names. + +**DEPRECATION**: Deprecated the `setRefreshInterval` and `setRefreshIntervalSeconds` methods on the `CatalogBuilder` for the new `setProcessingInterval` and `setProcessingIntervalSeconds` methods. Please migrate to use the new names. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 6359493d4b..c7971f6c52 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -20,7 +20,6 @@ Autoscaling autoselect Avro aws -backrub backported backporting Bigtable @@ -39,8 +38,6 @@ Changesets chanwit Chanwit ci -cisphobia -cissexist classname cli cloudbuild @@ -68,7 +65,6 @@ css Datadog dataflow dayjs -deadnaming debounce Debounce declaratively @@ -171,7 +167,6 @@ Minikube Minio misconfiguration misconfigured -misgendering mkdocs Mkdocs monorepo @@ -213,6 +208,7 @@ parallelization Patrik Peloton performant +Performant plantuml Platformize Podman diff --git a/README.md b/README.md index 1078631926..e78800ef84 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # [Backstage](https://backstage.io) +_During March 7 to March 11 the maintainers will be taking part in Spotify's annual hack week. Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_ + [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) diff --git a/contrib/search/README.md b/contrib/search/README.md new file mode 100644 index 0000000000..9a43a05c4c --- /dev/null +++ b/contrib/search/README.md @@ -0,0 +1,3 @@ +# Search + +Contributions/extensions to the Search plugin diff --git a/contrib/search/confluence/ConfluenceCollator.md b/contrib/search/confluence/ConfluenceCollator.md new file mode 100644 index 0000000000..f636ae88ea --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator.md @@ -0,0 +1,95 @@ +ConfluenceCollator.ts reference + +```ts +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch'; + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + const ConfluenceUrlBase = + 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api'; + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {}; + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + Authorization: `Basic ${process.env.CONFLUENCE_TOKEN}`, + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson; + } + + async function getSpaces(): Promise { + const data = await getConfluenceData( + `${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`, + ); + let spacesList = []; + if (data['results']) { + const results = data['results']; + for (const result of results) { + spacesList.push(result['key']); + } + } + return spacesList; + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = []; + for (var space of spaces) { + let next = true; + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}`; + while (next) { + const data = await getConfluenceData(requestUrl); + if (data['results']) { + const results = data['results']; + for (const result of results) { + documentsList.push(result['_links']['self']); + } + if (data['_links']['next']) { + requestUrl = data['_links']['base'] + data['_links']['next']; + } else { + next = false; + } + } else { + break; + } + } + } + return documentsList; + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = []; + for (var documentUrl of documents) { + const data = await getConfluenceData( + documentUrl + '?expand=body.storage', + ); + if (data['status'] && data['status'] == 'current') { + const documentMetaData = { + title: data['title'], + text: data['body']['storage']['value'], + location: data['_links']['base'] + data['_links']['webui'], + }; + documentInfo.push(documentMetaData); + } + } + return documentInfo; + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList; + } +} +``` diff --git a/contrib/search/confluence/ConfluenceResultListItem.md b/contrib/search/confluence/ConfluenceResultListItem.md new file mode 100644 index 0000000000..ba7c4f91a9 --- /dev/null +++ b/contrib/search/confluence/ConfluenceResultListItem.md @@ -0,0 +1,59 @@ +ConfluenceResultListItem.tsx reference + +```tsx +import React from 'react'; +import { Link } from '@backstage/core-components'; +import { IndexableDocument } from '@backstage/search-common'; +import { + Divider, + ListItem, + ListItemIcon, + ListItemText, +} from '@material-ui/core'; + +type Props = { + result: IndexableDocument; +}; + +export const ConfluenceResultListItem = ({ result }: Props) => { + // Remove html tags from document text before displaying + const chars = []; + let isTag = false; + for (const c of result.text.substring(0, 500)) { + if (c === '<') { + isTag = true; + continue; + } + if (c === '>') { + isTag = false; + chars.push(' '); + continue; + } + if (!isTag) { + chars.push(c); + } + } + const excerpt = + chars.join('').substring(0, 80) + (result.text.length > 80 ? '...' : ''); + + return ( + + + + + + + + + + ); +}; +``` diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md new file mode 100644 index 0000000000..80beb9c169 --- /dev/null +++ b/contrib/search/confluence/README.md @@ -0,0 +1,32 @@ +# Confluence + +These files help you add Confluence as a source to the Backstage Search plugin. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app. +Then, add the following code to your packages/app/src/components/search/SearchPage.tsx: + +```tsx +import { ConfluenceResultListItem } from './ConfluenceResultListItem'; +``` + +```tsx +case 'confluence': + return ( + + ); +``` + +and the following to packages/backend/src/plugins/search.ts: + +```ts +import { ConfluenceCollator } from './search/ConfluenceCollator'; +``` + +```ts +indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + collator: new ConfluenceCollator(), +}); +``` diff --git a/docs/features/search/concepts.md b/docs/features/search/concepts.md index 8cb17b12e6..052f71376f 100644 --- a/docs/features/search/concepts.md +++ b/docs/features/search/concepts.md @@ -54,13 +54,14 @@ An index is a collection of such documents of a given type. ### Collators You need to be able to search something! Collators are the way to define what -can be searched. Specifically, they're classes which return documents conforming -to a minimum set of fields (including a document title, location, and text), but -which can contain any other fields as defined by the collator itself. One -collator is responsible for defining and collecting documents of a type. +can be searched. Specifically, they're readable object streams of documents that +conform to a minimum set of fields (including a document title, location, and +text), but which can contain any other fields as defined by the collator itself. +One collator is responsible for defining and collecting documents of a type. -Some plugins, like the Catalog Backend, provide so-called "default" collators -which you can use out-of-the-box to start searching across Backstage quickly. +Some plugins, like the Catalog Backend, provide so-called "default" collator +factories which you can use out-of-the-box to start searching across Backstage +quickly. ### Decorators @@ -68,9 +69,15 @@ Sometimes you want to add extra information to a set of documents in your search index that the collator may not be aware of. For example, the Software Catalog knows about software entities, but it may not know about their usage or quality. -Decorators are classes which can add extra fields to pre-collated documents. -This extra metadata could then be used to bias search results or otherwise -improve the search experience in your Backstage instance. +Decorators are transform streams which sit between a collator (read stream) and +an indexer (write stream) during the indexing process. It can be used to add +extra fields to documents as they are being collated and indexed. This extra +metadata could then be used to bias search results or otherwise improve the +search experience in your Backstage instance. + +In addition to adding extra metadata, decorators (like any transform stream) can +also be used to remove metadata, filter out, or even add extra documents at +index-time. ### The Scheduler diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index a3c42c682c..ac36d64673 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -48,10 +48,10 @@ const app = createApp({ ## How to index TechDocs documents The TechDocs plugin has supported integrations to Search, meaning that it -provides a default collator ready to be used. +provides a default collator factory ready to be used. The purpose of this guide is to walk you through how to register the -[DefaultTechDocsCollator](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts) +[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts) in your App, so that you can get TechDocs documents indexed. If you have been through the @@ -60,18 +60,19 @@ you should have the `packages/backend/src/plugins/search.ts` file available. If so, you can go ahead and follow this guide - if not, start by going through the getting started guide. -1. Import the DefaultTechDocsCollator from `@backstage/plugin-techdocs-backend`. +1. Import the `DefaultTechDocsCollatorFactory` from + `@backstage/plugin-techdocs-backend`. ```typescript -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; ``` -2. Register the DefaultTechDocsCollator with the IndexBuilder. +2. Register the `DefaultTechDocsCollatorFactory` with the IndexBuilder. ```typescript indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, @@ -131,3 +132,264 @@ indexBuilder.addCollator({ As shown above, you can add a catalog entity filter to narrow down what catalog entities are indexed by the search engine. + +## How to migrate from Search Alpha to Beta + +For the purposes of this guide, Search Beta version is defined as: + +- **Search Plugin**: At least `v0.x.y` +- **Search Backend Plugin**: At least `v0.x.y` +- **Search Backend Node**: At least `v0.x.y` + +In the Beta version, the Search Platform's indexing process has been rewritten +as a stream pipeline in order to improve efficiency and performance on large +sets of documents. + +If you've not yet extended the Search Platform with custom code, and have +instead taken advantage of default collators, decorators, and search engines +provided by existing plugins, the migration process is fairly straightforward: + +1. Upgrade to at least version `0.x.y` of + `@backstage/plugin-search-backend-node`, as well as any backend plugins whose + collators you are using (e.g. at least version `0.x.y` of + `@backstage/plugin-catalog-backend` and/or version `0.x.y` of + `@backstage/plugin-techdocs-backend`). +2. Then, make the following changes to your + `/packages/backend/src/plugins/search.ts` file: + + ```diff + -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; + -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; + +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; + +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + // ... + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultCatalogCollator.fromConfig(config, { discovery }), + + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultTechDocsCollator.fromConfig(config, { + + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); + ``` + +Any custom collators, decorators, or search engine implementations will require +minor refactoring. Continue on for details. + +### Rewriting alpha-style collators for beta + +In alpha versions of the Backstage Search Platform, collators were classes that +implemented an `execute` method which resolved an `IndexableDocument` array. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an [object-mode][obj-mode] `Readable` stream where each +object pushed onto the stream is of type `IndexableDocument`. Instances of this +stream are instantiated by a factory class conforming to the +`DocumentCollatorFactory` interface. + +The optimal conversion strategy will vary depending on the collator's logic, but +the simplest conversion can follow a process like this: + +1. Rename your collator class to something like `YourCollatorFactory` and update + it to implement `DocumentCollatorFactory` instead of `DocumentCollator`. +2. Update its `execute` method so that it resolves + `AsyncGenerator` instead of `YourIndexableDocument[]`. +3. Implement `DocumentCollatorFactory`'s `getCollator` method which resolves to + `Readable.from(this.execute())` (which is a utility for creating [readable + streams][read-stream] from [async generators][async-gen]). + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { Readable } from 'stream'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async *execute(): AsyncGenerator { + const widgets = await this.client.getWidgets(); + for (const widget of widgets) { + yield { + title: widget.name, + location: widget.url, + text: widget.description, + }; + } + } + getCollator() { + return Readable.from(this.execute()); + } +} +``` + +Note: it may be possible to simplify your collator dramatically! If your custom +collator was previously using streams under the hood (for example, by reading +newline delimited JSON from a local or remote file), you could just expose the +stream directly via a simple factory class: + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { createReadStream } from 'fs'; +import { parse } from '@jsonlines/core'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async getCollator() { + const parseStream = parse(); + return createReadStream('./documents.ndjson').pipe(parseStream); + } +} +``` + +### Rewriting alpha-style decorators for beta + +In alpha versions of the Backstage Search Platform, decorators were classes that +implemented an `execute` method which took an `IndexableDocument` array as an +argument, and resolved a modified array of the same type. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an object-mode `Transform` stream which reads objects of +type `IndexableDocument`, and writes objects of a conforming type. Similar to +collators, instances of this stream are instantiated by a factory class +conforming to the `DocumentDecoratorFactory` interface. + +Although you can choose to implement a `Transform` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a `DecoratorBase` class +in order to simplify the developer experience. With this base class, all that's +needed is to transfer your old decorator class logic into the base class' three +methods (`initialize`, `decorate`, and `finalize`), and implement the factory +class that instantiates the stream: + +```ts +import { DecoratorBase } from '@backstage/plugin-search-backend-node'; +export class YourDecorator extends DecoratorBase { + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async decorate( + document: YourIndexableDocument, + ): Promise { + // Perform transformation logic here. + return document; + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator() { + return new YourDecorator(); + } +} +``` + +Note the return type of the `decorate` method and how each can be used to +different effect. + +- By resolving a single `YourIndexableDocument` object, your decorator can be + used to make simple transformations: + + ```ts + class BooleanWidgetCoolnessDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple, 1:1 transformation. + widget.isCool = widget.isCool === 'true' ? true : false; + return widget; + } + } + ``` + +- By resolving `undefined`, your decorator can filter out documents which + shouldn't be in the index: + + ```ts + class OnlyCoolWidgetsDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple filter operation. + return widget.isCool ? widget : undefined; + } + } + ``` + +- By resolving an array of `YourIndexableDocument` objects, you can generate + multiple documents based on the content of one: + + ```ts + class WidgetByVariantDecorator extends DecoratorBase { + async decorator(widget) { + // Generate one widget doc per widget variant. + return widget.variants.map(variant => { + // Each widget doc is the given widget plus a "variant" property + // pulled from a widget.variants string array. + return { + ...widget, + variant, + }; + }); + } + } + ``` + +In alpha versions, a decorator had access to every `IndexableDocument` +simultaneously. This is no longer possible in beta versions (precisely to make +the indexing process more efficient and performant). You will need to modify +your decorator's logic so that it does not need access to every document at +once. + +### Rewriting alpha-style search engines for beta + +Search Engines are responsible for both querying and indexing documents to an +underlying search engine technology. While the search engine query interface +didn't change between alpha and beta versions, the indexing half of the +interface _did_ change. + +In alpha versions of the Backstage Search Platform, a search engine implemented +an `index` method which took a `type` and an `IndexableDocument` array and was +responsible for writing these documents to the underlying search engine. + +In beta versions, the logic encapsulated by the aforementioned `index` method is +contained within an object-mode `Writable` stream which expects objects of type +`IndexableDocument`. On the search engine class itself, the `index` method is +replaced with a `getIndexer` factory method which still takes the `type`, but +resolves an instance of the aforementioned `Writable` stream. + +Although you can choose to implement a `Writable` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a +`BatchSearchEngineIndexer` class in order to simplify the developer experience. +With this base class, which collects documents in batches of a configurable size +on your behalf, all that's needed is to transfer your old `index` method logic +into the base class' three methods (`initialize`, `index`, and `finalize`), and +implement the factory method that instantiates the stream: + +```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { SearchEngine } from '@backstage/search-common'; +export class YourSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor({ type }: { type: string }) { + // Customize the number of documents passed to the index method per batch. + super({ batchSize: 500 }); + // An imaginary search engine indexing client. + this.index = new SomeSearchEngineIndex({ indexName: type }); + } + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async index(documents: IndexableDocument[]) { + await this.index.batchOf(documents); + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourSearchEngine implements SearchEngine { + async getIndexer(type: string) { + return new YourSearchEngineIndexer({ type }); + } +} +``` + +[obj-mode]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_object_mode +[read-stream]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_readable_streams +[async-gen]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators diff --git a/docs/features/techdocs/cli.md b/docs/features/techdocs/cli.md index a5dc8c3c54..2a11e1181a 100644 --- a/docs/features/techdocs/cli.md +++ b/docs/features/techdocs/cli.md @@ -130,6 +130,8 @@ Options: if not found. --etag A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json. + --omitTechdocsCoreMkdocsPlugin An option to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. -v --verbose Enable verbose output. (default: false) -h, --help display help for command ``` diff --git a/docs/features/techdocs/concepts.md b/docs/features/techdocs/concepts.md index 408f092ebd..e34f58bc70 100644 --- a/docs/features/techdocs/concepts.md +++ b/docs/features/techdocs/concepts.md @@ -46,6 +46,24 @@ between `techdocs-backend` and the storage) [TechDocs Backend](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend) +## TechDocs Build Strategy + +To accommodate more complex logic surrounding whether or not to build TechDocs, the TechDocs backend +supports selecting a Build Strategy. +The Build Strategy is responsible for deciding whether the documentation requested should be built locally +by the TechDocs backend or not. +Customization of the Build Strategy allows for more complex behaviour regarding whether the TechDocs backend +is responsible for building TechDocs, whether an external process is responsible, or whether a combination +of local builds and an external process is responsible, on an entity-by-entity basis. + +The default Build Strategy results in the TechDocs backend building documentation locally if the +`techdocs.builder` configuration option is set to `'local'`, and skipping any building otherwise. +However any logic that satisfies the Build Strategy interface can be implemented, using the Backstage +config as well as the entity being processed to make a decision. + +For an example of how the Build Strategy can be used to implement a 'hybrid' build model, refer to +the [How to implement a hybrid build strategy](./how-to-guides.md#how-to-implement-a-hybrid-build-strategy) guide. + ## TechDocs Container The TechDocs container is a Docker container available at diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 6317ae365b..3edc55574f 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -37,12 +37,22 @@ techdocs: pullImage: true + mkdocs: + # (Optional) techdocs.generator.omitTechdocsCoreMkdocsPlugin can be used to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + # Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. + omitTechdocsCorePlugin: false + # techdocs.builder can be either 'local' or 'external. - # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage - # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. - # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to generate and publish. In this case of 'external', - # we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). This is the "Recommended" setup of - # the architecture. Read more here https://backstage.io/docs/features/techdocs/architecture + # Using the default build strategy, if builder is set to 'local' and you open a TechDocs page, + # techdocs-backend will try to generate the docs, publish to storage and show the generated docs afterwords. + # This is the "Basic" setup of the TechDocs Architecture. + # Using the default build strategy, if builder is set to 'external' (or anything other than 'local'), techdocs-backend + # will only fetch the docs and will NOT try to generate and publish. + # In this case, we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). + # This is the "Recommended" setup of the architecture. + # Note that custom build strategies may alter this behaviour. + # Read more about the "Basic" and "Recommended" setups here https://backstage.io/docs/features/techdocs/architecture + # Read more about build strategies here: https://backstage.io/docs/features/techdocs/concepts#techdocs-build-strategy builder: 'local' diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 383c3ae6d2..f63ff43b7f 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -79,6 +79,9 @@ plugins: - techdocs-core ``` +> Note - The plugins section above is optional. Backstage automatically adds the `techdocs-core` plugin to the +> mkdocs file if it is missing. This functionality can be turned off with a [configuration option](./configuration.md) in Backstage. + Update your component's entity description by adding the following lines to its `catalog-info.yaml` in the root of its repository: diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 863b4b3ecb..56ca2252d0 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -538,3 +538,58 @@ Done! Now you have a support of the following diagrams along with mermaid: - `Vega` - `Vega-Lite` - `WaveDrom` + +## How to implement a hybrid build strategy + +One limitation of the [Recommended deployment](./architecture.md#recommended-deployment) is that +the experience for users requires modifying their CI/CD process to publish +their TechDocs. For some users, this may be unnecessary, and provides a barrier +to entry for onboarding users to Backstage. However, a purely local TechDocs +build restricts TechDocs creators to using the tooling provided in Backstage, +as well as the plugins and features provided in the Backstage-included `mkdocs` +installation. + +To accommodate both of these use-cases, users can implement a custom [Build Strategy](./concepts.md#techdocs-build-strategy) +with logic to encode which TechDocs should be built locally, and which will be +built externally. + +To achieve this hybrid build model: + +1. In your Backstage instance's `app-config.yaml`, set `techdocs.builder` to + `'local'`. This ensures that Backstage will build docs for users who want the + 'out-of-the-box' experience. +2. Configure external storage of TechDocs as normal for a production deployment. + This allows Backstage to publish documentation to your storage, as well as + allowing other users to publish documentation from their CI/CD pipelines. +3. Create a custom build strategy, that implements the `DocsBuildStrategy` interface, + and which implements your custom logic for determining whether to build docs for + a given entity. + For example, to only build docs when an entity has the `company.com/techdocs-builder` + annotation set to `'local'`: + + ```typescript + export class AnnotationBasedBuildStrategy { + private readonly config: Config; + + constructor(config: Config) { + this.config = config; + } + + async shouldBuild(_: Entity): Promise { + return ( + this.entity.metadata?.annotations?.['company.com/techdocs-builder'] === + 'local' + ); + } + } + ``` + +4. Pass an instance of this Build Strategy as the `docsBuildStrategy` parameter of the + TechDocs backend `createRouter` method. + +Users should now be able to choose to have their documentation built and published by +the TechDocs backend by adding the `company.com/techdocs-builder` annotation to their +entity. If the value of this annotation is `'local'`, the TechDocs backend will build +and publish the documentation for them. If the value of the `company.com/techdocs-builder` +annotation is anything other than `'local'`, the user is responsible for publishing +documentation to the appropriate location in the TechDocs external storage. diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index 75f0606752..f57efc1255 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -552,3 +552,36 @@ The following is an excerpt of a typical setup of an isomorphic library package: }, "files": ["dist"], ``` + +## Experimental Type Build + +The Backstage CLI has an experimental feature where multiple different type definition files can be generated for different release stages. The release stages are marked in the [TSDoc](https://tsdoc.org/) for each individual export, using either `@public`, `@alpha`, or `@beta`. Rather than just building a single `index.d.ts` file, the build process will instead output `index.d.ts`, `index.beta.d.ts`, and `index.alpha.d.ts`. Each of these files will have exports from more unstable release stages stripped, meaning that `index.d.ts` will omit all exports marked with `@alpha` or `@beta`, while `index.beta.d.ts` will omit all exports marked with `@alpha`. + +This feature is aimed at projects that publish to package registries and wish to maintain different levels of API stability within each package. There is no need to use this within a single monorepo, as it has no effect due to only applying to built and published packages. + +In order for the experimental type build to work, `@microsoft/api-extractor` must be installed in your project, as it is an optional peer dependency of the Backstage CLI. There are then three steps that need to be taken for each package where you want to enable this feature: + +- Add the `--experimental-type-build` flag to the `"build"` script of the package. +- Add either one or both of `"alphaTypes"` and `"betaTypes"` to the `"publishConfig"` of the package: + ```json + "publishConfig": { + ... + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts", + "betaTypes": "dist/index.beta.d.ts" + }, + ``` +- Add either one or both of `"alpha"` and `"beta"` to the `"files"` of the package: + ```json + "files": [ + "dist", + "alpha", + "beta" + ] + ``` + +Once this setup is complete, users of the published packages will only be able to access the stable API via the main package entry point, for example `@acme/my-plugin`. Exports marked with `@alpha` or `@beta` will only be available via the `/alpha` entry point, for example `@acme/my-plugin/alpha`, and exports marked with `@beta` will only be available via `/beta`. This does not apply within the monorepo that contains the package. There all exports still have to be imported via the main entry point. + +Note that these different entry points are only separated during type checking. At runtime they all share the same code which contains the exports from all releases stages. + +An example of this setup can be seen in the [`@backstage/catalog-model`](https://github.com/backstage/backstage/blob/da0675bf9f28ed1460f03635a22d3c26abd14707/packages/catalog-model/package.json#L14) package, which has enabled `alpha` type exports. With this setup, exports marked as `@alpha` are only available for import via `@backstage/catalog-model/alpha`. The `@backstage/catalog-model` package currently does not have any exports marked as `@beta`, or a `/beta` entry point. diff --git a/docs/publishing.md b/docs/publishing.md index a0d67b7d24..fbbef93493 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -67,3 +67,12 @@ process is used to release an emergency fix as version `6.5.1` in the patch rele - [ ] The fix, which you can likely cherry-pick from your patch branch: `git cherry-pick origin/patch/v1.18.0^` - [ ] An updated `CHANGELOG.md` of all patched packages from the tip of the patch branch, `git checkout origin/patch/v1.18.0 -- {packages,plugins}/*/CHANGELOG.md`. - [ ] A changeset with the message "Applied the fix from version `6.5.1` of this package, which is part of the `v1.18.1` release of Backstage." + - [ ] An entry in `.changeset/patched.json` that sets the current release version to `6.5.1`: + + ```json + { + "currentReleaseVersion": { + "@backstage/plugin-foo": "6.5.1" + } + } + ``` diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 9b963f3fc9..4ed9bb1514 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -234,6 +234,21 @@ export type ErrorHandlerOptions = { logClientErrors?: boolean; }; +// @public +export class FetchUrlReader implements UrlReader { + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export type FromReadableArrayOptions = Array<{ data: Readable; diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index ea8e4626be..371e11b364 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -270,22 +270,6 @@ describe('BitbucketUrlReader', () => { expect(response.etag).toBe('12ab34cd56ef'); }); - - it('should throw error when apiBaseUrl is missing', () => { - expect(() => { - /* eslint-disable no-new */ - new BitbucketUrlReader( - new BitbucketIntegration( - readBitbucketIntegrationConfig( - new ConfigReader({ - host: 'bitbucket.mycompany.net', - }), - ), - ), - { treeResponseFactory }, - ); - }).toThrowError('must configure an explicit apiBaseUrl'); - }); }); describe('search hosted', () => { diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 2006637545..e22137ea69 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -62,14 +62,9 @@ export class BitbucketUrlReader implements UrlReader { private readonly integration: BitbucketIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { - const { host, apiBaseUrl, token, username, appPassword } = - integration.config; + const { host, token, username, appPassword } = integration.config; - if (!apiBaseUrl) { - throw new Error( - `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, - ); - } else if (!token && username && !appPassword) { + if (!token && username && !appPassword) { throw new Error( `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, ); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 2c3394e4fc..9c29bc93fe 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,6 +19,7 @@ export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { FetchUrlReader } from './FetchUrlReader'; export type { FromReadableArrayOptions, ReaderFactory, diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 4b5363e73d..a90b47cd81 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -18,7 +18,7 @@ import { useHotCleanup, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; import { createRouter } from '@backstage/plugin-search-backend'; import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; @@ -27,7 +27,7 @@ import { LunrSearchEngine, SearchEngine, } from '@backstage/plugin-search-backend-node'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; import { Logger } from 'winston'; import { PluginEnvironment } from '../types'; @@ -70,7 +70,7 @@ export default async function createPlugin({ // particular collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -78,7 +78,7 @@ export default async function createPlugin({ indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index c957859fec..9bbb24463b 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -3,8 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; // @public export type AddLocationRequest = { @@ -38,8 +38,13 @@ export interface CatalogApi { request: GetEntityAncestorsRequest, options?: CatalogRequestOptions, ): Promise; + // @deprecated getEntityByName( - name: EntityName, + name: CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( @@ -91,7 +96,11 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise; getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( @@ -130,18 +139,6 @@ export class CatalogClient implements CatalogApi { ): Promise; } -// @public @deprecated (undocumented) -export type CatalogEntitiesRequest = GetEntitiesRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; - -// @public @deprecated (undocumented) -export type CatalogListResponse<_Entity> = GetEntitiesResponse; - // @public export interface CatalogRequestOptions { // (undocumented) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 0035691f2c..0b838eb4c2 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -195,6 +195,60 @@ describe('CatalogClient', () => { }); }); + describe('getEntityByRef', () => { + const existingEntity: Entity = { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'exists', + }, + }; + + beforeEach(() => { + server.use( + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/exists`, + (_, res, ctx) => { + return res(ctx.json(existingEntity)); + }, + ), + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/missing`, + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), + ); + }); + + it('finds by string and compound', async () => { + await expect( + client.getEntityByRef('customkind:default/exists'), + ).resolves.toEqual(existingEntity); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'exists', + }), + ).resolves.toEqual(existingEntity); + }); + + it('returns undefined for 404s', async () => { + await expect( + client.getEntityByRef('customkind:default/missing'), + ).resolves.toBeUndefined(); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'missing', + }), + ).resolves.toBeUndefined(); + }); + }); + describe('getLocationById', () => { const defaultResponse = { data: { diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index bb3c9604b3..f2bc3ed8f6 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -18,7 +18,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, stringifyLocationRef, @@ -170,11 +170,32 @@ export class CatalogClient implements CatalogApi { return { items: entities.sort(refCompare) }; } + /** + * {@inheritdoc CatalogApi.getEntityByRef} + */ + async getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise { + const { kind, namespace, name } = parseEntityRef(entityRef); + return this.requestOptional( + 'GET', + `/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent( + namespace, + )}/${encodeURIComponent(name)}`, + options, + ); + } + + // NOTE(freben): When we deprecate getEntityByName from the interface, we may + // still want to leave this implementation in place for quite some time + // longer, to minimize the risk for breakages. Suggested date for removal: + // August 2022 /** * {@inheritdoc CatalogApi.getEntityByName} */ async getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise { const { kind, namespace = 'default', name } = compoundName; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 4a86ca41f3..26af3889e2 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; /** * This symbol can be used in place of a value when passed to filters in e.g. @@ -306,11 +306,25 @@ export interface CatalogApi { * Gets a single entity from the catalog by its ref (kind, namespace, name) * triplet. * + * @param entityRef - A complete entity ref, either on string or compound form + * @param options - Additional options + * @returns The matching entity, or undefined if there was no entity with that ref + */ + getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + + /** + * Gets a single entity from the catalog by its ref (kind, namespace, name) + * triplet. + * + * @deprecated Use getEntityRef instead * @param name - A complete entity ref * @param options - Additional options */ getEntityByName( - name: EntityName, + name: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; diff --git a/packages/catalog-client/src/types/deprecated.ts b/packages/catalog-client/src/types/deprecated.ts deleted file mode 100644 index 0b145f3024..0000000000 --- a/packages/catalog-client/src/types/deprecated.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - GetEntitiesRequest, - GetEntitiesResponse, - GetEntityAncestorsRequest, - GetEntityAncestorsResponse, -} from './api'; - -/** - * @public - * @deprecated use GetEntitiesRequest instead - */ -export type CatalogEntitiesRequest = GetEntitiesRequest; -/** - * @public - * @deprecated use GetEntitiesResponse instead - */ -export type CatalogListResponse<_Entity> = GetEntitiesResponse; -/** - * @public - * @deprecated use GetEntityAncestorsRequest instead - */ -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; -/** - * @public - * @deprecated use GetEntityAncestorsResponse instead - */ -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 39c8962b68..8bf4e34da2 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -28,5 +28,4 @@ export type { GetEntityFacetsRequest, GetEntityFacetsResponse, } from './api'; -export * from './deprecated'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 4574129d6f..99b4b39038 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -88,6 +88,13 @@ export { ComponentEntityV1alpha1 }; // @public export const componentEntityV1alpha1Validator: KindValidator; +// @public +export type CompoundEntityRef = { + kind: string; + namespace: string; + name: string; +}; + // @public export const DEFAULT_NAMESPACE = 'default'; @@ -166,12 +173,8 @@ export type EntityMeta = JsonObject & { links?: EntityLink[]; }; -// @public -export type EntityName = { - kind: string; - namespace: string; - name: string; -}; +// @public @deprecated +export type EntityName = CompoundEntityRef; // @public export const EntityPolicies: { @@ -196,7 +199,7 @@ export type EntityRef = // @public export type EntityRelation = { type: string; - target: EntityName; + target: CompoundEntityRef; targetRef: string; }; @@ -229,7 +232,10 @@ export class FieldFormatEntityPolicy implements EntityPolicy { } // @public -export function getEntityName(entity: Entity): EntityName; +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef; + +// @public @deprecated +export const getEntityName: typeof getCompoundEntityRef; // @public export function getEntitySourceLocation(entity: Entity): { @@ -337,7 +343,7 @@ export function parseEntityName( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseEntityRef( @@ -352,7 +358,7 @@ export function parseEntityRef( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseLocationRef(ref: string): { diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 70ed0fbe4d..f9781b14a2 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -37,7 +37,6 @@ "@backstage/config": "^0.1.15", "@backstage/errors": "^0.2.2", "@backstage/types": "^0.1.3", - "@types/json-schema": "^7.0.5", "ajv": "^7.0.3", "json-schema": "^0.4.0", "lodash": "^4.17.21", @@ -46,6 +45,7 @@ "devDependencies": { "@backstage/cli": "^0.14.1", "@types/jest": "^26.0.7", + "@types/json-schema": "^7.0.5", "@types/lodash": "^4.14.151", "yaml": "^1.9.2" }, diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index ef9b4352da..029d9e095b 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -15,7 +15,7 @@ */ import { JsonObject } from '@backstage/types'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { EntityStatus } from './EntityStatus'; /** @@ -201,7 +201,7 @@ export type EntityRelation = { * * @deprecated use targetRef instead */ - target: EntityName; + target: CompoundEntityRef; /** * The entity ref of the target of this relation. diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index 3a346b217c..fb9eee0353 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -34,6 +34,7 @@ export type { } from './EntityStatus'; export * from './policies'; export { + getCompoundEntityRef, getEntityName, parseEntityName, parseEntityRef, diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index f9bfcf38cd..4529b1174c 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -15,7 +15,7 @@ */ import { DEFAULT_NAMESPACE } from './constants'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { Entity } from './Entity'; function parseRefString(ref: string): { @@ -38,14 +38,25 @@ function parseRefString(ref: string): { } /** - * Extracts the kind, namespace and name that form the name triplet of the - * given entity. + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. + * + * @public + * @deprecated Use getCompoundEntityRef instead + * @param entity - An entity + * @returns The compound entity ref + */ +export const getEntityName = getCompoundEntityRef; + +/** + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. * * @public * @param entity - An entity - * @returns The complete entity name + * @returns The compound entity ref */ -export function getEntityName(entity: Entity): EntityName { +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef { return { kind: entity.kind, namespace: entity.metadata.namespace || DEFAULT_NAMESPACE, @@ -77,7 +88,7 @@ export function parseEntityName( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; } = {}, -): EntityName { +): CompoundEntityRef { const { kind, namespace, name } = parseEntityRef(ref, { defaultNamespace: DEFAULT_NAMESPACE, ...context, @@ -114,7 +125,7 @@ export function parseEntityRef( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; }, -): EntityName { +): CompoundEntityRef { if (!ref) { throw new Error(`Entity reference must not be empty`); } diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts index 986341bd33..19eacd4925 100644 --- a/packages/catalog-model/src/index.ts +++ b/packages/catalog-model/src/index.ts @@ -24,5 +24,5 @@ export * from './entity'; export { EntityPolicies } from './EntityPolicies'; export * from './kinds'; export * from './location'; -export type { EntityName, EntityRef } from './types'; +export type { EntityName, EntityRef, CompoundEntityRef } from './types'; export * from './validation'; diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 5b5f063ca3..e7d819ace9 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -15,16 +15,25 @@ */ /** - * A complete entity name, with the full kind-namespace-name triplet. + * All parts of a complete entity ref, forming a full kind-namespace-name + * triplet. * * @public */ -export type EntityName = { +export type CompoundEntityRef = { kind: string; namespace: string; name: string; }; +/** + * A complete entity name, with the full kind-namespace-name triplet. + * + * @deprecated Use CompoundEntityRef instead + * @public + */ +export type EntityName = CompoundEntityRef; + /** * A reference by name to an entity, either as a compact string representation, * or as a compound reference structure. diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index fe64f23209..bfd05807ad 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; -import chalk from 'chalk'; import { resolve as resolvePath } from 'path'; import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'; import HtmlWebpackPlugin from 'html-webpack-plugin'; @@ -118,43 +117,11 @@ export async function createConfig( }), ); - const appParamDeprecationMsg = chalk.red( - 'DEPRECATION WARNING: using `app.` in the index.html template is deprecated, use `config.getString("app.")` instead.', - ); plugins.push( new HtmlWebpackPlugin({ template: paths.targetHtml, templateParameters: { publicPath: validBaseUrl.pathname.replace(/\/$/, ''), - app: { - get title() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getString('app.title'); - }, - get baseUrl() { - console.warn(appParamDeprecationMsg); - return validBaseUrl.href; - }, - get googleAnalyticsTrackingId() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getOptionalString( - 'app.googleAnalyticsTrackingId', - ); - }, - get datadogRum() { - console.warn(appParamDeprecationMsg); - return { - env: frontendConfig.getOptionalString('app.datadogRum.env'), - clientToken: frontendConfig.getOptionalString( - 'app.datadogRum.clientToken', - ), - applicationId: frontendConfig.getOptionalString( - 'app.datadogRum.applicationId', - ), - site: frontendConfig.getOptionalString('app.datadogRum.site'), - }; - }, - }, config: frontendConfig, }, }), diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index a05796139b..3ff161cec4 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -182,7 +182,7 @@ function compileTsSchemas(paths: string[]) { program, // All schemas should export a `Config` symbol 'Config', - // This enables usage of @visibility and @deprecated in doc comments + // This enables the use of these tags in TSDoc comments { required: true, validationKeywords: ['visibility', 'deprecated'], diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index c52b953465..116b33c598 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -376,8 +376,6 @@ export type FlatRoutesProps = { export class GithubAuth { // (undocumented) static create(options: OAuthApiCreateOptions): typeof githubAuthApiRef.T; - // @deprecated (undocumented) - static normalizeScope(scope?: string): Set; } // @public diff --git a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts index b0af4c7ade..7efe4e95c6 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -47,19 +47,4 @@ export default class GithubAuth { defaultScopes, }); } - - /** - * @deprecated This method is deprecated and will be removed in a future release. - */ - static normalizeScope(scope?: string): Set { - if (!scope) { - return new Set(); - } - - const scopeList = Array.isArray(scope) - ? scope - : scope.split(/[\s|,]/).filter(Boolean); - - return new Set(scopeList); - } } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts index a0a1cc3701..c359cb4986 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts @@ -5,8 +5,8 @@ import { LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; export default async function createPlugin({ logger, @@ -23,7 +23,7 @@ export default async function createPlugin({ // collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -32,7 +32,7 @@ export default async function createPlugin({ // collator gathers entities from techdocs. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 10da05784c..9a6ba03f8e 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -88,7 +88,7 @@ export class BitbucketIntegration implements ScmIntegration { // @public export type BitbucketIntegrationConfig = { host: string; - apiBaseUrl?: string; + apiBaseUrl: string; token?: string; username?: string; appPassword?: string; diff --git a/packages/integration/src/bitbucket/config.ts b/packages/integration/src/bitbucket/config.ts index 1cd911aad3..44a2f0cc1f 100644 --- a/packages/integration/src/bitbucket/config.ts +++ b/packages/integration/src/bitbucket/config.ts @@ -36,12 +36,10 @@ export type BitbucketIntegrationConfig = { * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", * with no trailing slash. * - * May be omitted specifically for Bitbucket Cloud; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. + * Values omitted at the optional property at the app-config will be deduced + * from the "host" value. */ - apiBaseUrl?: string; + apiBaseUrl: string; /** * The authorization token to use for requests to a Bitbucket Server provider. @@ -90,6 +88,8 @@ export function readBitbucketIntegrationConfig( apiBaseUrl = trimEnd(apiBaseUrl, '/'); } else if (host === BITBUCKET_HOST) { apiBaseUrl = BITBUCKET_API_BASE_URL; + } else { + apiBaseUrl = `https://${host}/rest/api/1.0`; } return { diff --git a/packages/integration/src/helpers.test.ts b/packages/integration/src/helpers.test.ts index a69c7faec9..60a2789f05 100644 --- a/packages/integration/src/helpers.test.ts +++ b/packages/integration/src/helpers.test.ts @@ -24,7 +24,10 @@ import { describe('basicIntegrations', () => { describe('byUrl', () => { it('handles hosts without a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com' }); + const integration = new BitbucketIntegration({ + host: 'host.com', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, @@ -33,7 +36,10 @@ describe('basicIntegrations', () => { expect(integrations.byUrl('https://host.com:8080/a')).toBeUndefined(); }); it('handles hosts with a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com:8080' }); + const integration = new BitbucketIntegration({ + host: 'host.com:8080', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, diff --git a/packages/search-common/api-report.md b/packages/search-common/api-report.md index 65f9f25180..475d85e11c 100644 --- a/packages/search-common/api-report.md +++ b/packages/search-common/api-report.md @@ -3,38 +3,33 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { JsonObject } from '@backstage/types'; import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export interface DocumentCollator { - // (undocumented) - execute(): Promise; +// @beta +export interface DocumentCollatorFactory { + getCollator(): Promise; readonly type: string; readonly visibilityPermission?: Permission; } -// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export interface DocumentDecorator { - // (undocumented) - execute(documents: IndexableDocument[]): Promise; +// @beta +export interface DocumentDecoratorFactory { + getDecorator(): Promise; readonly types?: string[]; } -// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type DocumentTypeInfo = { visibilityPermission?: Permission; }; -// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface IndexableDocument { authorization?: { resourceRef: string; @@ -44,23 +39,17 @@ export interface IndexableDocument { title: string; } -// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta export type QueryRequestOptions = { token?: string; }; -// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type QueryTranslator = (query: SearchQuery) => unknown; -// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface SearchEngine { - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; query( query: SearchQuery, options?: QueryRequestOptions, @@ -68,9 +57,7 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; } -// Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchQuery { // (undocumented) filters?: JsonObject; @@ -82,9 +69,7 @@ export interface SearchQuery { types?: string[]; } -// Warning: (ae-missing-release-tag) "SearchResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResult { // (undocumented) document: IndexableDocument; @@ -92,9 +77,7 @@ export interface SearchResult { type: string; } -// Warning: (ae-missing-release-tag) "SearchResultSet" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResultSet { // (undocumented) nextPageCursor?: string; diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 4e61767ea0..51ce45617b 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -16,7 +16,11 @@ import { Permission } from '@backstage/plugin-permission-common'; import { JsonObject } from '@backstage/types'; +import { Readable, Transform, Writable } from 'stream'; +/** + * @beta + */ export interface SearchQuery { term: string; filters?: JsonObject; @@ -24,11 +28,17 @@ export interface SearchQuery { pageCursor?: string; } +/** + * @beta + */ export interface SearchResult { type: string; document: IndexableDocument; } +/** + * @beta + */ export interface SearchResultSet { results: SearchResult[]; nextPageCursor?: string; @@ -38,6 +48,7 @@ export interface SearchResultSet { /** * Base properties that all indexed documents must include, as well as some * common properties that documents are encouraged to use where appropriate. + * @beta */ export interface IndexableDocument { /** @@ -72,6 +83,7 @@ export interface IndexableDocument { * Information about a specific document type. Intended to be used in the * {@link @backstage/search-backend-node#IndexBuilder} to collect information * about the types stored in the index. + * @beta */ export type DocumentTypeInfo = { /** @@ -82,10 +94,10 @@ export type DocumentTypeInfo = { }; /** - * Interface that must be implemented in order to expose new documents to - * search. + * Factory class for instantiating collators. + * @beta */ -export interface DocumentCollator { +export interface DocumentCollatorFactory { /** * The type or name of the document set returned by this collator. Used as an * index name by Search Engines. @@ -98,29 +110,41 @@ export interface DocumentCollator { */ readonly visibilityPermission?: Permission; - execute(): Promise; + /** + * Instantiates and resolves a document collator. + */ + getCollator(): Promise; } /** - * Interface that must be implemented in order to decorate existing documents with - * additional metadata. + * Factory class for instantiating decorators. + * @beta */ -export interface DocumentDecorator { +export interface DocumentDecoratorFactory { /** * An optional array of document/index types on which this decorator should * be applied. If no types are provided, this decorator will be applied to * all document/index types. */ readonly types?: string[]; - execute(documents: IndexableDocument[]): Promise; + + /** + * Instantiates and resolves a document decorator. + */ + getDecorator(): Promise; } /** * A type of function responsible for translating an abstract search query into * a concrete query relevant to a particular search engine. + * @beta */ export type QueryTranslator = (query: SearchQuery) => unknown; +/** + * Options when querying a search engine. + * @beta + */ export type QueryRequestOptions = { token?: string; }; @@ -129,6 +153,7 @@ export type QueryRequestOptions = { * Interface that must be implemented by specific search engines, responsible * for performing indexing and querying and translating abstract queries into * concrete, search engine-specific queries. + * @beta */ export interface SearchEngine { /** @@ -137,9 +162,15 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; /** - * Add the given documents to the SearchEngine index of the given type. + * Factory method for getting a search engine indexer for a given document + * type. + * + * @param type - The type or name of the document set for which an indexer + * should be retrieved. This corresponds to the `type` property on the + * document collator/decorator factories and will most often be used to + * identify an index or group to which documents should be written. */ - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; /** * Perform a search query against the SearchEngine. diff --git a/packages/techdocs-cli-embedded-app/src/apis.ts b/packages/techdocs-cli-embedded-app/src/apis.ts index 6234e3b719..1fe6ba8086 100644 --- a/packages/techdocs-cli-embedded-app/src/apis.ts +++ b/packages/techdocs-cli-embedded-app/src/apis.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { scmIntegrationsApiRef, @@ -81,7 +81,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return this.configApi.getString('techdocs.builder'); } - async getEntityDocs(_entityId: EntityName, path: string) { + async getEntityDocs(_entityId: CompoundEntityRef, path: string) { const apiOrigin = await this.getApiOrigin(); // Irrespective of the entity, use mkdocs server to find the file for the path. const url = `${apiOrigin}/${path}`; @@ -97,7 +97,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return request.text(); } - async syncEntityDocs(_: EntityName): Promise { + async syncEntityDocs(_: CompoundEntityRef): Promise { // this is just stub of this function as we don't need to check if docs are up to date, // we always want to retrigger a new build return 'cached'; @@ -106,7 +106,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { // Used by transformer to modify the request to assets (CSS, Image) from inside the HTML. async getBaseUrl( oldBaseUrl: string, - _entityId: EntityName, + _entityId: CompoundEntityRef, path: string, ): Promise { const apiOrigin = await this.getApiOrigin(); @@ -154,7 +154,7 @@ class TechDocsDevApi implements TechDocsApi { }; } - async getTechDocsMetadata(_entityId: EntityName) { + async getTechDocsMetadata(_entityId: CompoundEntityRef) { return { site_name: 'Live preview environment', site_description: '', diff --git a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx index d9a518c165..f2dcefc00e 100644 --- a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx +++ b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx @@ -29,7 +29,7 @@ import LightIcon from '@material-ui/icons/Brightness7'; import DarkIcon from '@material-ui/icons/Brightness4'; import { lightTheme, darkTheme } from '@backstage/theme'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Content } from '@backstage/core-components'; @@ -127,7 +127,7 @@ const TechDocsPageContent = ({ onReady, entityRef, }: { - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => { const classes = useStyles(); diff --git a/packages/techdocs-cli/src/commands/generate/generate.ts b/packages/techdocs-cli/src/commands/generate/generate.ts index a255805139..0f5213df34 100644 --- a/packages/techdocs-cli/src/commands/generate/generate.ts +++ b/packages/techdocs-cli/src/commands/generate/generate.ts @@ -39,6 +39,7 @@ export default async function generate(cmd: Command) { const sourceDir = resolve(cmd.sourceDir); const outputDir = resolve(cmd.outputDir); + const omitTechdocsCorePlugin = cmd.omitTechdocsCoreMkdocsPlugin; const dockerImage = cmd.dockerImage; const pullImage = cmd.pull; @@ -55,6 +56,9 @@ export default async function generate(cmd: Command) { runIn: cmd.docker ? 'docker' : 'local', dockerImage, pullImage, + mkdocs: { + omitTechdocsCorePlugin, + }, }, }, }); diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index 115fe8e364..6f1d9847e1 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -54,6 +54,11 @@ export function registerCommands(program: CommanderStatic) { 'A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json.', ) .option('-v --verbose', 'Enable verbose output.', false) + .option( + '--omitTechdocsCoreMkdocsPlugin', + "Don't patch MkDocs file automatically with techdocs-core plugin.", + false, + ) .alias('build') .action(lazy(() => import('./generate/generate').then(m => m.default))); diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index 7c4fc95abb..13d610f66b 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -5,10 +5,10 @@ ```ts /// +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { ContainerRunner } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; @@ -157,7 +157,9 @@ export class Publisher { // @public export interface PublisherBase { docsRouter(): express.Handler; - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; getReadiness(): Promise; hasDocsBeenGenerated(entityName: Entity): Promise; migrateDocsCase?(migrateRequest: MigrateRequest): Promise; diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml new file mode 100644 index 0000000000..09e8fd7ac7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml @@ -0,0 +1,6 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ +plugins: + - not-techdocs-core + - also-not-techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml new file mode 100644 index 0000000000..eea9a8a3d9 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml @@ -0,0 +1,5 @@ +site_name: Test site name +site_description: Test site description +# This is a comment that is removed after editing +plugins: + - techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml new file mode 100644 index 0000000000..e75b06ada7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml @@ -0,0 +1,3 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index 9323ab4f18..7590df94fb 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -28,10 +28,14 @@ import { getMkdocsYml, getRepoUrlFromLocationAnnotation, patchIndexPreBuild, - patchMkdocsYmlPreBuild, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; +import yaml from 'js-yaml'; const mockEntity = { apiVersion: 'version', @@ -65,6 +69,15 @@ const mkdocsYmlWithInvalidDocDir2 = fs.readFileSync( const mkdocsYmlWithComments = fs.readFileSync( resolvePath(__filename, '../__fixtures__/mkdocs_with_comments.yml'), ); +const mkdocsYmlWithTechdocsPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_techdocs_plugin.yml'), +); +const mkdocsYmlWithoutPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_without_plugins.yml'), +); +const mkdocsYmlWithAdditionalPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_additional_plugins.yml'), +); const mockLogger = getVoidLogger(); const warn = jest.spyOn(mockLogger, 'warn'); @@ -289,6 +302,60 @@ describe('helpers', () => { }); }); + describe('pathMkdocsYmlWithTechdocsPlugin', () => { + beforeEach(() => { + mockFs({ + '/mkdocs_with_techdocs_plugin.yml': mkdocsYmlWithTechdocsPlugins, + '/mkdocs_without_plugins.yml': mkdocsYmlWithoutPlugins, + '/mkdocs_with_additional_plugins.yml': mkdocsYmlWithAdditionalPlugins, + }); + }); + it('should not add additional plugins if techdocs exists already in mkdocs file', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_techdocs_plugin.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_techdocs_plugin.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it("should add the needed plugin if it doesn't exist in mkdocs file", async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_without_plugins.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile('/mkdocs_without_plugins.yml'); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it('should not override existing plugins', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_additional_plugins.yml', + mockLogger, + ); + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_additional_plugins.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(3); + expect(parsedYml.plugins).toContain('techdocs-core'); + expect(parsedYml.plugins).toContain('not-techdocs-core'); + expect(parsedYml.plugins).toContain('also-not-techdocs-core'); + }); + }); + describe('patchIndexPreBuild', () => { afterEach(() => { warn.mockClear(); diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index 2debd5822c..349783d2bd 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -125,7 +125,7 @@ class UnknownTag { constructor(public readonly data: any, public readonly type?: string) {} } -const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ +export const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ new Type('', { kind: 'scalar', multi: true, @@ -203,101 +203,6 @@ export const validateMkdocsYaml = async ( return parsedMkdocsYml.docs_dir; }; -/** - * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. - * - * List of tasks: - * - Add repo_url or edit_uri if it does not exists - * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. - * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get - * the repository URL. - * - * This function will not throw an error since this is not critical to the whole TechDocs pipeline. - * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. - * - * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site - * @param logger - A logger instance - * @param parsedLocationAnnotation - Object with location url and type - * @param scmIntegrations - the scmIntegration to do url transformations - */ -export const patchMkdocsYmlPreBuild = async ( - mkdocsYmlPath: string, - logger: Logger, - parsedLocationAnnotation: ParsedLocationAnnotation, - scmIntegrations: ScmIntegrationRegistry, -) => { - // We only want to override the mkdocs.yml if it has actually changed. This is relevant if - // used with a 'dir' location on the file system as this would permanently update the file. - let didEdit = false; - - let mkdocsYmlFileString; - try { - mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); - } catch (error) { - assertError(error); - logger.warn( - `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, - ); - return; - } - - let mkdocsYml: any; - try { - mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); - - // mkdocsYml should be an object type after successful parsing. - // But based on its type definition, it can also be a string or undefined, which we don't want. - if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { - throw new Error('Bad YAML format.'); - } - } catch (error) { - assertError(error); - logger.warn( - `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, - ); - return; - } - - // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. - // This will enable the Page edit button generated by MkDocs. - // If the either has been set, keep the original value - if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { - const result = getRepoUrlFromLocationAnnotation( - parsedLocationAnnotation, - scmIntegrations, - mkdocsYml.docs_dir, - ); - - if (result.repo_url || result.edit_uri) { - mkdocsYml.repo_url = result.repo_url; - mkdocsYml.edit_uri = result.edit_uri; - didEdit = true; - - logger.info( - `Set ${JSON.stringify( - result, - )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, - ); - } - } - - try { - if (didEdit) { - await fs.writeFile( - mkdocsYmlPath, - yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), - 'utf8', - ); - } - } catch (error) { - assertError(error); - logger.warn( - `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, - ); - return; - } -}; - /** * Update docs/index.md file before TechDocs generator uses it to generate docs site, * falling back to docs/README.md or README.md in case a default docs/index.md diff --git a/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts new file mode 100644 index 0000000000..d03b5d83c2 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import fs from 'fs-extra'; +import yaml from 'js-yaml'; +import { ParsedLocationAnnotation } from '../../helpers'; +import { getRepoUrlFromLocationAnnotation, MKDOCS_SCHEMA } from './helpers'; +import { assertError } from '@backstage/errors'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +type MkDocsObject = { + plugins?: string[]; + docs_dir: string; + repo_url?: string; + edit_uri?: string; +}; + +const patchMkdocsFile = async ( + mkdocsYmlPath: string, + logger: Logger, + updateAction: (mkdocsYml: MkDocsObject) => boolean, +) => { + // We only want to override the mkdocs.yml if it has actually changed. This is relevant if + // used with a 'dir' location on the file system as this would permanently update the file. + let didEdit = false; + + let mkdocsYmlFileString; + try { + mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); + } catch (error) { + assertError(error); + logger.warn( + `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, + ); + return; + } + + let mkdocsYml: any; + try { + mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); + + // mkdocsYml should be an object type after successful parsing. + // But based on its type definition, it can also be a string or undefined, which we don't want. + if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { + throw new Error('Bad YAML format.'); + } + } catch (error) { + assertError(error); + logger.warn( + `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, + ); + return; + } + + didEdit = updateAction(mkdocsYml); + + try { + if (didEdit) { + await fs.writeFile( + mkdocsYmlPath, + yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), + 'utf8', + ); + } + } catch (error) { + assertError(error); + logger.warn( + `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, + ); + return; + } +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add repo_url or edit_uri if it does not exists + * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. + * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get + * the repository URL. + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + * @param parsedLocationAnnotation - Object with location url and type + * @param scmIntegrations - the scmIntegration to do url transformations + */ +export const patchMkdocsYmlPreBuild = async ( + mkdocsYmlPath: string, + logger: Logger, + parsedLocationAnnotation: ParsedLocationAnnotation, + scmIntegrations: ScmIntegrationRegistry, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { + // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. + // This will enable the Page edit button generated by MkDocs. + // If the either has been set, keep the original value + const result = getRepoUrlFromLocationAnnotation( + parsedLocationAnnotation, + scmIntegrations, + mkdocsYml.docs_dir, + ); + + if (result.repo_url || result.edit_uri) { + mkdocsYml.repo_url = result.repo_url; + mkdocsYml.edit_uri = result.edit_uri; + + logger.info( + `Set ${JSON.stringify( + result, + )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, + ); + return true; + } + } + return false; + }); +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add techdocs-core plugin to mkdocs file if it doesn't exist + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + */ +export const pathMkdocsYmlWithTechdocsPlugin = async ( + mkdocsYmlPath: string, + logger: Logger, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + // Modify mkdocs.yaml to contain the needed techdocs-core plugin if it is not there + if (!('plugins' in mkdocsYml)) { + mkdocsYml.plugins = ['techdocs-core']; + return true; + } + + if (mkdocsYml.plugins && !mkdocsYml.plugins.includes('techdocs-core')) { + mkdocsYml.plugins.push('techdocs-core'); + return true; + } + return false; + }); +}; diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index baa2e008a1..b46f52f99a 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -26,11 +26,15 @@ import { createOrUpdateMetadata, getMkdocsYml, patchIndexPreBuild, - patchMkdocsYmlPreBuild, runCommand, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; + +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; import { GeneratorBase, GeneratorConfig, @@ -110,6 +114,10 @@ export class TechdocsGenerator implements GeneratorBase { await patchIndexPreBuild({ inputDir, logger: childLogger, docsDir }); } + if (!this.options.omitTechdocsCoreMkdocsPlugin) { + await pathMkdocsYmlWithTechdocsPlugin(mkdocsYmlPath, childLogger); + } + // Directories to bind on container const mountDirs = { [inputDir]: '/input', @@ -207,5 +215,8 @@ export function readGeneratorConfig( 'docker', dockerImage: config.getOptionalString('techdocs.generator.dockerImage'), pullImage: config.getOptionalBoolean('techdocs.generator.pullImage'), + omitTechdocsCoreMkdocsPlugin: config.getOptionalBoolean( + 'techdocs.generator.mkdocs.omitTechdocsCorePlugin', + ), }; } diff --git a/packages/techdocs-common/src/stages/generate/types.ts b/packages/techdocs-common/src/stages/generate/types.ts index 2ff3064991..f46dbdc56a 100644 --- a/packages/techdocs-common/src/stages/generate/types.ts +++ b/packages/techdocs-common/src/stages/generate/types.ts @@ -39,6 +39,7 @@ export type GeneratorConfig = { runIn: GeneratorRunInType; dockerImage?: string; pullImage?: boolean; + omitTechdocsCoreMkdocsPlugin?: boolean; }; /** diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index ed76edc0cb..e5914d2fdf 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import aws, { Credentials } from 'aws-sdk'; @@ -321,7 +321,7 @@ export class AwsS3Publish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { try { return await new Promise(async (resolve, reject) => { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index b082079be2..bcbc10a94d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -19,7 +19,7 @@ import { ContainerClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import express from 'express'; @@ -300,7 +300,7 @@ export class AzureBlobStoragePublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityRootDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index c3d9d3330b..987dfc6f09 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError } from '@backstage/errors'; import { File, FileExistsResponse, Storage } from '@google-cloud/storage'; @@ -238,7 +238,9 @@ export class GoogleGCSPublish implements PublisherBase { return { objects }; } - fetchTechDocsMetadata(entityName: EntityName): Promise { + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise { return new Promise((resolve, reject) => { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 9c146d2935..af9bb30d46 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -17,7 +17,7 @@ import { PluginEndpointDiscovery, resolvePackagePath, } from '@backstage/backend-common'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -142,7 +142,7 @@ export class LocalPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const metadataPath = this.staticEntityPathJoin( entityName.namespace, diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index b240ce73da..da0a60343f 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; @@ -45,7 +45,7 @@ const createMockEntity = (annotations = {}): Entity => { }; }; -const createMockEntityName = (): EntityName => ({ +const createMockEntityName = (): CompoundEntityRef => ({ kind: 'TestKind', name: 'test-component-name', namespace: 'test-namespace', diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 62b40f9a76..734a7e4451 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -194,7 +194,7 @@ export class OpenStackSwiftPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { return await new Promise(async (resolve, reject) => { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index c6ad032484..86772c25f4 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Logger } from 'winston'; import express from 'express'; @@ -133,7 +133,9 @@ export interface PublisherBase { * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. */ - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; /** * Route middleware to serve static documentation files for an entity. diff --git a/plugins/airbrake-backend/README.md b/plugins/airbrake-backend/README.md index 9c9b56ade4..21056229de 100644 --- a/plugins/airbrake-backend/README.md +++ b/plugins/airbrake-backend/README.md @@ -24,7 +24,7 @@ This method of serving the plugin provides quicker iteration speed and a faster 3. Go into the plugin's directory and run it in standalone mode by running `yarn start`. -Access it from http://localhost:7007/api/airbrake. Or use the Airbrake plugin which will talk to it automatically. +Access it from http://localhost:7007/api/airbrake. Or use the [Airbrake plugin in standalone mode](../airbrake/README.md#local-development) which will talk to it automatically. Here are some example endpoints: diff --git a/plugins/airbrake/README.md b/plugins/airbrake/README.md index d5ea9bb006..bc830c326d 100644 --- a/plugins/airbrake/README.md +++ b/plugins/airbrake/README.md @@ -20,7 +20,7 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: yarn add @backstage/plugin-airbrake-backend ``` -3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx`: +3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx` for all the entity pages you want Airbrake to be in: ```typescript jsx import { EntityAirbrakeContent } from '@backstage/plugin-airbrake'; @@ -32,40 +32,78 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: ); + + const websiteEntityPage = ( + + + + + + ); + + const defaultEntityPage = ( + + + + + + ); ``` -4. Setup the Backend code in `packages/backend/src/index.ts`: +4. Create `packages/backend/src/plugins/airbrake.ts` with these contents: ```typescript + import { Router } from 'express'; + import { PluginEnvironment } from '../types'; import { - createRouter as createAirbrakeRouter, + createRouter, extractAirbrakeConfig, } from '@backstage/plugin-airbrake-backend'; - async function main() { - //... After const config = await loadBackendConfig({ ... - - const airbrakeRouter = await createAirbrakeRouter({ + export default async function createPlugin({ + logger, + config, + }: PluginEnvironment): Promise { + return createRouter({ logger, airbrakeConfig: extractAirbrakeConfig(config), }); - - const service = createServiceBuilder(module) - // ... Add the airbrakeRouter here - .addRouter('/api/airbrake', airbrakeRouter); } ``` -5. Add this config as a top level section in your `app-config.yaml`: +5. Setup the Backend code in `packages/backend/src/index.ts`: + + ```typescript + import airbrake from './plugins/airbrake'; + + async function main() { + //... After const createEnv = makeCreateEnv(config) ... + + const airbrakeEnv = useHotMemoize(module, () => createEnv('airbrake')); + + //... After const apiRouter = Router() ... + apiRouter.use('/airbrake', await airbrake(airbrakeEnv)); + } + ``` + +6. Add this config as a top level section in your `app-config.yaml`: ```yaml airbrake: apiKey: ${AIRBRAKE_API_KEY} ``` -6. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) +7. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) before starting Backstage backend. +8. Add the following annotation to the `catalog-info.yaml` for a repo you want to link to an Airbrake project: + + ```yaml + metadata: + annotations: + airbrake.io/project-id: '123456' + ``` + ## Local Development Start this plugin in standalone mode by running `yarn start` inside the plugin directory. This method of serving the plugin provides quicker diff --git a/plugins/api-docs/dev/index.tsx b/plugins/api-docs/dev/index.tsx index 633a285b98..9975e91e98 100644 --- a/plugins/api-docs/dev/index.tsx +++ b/plugins/api-docs/dev/index.tsx @@ -53,8 +53,8 @@ createDevApp() items: mockEntities.slice(), }; }, - async getEntityByName(name: string) { - return mockEntities.find(e => e.metadata.name === name); + async getEntityByRef(ref: string) { + return mockEntities.find(e => e.metadata.name === ref); }, } as unknown as typeof catalogApiRef.T), }) diff --git a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx index 5924af3ef3..bc413661a8 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, RELATION_MEMBER_OF } from '@backstage/catalog-model'; +import { + Entity, + parseEntityRef, + RELATION_MEMBER_OF, +} from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/core-app-api'; import { TableColumn, TableProps } from '@backstage/core-components'; import { @@ -60,11 +64,11 @@ describe('DefaultApiExplorerPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, diff --git a/plugins/auth-backend/src/identity/TokenFactory.test.ts b/plugins/auth-backend/src/identity/TokenFactory.test.ts index f5f06f1209..4b1c1804df 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.test.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.test.ts @@ -18,6 +18,7 @@ import { MemoryKeyStore } from './MemoryKeyStore'; import { TokenFactory } from './TokenFactory'; import { getVoidLogger } from '@backstage/backend-common'; import { JWKS, JSONWebKey, JWT } from 'jose'; +import { stringifyEntityRef } from '@backstage/catalog-model'; const logger = getVoidLogger(); @@ -28,6 +29,12 @@ function jwtKid(jwt: string): string { return header.kid; } +const entityRef = stringifyEntityRef({ + kind: 'User', + namespace: 'default', + name: 'JackFrost', +}); + describe('TokenFactory', () => { it('should issue valid tokens signed by a listed key', async () => { const keyDurationSeconds = 5; @@ -39,7 +46,7 @@ describe('TokenFactory', () => { }); await expect(factory.listPublicKeys()).resolves.toEqual({ keys: [] }); - const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const token = await factory.issueToken({ claims: { sub: entityRef } }); const { keys } = await factory.listPublicKeys(); const keyStore = JWKS.asKeyStore({ @@ -53,7 +60,7 @@ describe('TokenFactory', () => { expect(payload).toEqual({ iss: 'my-issuer', aud: 'backstage', - sub: 'foo', + sub: entityRef, iat: expect.any(Number), exp: expect.any(Number), }); @@ -71,8 +78,12 @@ describe('TokenFactory', () => { logger, }); - const token1 = await factory.issueToken({ claims: { sub: 'foo' } }); - const token2 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token1 = await factory.issueToken({ + claims: { sub: entityRef }, + }); + const token2 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token1)).toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -89,7 +100,9 @@ describe('TokenFactory', () => { keys: [], }); - const token3 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token3 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token3)).not.toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -100,4 +113,20 @@ describe('TokenFactory', () => { ], }); }); + + it('should throw an error with a non entityRef sub claim', async () => { + const keyDurationSeconds = 5; + const factory = new TokenFactory({ + issuer: 'my-issuer', + keyStore: new MemoryKeyStore(), + keyDurationSeconds, + logger, + }); + + await expect(() => { + return factory.issueToken({ + claims: { sub: 'UserId' }, + }); + }).rejects.toThrowError(); + }); }); diff --git a/plugins/auth-backend/src/identity/TokenFactory.ts b/plugins/auth-backend/src/identity/TokenFactory.ts index 041a292d44..cb2a8b453d 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.ts @@ -19,6 +19,7 @@ import { JSONWebKey, JWK, JWS } from 'jose'; import { Logger } from 'winston'; import { v4 as uuid } from 'uuid'; import { DateTime } from 'luxon'; +import { parseEntityRef } from '@backstage/catalog-model'; const MS_IN_S = 1000; @@ -72,6 +73,15 @@ export class TokenFactory implements TokenIssuer { const iat = Math.floor(Date.now() / MS_IN_S); const exp = iat + this.keyDurationSeconds; + // Validate that the subject claim is a valid EntityRef + try { + parseEntityRef(sub); + } catch (error) { + throw new Error( + '"sub" claim provided by the auth resolver is not a valid EntityRef.', + ); + } + this.logger.info(`Issuing token for ${sub}, with entities ${ent ?? []}`); return JWS.sign({ iss, sub, aud, iat, exp, ent }, key, { diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts index d2cee7bd89..08c46f2c21 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts @@ -26,6 +26,7 @@ import { CatalogIdentityClient } from './CatalogIdentityClient'; describe('CatalogIdentityClient', () => { const catalogApi: jest.Mocked = { getLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getEntities: jest.fn(), addLocation: jest.fn(), diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts index d948af2e05..502da913af 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts @@ -18,7 +18,7 @@ import { Logger } from 'winston'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; import { - EntityName, + CompoundEntityRef, parseEntityRef, RELATION_MEMBER_OF, stringifyEntityRef, @@ -96,7 +96,7 @@ export class CatalogIdentityClient { return null; } }) - .filter((ref): ref is EntityName => ref !== null); + .filter((ref): ref is CompoundEntityRef => ref !== null); const filter = resolvedEntityRefs.map(ref => ({ kind: ref.kind, diff --git a/plugins/badges-backend/src/service/router.test.ts b/plugins/badges-backend/src/service/router.test.ts index 56dbfb0655..c492449cb3 100644 --- a/plugins/badges-backend/src/service/router.test.ts +++ b/plugins/badges-backend/src/service/router.test.ts @@ -60,6 +60,7 @@ describe('createRouter', () => { catalog = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -103,7 +104,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge-specs', () => { it('returns all badge specs for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.getBadges.mockResolvedValueOnce([{ id: badge.id }]); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); @@ -115,8 +116,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.text).toEqual(JSON.stringify([badge], null, 2)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -142,7 +143,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge/test-badge', () => { it('returns badge for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); const image = '...'; badgeBuilder.createBadgeSvg.mockResolvedValueOnce(image); @@ -154,8 +155,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.body).toEqual(Buffer.from(image)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -179,7 +180,7 @@ describe('createRouter', () => { }); it('returns badge spec for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); const url = '/entity/default/service/test/badge/test-badge?format=json'; @@ -192,7 +193,7 @@ describe('createRouter', () => { describe('Errors', () => { it('returns 404 for unknown entities', async () => { - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); async function testUrl(url: string) { const response = await request(app).get(url); expect(response.status).toEqual(404); diff --git a/plugins/badges-backend/src/service/router.ts b/plugins/badges-backend/src/service/router.ts index 196ddf36da..b7211a0aad 100644 --- a/plugins/badges-backend/src/service/router.ts +++ b/plugins/badges-backend/src/service/router.ts @@ -46,7 +46,7 @@ export async function createRouter( router.get('/entity/:namespace/:kind/:name/badge-specs', async (req, res) => { const { namespace, kind, name } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), @@ -84,7 +84,7 @@ export async function createRouter( '/entity/:namespace/:kind/:name/badge/:badgeId', async (req, res) => { const { namespace, kind, name, badgeId } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 0e088d3a4e..a253359471 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -35,17 +35,38 @@ catalog: # the App registration in the Microsoft Azure Portal. clientId: ${MICROSOFT_GRAPH_CLIENT_ID} clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + userExpand: manager # Optional filter for user, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter # This and userGroupMemberFilter are mutually exclusive, only one can be specified userFilter: accountEnabled eq true and userType eq 'member' # Optional filter for users, use group membership to get users. + # (Filtered groups and fetch their members.) # This and userFilter are mutually exclusive, only one can be specified + # See https://docs.microsoft.com/en-us/graph/search-query-parameter userGroupMemberFilter: "displayName eq 'Backstage Users'" + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + groupExpand: member + # Optional search for users, use group membership to get users. + # (Search for groups and fetch their members.) + # This and userFilter are mutually exclusive, only one can be specified + userGroupMemberSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' # Optional filter for group, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + # Optional search for groups, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/search-query-parameter + groupSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' ``` `userFilter` and `userGroupMemberFilter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 7ad603e6ca..8df06f54b1 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -78,7 +78,10 @@ export class MicrosoftGraphClient { userId: string, maxSize: number, ): Promise; - getUserProfile(userId: string): Promise; + getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise; getUsers(query?: ODataQuery): AsyncIterable; requestApi( path: string, @@ -158,9 +161,10 @@ export type MicrosoftGraphProviderConfig = { clientId: string; clientSecret: string; userFilter?: string; - userExpand?: string[]; + userExpand?: string; userGroupMemberFilter?: string; userGroupMemberSearch?: string; + groupExpand?: string; groupFilter?: string; groupSearch?: string; }; @@ -172,7 +176,7 @@ export function normalizeEntityName(name: string): string; export type ODataQuery = { search?: string; filter?: string; - expand?: string[]; + expand?: string; select?: string[]; }; @@ -191,10 +195,11 @@ export function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts index 6413a97c15..94a126e1a7 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts @@ -85,7 +85,7 @@ describe('MicrosoftGraphClient', () => { const response = await client.requestApi('users', { filter: 'test eq true', - expand: ['children'], + expand: 'children', select: ['id', 'children'], }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index db749a536e..cba5d9f640 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -38,7 +38,7 @@ export type ODataQuery = { /** * specifies the related resources or media streams to be included in line with retrieved resources */ - expand?: string[]; + expand?: string; /** * request a specific set of properties for each entity or complex type */ @@ -155,7 +155,7 @@ export class MicrosoftGraphClient { $search: query?.search, $filter: query?.filter, $select: query?.select?.join(','), - $expand: query?.expand?.join(','), + $expand: query?.expand, }, { addQueryPrefix: true, @@ -203,10 +203,14 @@ export class MicrosoftGraphClient { * * @public * @param userId - The unique identifier for the `User` resource + * @param query - OData Query {@link ODataQuery} * */ - async getUserProfile(userId: string): Promise { - const response = await this.requestApi(`users/${userId}`); + async getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise { + const response = await this.requestApi(`users/${userId}`, query); if (response.status !== 200) { await this.handleError('user profile', response); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index efbbca1f5c..cfc5c1cbb0 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -53,7 +53,9 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com/', + userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ], @@ -66,7 +68,9 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com', + userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ]; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 6809144f80..b7fbcfb8a9 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -57,7 +57,7 @@ export type MicrosoftGraphProviderConfig = { * * E.g. "manager" */ - userExpand?: string[]; + userExpand?: string; /** * The filter to apply to extract users by groups memberships. * @@ -70,6 +70,12 @@ export type MicrosoftGraphProviderConfig = { * E.g. "\"displayName:-team\"" would only match groups which contain '-team' */ userGroupMemberSearch?: string; + /** + * The "expand" argument to apply to groups. + * + * E.g. "member" + */ + groupExpand?: string; /** * The filter to apply to extract groups. * @@ -106,6 +112,8 @@ export function readMicrosoftGraphConfig( const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); const clientSecret = providerConfig.getString('clientSecret'); + + const userExpand = providerConfig.getOptionalString('userExpand'); const userFilter = providerConfig.getOptionalString('userFilter'); const userGroupMemberFilter = providerConfig.getOptionalString( 'userGroupMemberFilter', @@ -113,6 +121,7 @@ export function readMicrosoftGraphConfig( const userGroupMemberSearch = providerConfig.getOptionalString( 'userGroupMemberSearch', ); + const groupExpand = providerConfig.getOptionalString('groupExpand'); const groupFilter = providerConfig.getOptionalString('groupFilter'); const groupSearch = providerConfig.getOptionalString('groupSearch'); @@ -133,9 +142,11 @@ export function readMicrosoftGraphConfig( tenantId, clientId, clientSecret, + userExpand, userFilter, userGroupMemberFilter, userGroupMemberSearch, + groupExpand, groupFilter, groupSearch, }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index b0273e1f58..e3d6897d66 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -118,7 +118,7 @@ describe('read microsoft graph', () => { expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand and custom transformer', async () => { async function* getExampleUsers() { yield { id: 'userid', @@ -133,6 +133,7 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsers(client, { + userExpand: 'manager', userFilter: 'accountEnabled eq true', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', @@ -154,6 +155,7 @@ describe('read microsoft graph', () => { expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); @@ -227,12 +229,14 @@ describe('read microsoft graph', () => { expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: undefined, + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand, groupExpand and custom transformer', async () => { async function* getExampleGroups() { yield { id: 'groupid', @@ -266,7 +270,9 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsersInGroups(client, { + userExpand: 'manager', userGroupMemberFilter: 'securityEnabled eq true', + groupExpand: 'member', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', kind: 'User', @@ -287,13 +293,16 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledTimes(1); expect(client.getGroups).toBeCalledWith({ + expand: 'member', filter: 'securityEnabled eq true', }); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: 'manager', + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); @@ -446,6 +455,100 @@ describe('read microsoft graph', () => { // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); }); + it('should read groups with groupExpand', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + groupExpand: 'member', + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + expand: 'member', + filter: 'securityEnabled eq false', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); + it('should read security groups', async () => { async function* getExampleGroups() { yield { @@ -634,6 +737,14 @@ describe('read microsoft graph', () => { }; } + async function getExampleUserProfile(userId: string) { + return { + id: userId, + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + async function* getExampleGroups() { yield { id: 'groupid', @@ -686,7 +797,7 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userFilter', async () => { + it('should read users using userExpand and userFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', @@ -705,12 +816,14 @@ describe('read microsoft graph', () => { await readMicrosoftGraphOrg(client, 'tenantid', { logger: getVoidLogger(), + userExpand: 'manager', userFilter: 'accountEnabled eq true', groupFilter: 'securityEnabled eq false', }); expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getGroups).toBeCalledTimes(1); @@ -719,13 +832,14 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userGroupMemberFilter', async () => { + it('should read users using userExpand and userGroupMemberFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', }); client.getUsers.mockImplementation(getExampleUsers); + client.getUserProfile.mockImplementation(getExampleUserProfile); client.getUserPhotoWithSizeLimit.mockResolvedValue( 'data:image/jpeg;base64,...', ); @@ -750,6 +864,8 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledWith({ filter: 'securityEnabled eq false', }); + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); }); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index f862fd5ace..ed6ca76e68 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -85,7 +85,7 @@ export async function readMicrosoftGraphUsers( client: MicrosoftGraphClient, options: { userFilter?: string; - userExpand?: string[]; + userExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -137,8 +137,10 @@ export async function readMicrosoftGraphUsers( export async function readMicrosoftGraphUsersInGroups( client: MicrosoftGraphClient, options: { + userExpand?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -149,15 +151,16 @@ export async function readMicrosoftGraphUsersInGroups( const limiter = limiterFactory(10); - const transformer = options?.transformer ?? defaultUserTransformer; + const transformer = options.transformer ?? defaultUserTransformer; const userGroupMemberPromises: Promise[] = []; const userPromises: Promise[] = []; const groupMemberUsers: Set = new Set(); for await (const group of client.getGroups({ - search: options?.userGroupMemberSearch, - filter: options?.userGroupMemberFilter, + expand: options.groupExpand, + search: options.userGroupMemberSearch, + filter: options.userGroupMemberFilter, })) { // Process all groups in parallel, otherwise it can take quite some time userGroupMemberPromises.push( @@ -186,7 +189,9 @@ export async function readMicrosoftGraphUsersInGroups( let user; let userPhoto; try { - user = await client.getUserProfile(userId); + user = await client.getUserProfile(userId, { + expand: options.userExpand, + }); } catch (e) { options.logger.warn(`Unable to load user for ${userId}`); } @@ -326,8 +331,9 @@ export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, options?: { - groupSearch?: string; + groupExpand?: string; groupFilter?: string; + groupSearch?: string; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }, @@ -354,6 +360,7 @@ export async function readMicrosoftGraphGroups( const promises: Promise[] = []; for await (const group of client.getGroups({ + expand: options?.groupExpand, search: options?.groupSearch, filter: options?.groupFilter, })) { @@ -506,10 +513,11 @@ export async function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 942bde19eb..9037bd8728 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -109,6 +109,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { userFilter: provider.userFilter, userGroupMemberFilter: provider.userGroupMemberFilter, userGroupMemberSearch: provider.userGroupMemberSearch, + groupExpand: provider.groupExpand, groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, userTransformer: this.userTransformer, diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6f0147a1f4..a1125ec529 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -7,12 +7,12 @@ import { BitbucketIntegration } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import express from 'express'; import { GetEntitiesRequest } from '@backstage/catalog-client'; @@ -30,6 +30,7 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common'; import { PermissionRule } from '@backstage/plugin-permission-node'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TokenManager } from '@backstage/backend-common'; @@ -217,7 +218,13 @@ export class CatalogBuilder { key: string, resolver: PlaceholderResolver, ): CatalogBuilder; + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder; + setProcessingIntervalSeconds(seconds: number): CatalogBuilder; + // @deprecated setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder; + // @deprecated setRefreshIntervalSeconds(seconds: number): CatalogBuilder; } @@ -407,6 +414,12 @@ export const createCatalogPolicyDecision: ( ) => ConditionalPolicyDecision; // @public +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction; + +// @public @deprecated export function createRandomRefreshInterval(options: { minSeconds: number; maxSeconds: number; @@ -415,8 +428,8 @@ export function createRandomRefreshInterval(options: { // @public export function createRouter(options: RouterOptions): Promise; -// @public (undocumented) -export class DefaultCatalogCollator implements DocumentCollator { +// @public @deprecated (undocumented) +export class DefaultCatalogCollator { constructor(options: { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; @@ -456,6 +469,31 @@ export class DefaultCatalogCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + // @public (undocumented) export class DefaultCatalogProcessingOrchestrator implements CatalogProcessingOrchestrator @@ -623,9 +661,9 @@ export type EntityProviderMutation = // @public export type EntityRelationSpec = { - source: EntityName; + source: CompoundEntityRef; type: string; - target: EntityName; + target: CompoundEntityRef; }; // @public (undocumented) @@ -963,6 +1001,9 @@ export type PlaceholderResolverResolveUrl = ( base: string, ) => string; +// @public +export type ProcessingIntervalFunction = () => number; + // @public export const processingResult: Readonly<{ readonly notFoundError: ( @@ -997,7 +1038,7 @@ export type RecursivePartial = { : T[P]; }; -// @public +// @public @deprecated export type RefreshIntervalFunction = () => number; // @public diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index be0772e199..f7d946c531 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -73,6 +73,7 @@ "@backstage/backend-test-utils": "^0.1.19", "@backstage/cli": "^0.14.1", "@backstage/plugin-permission-common": "^0.5.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", diff --git a/plugins/catalog-backend/src/api/common.ts b/plugins/catalog-backend/src/api/common.ts index 1fd09aec32..f3b4a387ba 100644 --- a/plugins/catalog-backend/src/api/common.ts +++ b/plugins/catalog-backend/src/api/common.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * Holds the entity location information. @@ -42,7 +42,7 @@ export type EntityRelationSpec = { /** * The source entity of this relation. */ - source: EntityName; + source: CompoundEntityRef; /** * The type of the relation. @@ -52,5 +52,5 @@ export type EntityRelationSpec = { /** * The target entity of this relation. */ - target: EntityName; + target: CompoundEntityRef; }; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 729be442aa..c7cd6b7026 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../processing/refresh'; +import { createRandomProcessingInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; @@ -49,7 +49,7 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshInterval: createRandomRefreshInterval({ + refreshInterval: createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }), diff --git a/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts index 548cdf3115..9e420b9d65 100644 --- a/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts @@ -22,7 +22,7 @@ import { DomainEntity, domainEntityV1alpha1Validator, Entity, - getEntityName, + getCompoundEntityRef, GroupEntity, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, @@ -93,7 +93,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); /* * Utilities diff --git a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts index 81ebd52789..c6c5951153 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for the presence of an annotation on a given entity. * - * @public + * @alpha */ export const hasAnnotation = createCatalogPermissionRule({ name: 'HAS_ANNOTATION', diff --git a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts index 04b00d68fa..7296eecd89 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified label in its metadata. - * @public + * @alpha */ export const hasLabel = createCatalogPermissionRule({ name: 'HAS_LABEL', diff --git a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts index f5f25a5ecf..cf14faea14 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasMetadata = createPropertyRule('metadata'); diff --git a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts index 891cf1d58c..63ae67d01c 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasSpec = createPropertyRule('spec'); diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts index 6356c94dc4..c330c903d9 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified kind. - * @public + * @alpha */ export const isEntityKind = createCatalogPermissionRule({ name: 'IS_ENTITY_KIND', diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts index a6dda13c29..23f118abc4 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified owner. * - * @public + * @alpha */ export const isEntityOwner = createCatalogPermissionRule({ name: 'IS_ENTITY_OWNER', diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index dc8b07eceb..aec6a5d71c 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -23,5 +23,11 @@ export type { } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -export { createRandomRefreshInterval } from './refresh'; -export type { RefreshIntervalFunction } from './refresh'; +export { + createRandomRefreshInterval, + createRandomProcessingInterval, +} from './refresh'; +export type { + RefreshIntervalFunction, + ProcessingIntervalFunction, +} from './refresh'; diff --git a/plugins/catalog-backend/src/processing/refresh.ts b/plugins/catalog-backend/src/processing/refresh.ts index 3e03e3add8..941c338912 100644 --- a/plugins/catalog-backend/src/processing/refresh.ts +++ b/plugins/catalog-backend/src/processing/refresh.ts @@ -16,13 +16,21 @@ /** * Function that returns the catalog refresh interval in seconds. + * @deprecated use {@link ProcessingIntervalFunction} instead * @public */ export type RefreshIntervalFunction = () => number; +/** + * Function that returns the catalog processing interval in seconds. + * @public + */ +export type ProcessingIntervalFunction = () => number; + /** * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. * @returns A {@link RefreshIntervalFunction} that provides the next refresh interval + * @deprecated use {@link createRandomProcessingInterval} instead * @public */ export function createRandomRefreshInterval(options: { @@ -34,3 +42,18 @@ export function createRandomRefreshInterval(options: { return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } + +/** + * Creates a function that returns a random processing interval between minSeconds and maxSeconds. + * @returns A {@link ProcessingIntervalFunction} that provides the next processing interval + * @public + */ +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +} diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 96704a6c1a..c59318d0af 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -23,7 +23,6 @@ import { stringifyEntityRef, UserEntity, } from '@backstage/catalog-model'; -import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { CatalogApi, @@ -31,18 +30,14 @@ import { GetEntitiesRequest, } from '@backstage/catalog-client'; import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; -/** @public */ -export interface CatalogEntityDocument extends IndexableDocument { - componentType: string; - namespace: string; - kind: string; - lifecycle: string; - owner: string; -} - -/** @public */ -export class DefaultCatalogCollator implements DocumentCollator { +/** + * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultCatalogCollatorFactory` instead. + */ +export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; protected locationTemplate: string; protected filter?: GetEntitiesRequest['filter']; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts new file mode 100644 index 0000000000..ccb61b8680 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -0,0 +1,213 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; + +const server = setupServer(); + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-entity', + description: 'The expected description', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity-2', + description: 'The expected description 2', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultCatalogCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + }; + + beforeAll(() => { + server.listen(); + }); + + beforeEach(() => { + server.use( + rest.get('http://localhost:7007/entities', (req, res, ctx) => { + if (req.url.searchParams.has('filter')) { + const filter = req.url.searchParams.get('filter'); + if (filter === 'kind=Foo,kind=Bar') { + // When filtering on the 'Foo,Bar' kinds we simply return no items, to simulate a filter + return res(ctx.json([])); + } + throw new Error('Unexpected filter parameter'); + } + + // Imitate offset/limit pagination. + const offset = parseInt(req.url.searchParams.get('offset') || '0', 10); + const limit = parseInt(req.url.searchParams.get('limit') || '500', 10); + return res(ctx.json(expectedEntities.slice(offset, limit + offset))); + }), + ); + }); + + afterAll(() => { + server.close(); + }); + + afterEach(() => server.resetHandlers()); + + it('has expected type', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('software-catalog'); + }); + + describe('getCollator', () => { + let factory: DefaultCatalogCollatorFactory; + let collator: Readable; + + beforeEach(async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog service', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(documents).toHaveLength(expectedEntities.length); + }); + + it('maps a returned entity to an expected CatalogEntityDocument', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(documents[0]).toMatchObject({ + location: '/software/test-entity', + }); + }); + + it('allows filtering of the retrieved catalog entities', async () => { + // Provide a custom filter. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + filter: { + kind: ['Foo', 'Bar'], + }, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // The simulated 'Foo,Bar' filter should return in an empty list + expect(documents).toHaveLength(0); + }); + + it('paginates through catalog entities using batchSize', async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + batchSize: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(expectedEntities.length); + expect(documents[0].location).toBe( + '/catalog/default/component/test-entity', + ); + expect(documents[1].location).toBe( + '/catalog/default/component/test-entity-2', + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts new file mode 100644 index 0000000000..99377b2ca0 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -0,0 +1,173 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { + Entity, + stringifyEntityRef, + UserEntity, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + DocumentCollatorFactory, + IndexableDocument, +} from '@backstage/search-common'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { Readable } from 'stream'; + +/** @public */ +export interface CatalogEntityDocument extends IndexableDocument { + componentType: string; + namespace: string; + kind: string; + lifecycle: string; + owner: string; +} + +/** @public */ +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + +/** @public */ +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'software-catalog'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ) { + return new DefaultCatalogCollatorFactory(options); + } + + private constructor(options: DefaultCatalogCollatorFactoryOptions) { + const { + batchSize, + discovery, + locationTemplate, + filter, + catalogClient, + tokenManager, + } = options; + + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted.toLowerCase(); + } + + private isUserEntity(entity: Entity): entity is UserEntity { + return entity.kind.toLocaleUpperCase('en-US') === 'USER'; + } + + private getDocumentText(entity: Entity): string { + let documentText = entity.metadata.description || ''; + if (this.isUserEntity(entity)) { + if (entity.spec?.profile?.displayName && documentText) { + // combine displayName and description + const displayName = entity.spec?.profile?.displayName; + documentText = displayName.concat(' : ', documentText); + } else { + documentText = entity.spec?.profile?.displayName || documentText; + } + } + return documentText; + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield { + title: entity.metadata.title ?? entity.metadata.name, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + text: this.getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + }; + } + } + } +} diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 5641670333..93ff0b8b32 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -14,5 +14,11 @@ * limitations under the License. */ +export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; +export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; +export type { CatalogEntityDocument } from './DefaultCatalogCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting this in a future release. + */ export { DefaultCatalogCollator } from './DefaultCatalogCollator'; -export type { CatalogEntityDocument } from './DefaultCatalogCollator'; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f62377decb..b53335c469 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -76,8 +76,9 @@ import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; import { - createRandomRefreshInterval, + createRandomProcessingInterval, RefreshIntervalFunction, + ProcessingIntervalFunction, } from '../processing/refresh'; import { createRouter } from './createRouter'; import { DefaultRefreshService } from './DefaultRefreshService'; @@ -139,8 +140,8 @@ export class CatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshInterval: RefreshIntervalFunction = - createRandomRefreshInterval({ + private processingInterval: ProcessingIntervalFunction = + createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }); @@ -192,9 +193,28 @@ export class CatalogBuilder { * Seconds provided will be multiplied by 1.5 * The default refresh duration is 100-150 seconds. * setting this too low will potentially deplete request quotas to upstream services. + * + * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ setRefreshIntervalSeconds(seconds: number): CatalogBuilder { - this.refreshInterval = createRandomRefreshInterval({ + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshIntervalSeconds is deprecated. Use CatalogBuilder.setProcessingIntervalSeconds instead.', + ); + this.processingInterval = createRandomProcessingInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); + return this; + } + + /** + * Processing interval determines how often entities should be processed. + * Seconds provided will be multiplied by 1.5 + * The default processing interval is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. + */ + setProcessingIntervalSeconds(seconds: number): CatalogBuilder { + this.processingInterval = createRandomProcessingInterval({ minSeconds: seconds, maxSeconds: seconds * 1.5, }); @@ -204,9 +224,25 @@ export class CatalogBuilder { /** * Overwrites the default refresh interval function used to spread * entity updates in the catalog. + * + * @deprecated use {@link CatalogBuilder#setProcessingInterval} instead */ setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder { - this.refreshInterval = refreshInterval; + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshInterval is deprecated. Use CatalogBuilder.setProcessingInterval instead.', + ); + this.processingInterval = refreshInterval; + return this; + } + + /** + * Overwrites the default processing interval function used to spread + * entity updates in the catalog. + */ + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder { + this.processingInterval = processingInterval; return this; } @@ -396,7 +432,7 @@ export class CatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshInterval: this.refreshInterval, + refreshInterval: this.processingInterval, }); const integrations = ScmIntegrations.fromConfig(config); const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config); diff --git a/plugins/catalog-graph/api-report.md b/plugins/catalog-graph/api-report.md index 515a578d3c..82db1dac3b 100644 --- a/plugins/catalog-graph/api-report.md +++ b/plugins/catalog-graph/api-report.md @@ -6,8 +6,8 @@ /// import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { DependencyGraphTypes } from '@backstage/core-components'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { MouseEvent as MouseEvent_2 } from 'react'; @@ -129,7 +129,7 @@ export const EntityRelationsGraph: ({ className, zoom, }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number | undefined; unidirectional?: boolean | undefined; mergeRelations?: boolean | undefined; diff --git a/plugins/catalog-graph/dev/index.tsx b/plugins/catalog-graph/dev/index.tsx index 62d5eccef3..7d772702a1 100644 --- a/plugins/catalog-graph/dev/index.tsx +++ b/plugins/catalog-graph/dev/index.tsx @@ -16,7 +16,7 @@ import { GetEntitiesResponse } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, RELATION_API_CONSUMED_BY, RELATION_API_PROVIDED_BY, @@ -139,8 +139,12 @@ createDevApp() deps: {}, factory() { return { - async getEntityByName(name: EntityName): Promise { - return entities[stringifyEntityRef(name)]; + async getEntityByRef( + ref: string | CompoundEntityRef, + ): Promise { + return entities[ + typeof ref === 'string' ? ref : stringifyEntityRef(ref) + ]; }, async getEntities(): Promise { return { items: Object.values(entities) }; diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx index 7cebdee692..858fca1a83 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx @@ -57,7 +57,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByRef: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -88,7 +89,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders with custom title', async () => { diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx index b2dfd9f9b2..4a21418107 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx @@ -14,14 +14,14 @@ * limitations under the License. */ import { - getEntityName, + getCompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { InfoCard, InfoCardVariants } from '@backstage/core-components'; import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { - formatEntityRefTitle, + humanizeEntityRef, useEntity, entityRouteRef, } from '@backstage/plugin-catalog-react'; @@ -77,7 +77,7 @@ export const CatalogGraphCard = ({ zoom?: 'enabled' | 'disabled' | 'enable-on-click'; }) => { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const catalogEntityRoute = useRouteRef(entityRouteRef); const catalogGraphRoute = useRouteRef(catalogGraphRouteRef); const navigate = useNavigate(); @@ -94,7 +94,7 @@ export const CatalogGraphCard = ({ }); analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx index 2f686f8f62..0bf6469123 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx @@ -88,7 +88,10 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => (n.name === 'e' ? entityE : entityC)), + getEntityByRef: jest.fn(async (n: any) => + n === 'b:d/e' ? entityE : entityC, + ), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -128,7 +131,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findByText('b:d/e')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('should toggle filters', async () => { diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx index 2ae7a99dd8..15d0cd139b 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx @@ -24,7 +24,7 @@ import { import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Grid, makeStyles, Paper, Typography } from '@material-ui/core'; import FilterListIcon from '@material-ui/icons/FilterList'; @@ -149,14 +149,14 @@ export const CatalogGraphPage = ({ analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); } else { analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), ); setRootEntityNames([nodeEntityName]); } @@ -168,7 +168,7 @@ export const CatalogGraphPage = ({
formatEntityRefTitle(e)).join(', ')} + subtitle={rootEntityNames.map(e => humanizeEntityRef(e)).join(', ')} /> >; + rootEntityNames: CompoundEntityRef[]; + setRootEntityNames: Dispatch>; maxDepth: number; setMaxDepth: Dispatch>; selectedRelations: string[] | undefined; @@ -82,11 +82,12 @@ export function useCatalogGraphPage({ ); // Initial state - const [rootEntityNames, setRootEntityNames] = useState(() => - (Array.isArray(query.rootEntityRefs) - ? query.rootEntityRefs - : initialState?.rootEntityRefs ?? [] - ).map(r => parseEntityRef(r)), + const [rootEntityNames, setRootEntityNames] = useState( + () => + (Array.isArray(query.rootEntityRefs) + ? query.rootEntityRefs + : initialState?.rootEntityRefs ?? [] + ).map(r => parseEntityRef(r)), ); const [maxDepth, setMaxDepth] = useState(() => typeof query.maxDepth === 'string' diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx index 9cd752a5d7..ae541fa997 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { DependencyGraphTypes } from '@backstage/core-components'; -import { formatEntityRefTitle } from '@backstage/plugin-catalog-react'; +import { humanizeEntityRef } from '@backstage/plugin-catalog-react'; import { BackstageTheme } from '@backstage/theme'; import { makeStyles } from '@material-ui/core/styles'; import classNames from 'classnames'; @@ -95,7 +95,7 @@ export function CustomNode({ const displayTitle = title ?? (kind && name && namespace - ? formatEntityRefTitle({ kind, name, namespace }) + ? humanizeEntityRef({ kind, name, namespace }) : id); return ( diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx index 36dc816444..58ff81a91f 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, RELATION_HAS_PART, RELATION_OWNED_BY, RELATION_OWNER_OF, RELATION_PART_OF, - stringifyEntityRef, } from '@backstage/catalog-model'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; @@ -155,7 +155,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => entities[stringifyEntityRef(n)]), + getEntityByRef: jest.fn(async n => entities[n as string]), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -178,7 +179,7 @@ describe('', () => { }); test('renders a single node without exploding', async () => { - catalog.getEntityByName.mockResolvedValue({ + catalog.getEntityByRef.mockResolvedValue({ apiVersion: 'a', kind: 'b', metadata: { @@ -198,11 +199,11 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders a progress indicator while loading', async () => { - catalog.getEntityByName.mockImplementation(() => new Promise(() => {})); + catalog.getEntityByRef.mockImplementation(() => new Promise(() => {})); const { findByRole } = await renderInTestApp( @@ -213,12 +214,12 @@ describe('', () => { ); expect(await findByRole('progressbar')).toBeInTheDocument(); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('does not explode if an entity is missing', async () => { - catalog.getEntityByName.mockImplementation(async n => { - if (n.name === 'c') { + catalog.getEntityByRef.mockImplementation(async (n: any) => { + if (n === 'b:d/c') { return { apiVersion: 'a', kind: 'b', @@ -253,7 +254,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('renders at max depth of one', async () => { @@ -276,7 +277,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(3); + expect(catalog.getEntityByRef).toBeCalledTimes(3); }); test('renders simplied graph at full depth', async () => { @@ -301,7 +302,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth', async () => { @@ -328,7 +329,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(8); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth with merged relations', async () => { @@ -353,7 +354,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(4); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with multiple root nodes', async () => { @@ -379,7 +380,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with filtered kinds and relations', async () => { @@ -401,7 +402,7 @@ describe('', () => { expect(await findAllByText('ownerOf')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('handle clicks on a node', async () => { diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx index 76cd44394a..c5c83a54cd 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { DependencyGraph, DependencyGraphTypes, @@ -77,7 +80,7 @@ export const EntityRelationsGraph = ({ className, zoom = 'enabled', }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number; unidirectional?: boolean; mergeRelations?: boolean; diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts index a5dc82eab9..5349b64a72 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts @@ -29,6 +29,7 @@ describe('useEntityStore', () => { beforeEach(() => { catalogApi = { getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), @@ -64,7 +65,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity); + catalogApi.getEntityByRef.mockResolvedValue(entity); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -84,7 +85,7 @@ describe('useEntityStore', () => { test('handles request failures', async () => { const err = new Error('Hello World'); - catalogApi.getEntityByName.mockRejectedValue(err); + catalogApi.getEntityByRef.mockRejectedValue(err); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -101,7 +102,7 @@ describe('useEntityStore', () => { }); test('handles loading', async () => { - catalogApi.getEntityByName.mockReturnValue(new Promise(() => {})); + catalogApi.getEntityByRef.mockReturnValue(new Promise(() => {})); const { result } = renderHook(() => useEntityStore()); @@ -133,7 +134,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -150,7 +151,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities([ @@ -188,7 +189,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -205,7 +206,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities(['kind:namespace/name2']); @@ -233,6 +234,6 @@ describe('useEntityStore', () => { }); }); - expect(catalogApi.getEntityByName).toBeCalledTimes(2); + expect(catalogApi.getEntityByRef).toBeCalledTimes(2); }); }); diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts index b5ec3bef19..843b308281 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import limiterFactory from 'p-limit'; @@ -73,9 +73,7 @@ export function useEntityStore(): { return; } - const promise = catalogClient.getEntityByName( - parseEntityRef(entityRef), - ); + const promise = catalogClient.getEntityByRef(entityRef); outstandingEntities.set(entityRef, promise); diff --git a/plugins/catalog-import/api-report.md b/plugins/catalog-import/api-report.md index 164d831c96..5a9e8f03fd 100644 --- a/plugins/catalog-import/api-report.md +++ b/plugins/catalog-import/api-report.md @@ -8,11 +8,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi } from '@backstage/core-plugin-api'; import { Controller } from 'react-hook-form'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FieldErrors } from 'react-hook-form'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; @@ -33,7 +33,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -165,7 +165,7 @@ export interface EntityListComponentProps { // (undocumented) locations: Array<{ target: string; - entities: (Entity | EntityName)[]; + entities: (Entity | CompoundEntityRef)[]; }>; // (undocumented) onItemClick?: (target: string) => void; @@ -246,7 +246,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -258,7 +258,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-import/dev/index.tsx b/plugins/catalog-import/dev/index.tsx index a0cf034782..e2e5e1d22c 100644 --- a/plugins/catalog-import/dev/index.tsx +++ b/plugins/catalog-import/dev/index.tsx @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { createDevApp } from '@backstage/dev-utils'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { Grid, ListItem, ListItemIcon, ListItemText } from '@material-ui/core'; @@ -32,7 +32,7 @@ import { import { ImportPage } from '../src/components/ImportPage'; import { Content, Header, InfoCard, Page } from '@backstage/core-components'; -const getEntityNames = (url: string): EntityName[] => [ +const getEntityNames = (url: string): CompoundEntityRef[] => [ { kind: 'Component', namespace: url.replace(/^.*(folder-[^/]+).*|.*()$/, '$1') || 'default', diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 0923f4ac98..3cfd53470a 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { createApiRef } from '@backstage/core-plugin-api'; import { PartialEntity } from '../types'; @@ -38,7 +38,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index f689c0a883..969ff25c00 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -93,6 +93,7 @@ describe('CatalogImportClient', () => { getEntities: jest.fn(), addLocation: jest.fn(), removeLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index b856a4e359..c12e391ec9 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi, DiscoveryApi, @@ -212,7 +212,7 @@ the component will become available.\n\nFor more information, read an \ }): Promise< Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }> > { const { url, owner, repo, githubIntegrationConfig } = options; diff --git a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx index 2c59eec18f..47f0139e5e 100644 --- a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx +++ b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx @@ -14,11 +14,11 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useApp } from '@backstage/core-plugin-api'; import { EntityRefLink, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Collapse, @@ -41,9 +41,9 @@ const useStyles = makeStyles(theme => ({ }, })); -function sortEntities(entities: Array) { +function sortEntities(entities: Array) { return entities.sort((a, b) => - formatEntityRefTitle(a).localeCompare(formatEntityRefTitle(b)), + humanizeEntityRef(a).localeCompare(humanizeEntityRef(b)), ); } @@ -53,7 +53,10 @@ function sortEntities(entities: Array) { * @public */ export interface EntityListComponentProps { - locations: Array<{ target: string; entities: (Entity | EntityName)[] }>; + locations: Array<{ + target: string; + entities: (Entity | CompoundEntityRef)[]; + }>; locationListItemIcon: (target: string) => React.ReactElement; collapsed?: boolean; firstListItem?: React.ReactElement; @@ -130,7 +133,7 @@ export const EntityListComponent = (props: EntityListComponentProps) => { ) ?? WorkIcon; return ( { - + ); })} diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx index fc9b35136a..dd8087c51e 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx @@ -38,6 +38,7 @@ describe('', () => { const catalogApi: jest.Mocked = { getEntities: jest.fn(), addLocation: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx index 36e8c33a5c..1cbdfce355 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx @@ -19,7 +19,7 @@ import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { assertError } from '@backstage/errors'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Box, FormHelperText, Grid, Typography } from '@material-ui/core'; import { makeStyles } from '@material-ui/core/styles'; @@ -139,7 +139,7 @@ export const StepPrepareCreatePullRequest = ( }); return groupEntities.items - .map(e => formatEntityRefTitle(e, { defaultKind: 'group' })) + .map(e => humanizeEntityRef(e, { defaultKind: 'group' })) .sort(); }); diff --git a/plugins/catalog-import/src/components/useImportState.test.tsx b/plugins/catalog-import/src/components/useImportState.test.tsx index 1a01c61449..155adb8290 100644 --- a/plugins/catalog-import/src/components/useImportState.test.tsx +++ b/plugins/catalog-import/src/components/useImportState.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { cleanup } from '@testing-library/react'; import { act, renderHook } from '@testing-library/react-hooks'; import { AnalyzeResult } from '../api'; @@ -37,7 +37,7 @@ describe('useImportState', () => { locations: [ { target: 'https://0', - entities: [] as EntityName[], + entities: [] as CompoundEntityRef[], }, ], }; diff --git a/plugins/catalog-import/src/components/useImportState.ts b/plugins/catalog-import/src/components/useImportState.ts index 0cb6b59917..965f855f30 100644 --- a/plugins/catalog-import/src/components/useImportState.ts +++ b/plugins/catalog-import/src/components/useImportState.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useReducer } from 'react'; import { AnalyzeResult } from '../api'; @@ -43,7 +43,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -55,7 +55,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 3f9a2c47e1..ca5de26048 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -11,8 +11,8 @@ import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { CatalogApi } from '@backstage/catalog-client'; import { ComponentEntity } from '@backstage/catalog-model'; import { ComponentProps } from 'react'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { GetEntitiesResponse } from '@backstage/catalog-client'; import { IconButton } from '@material-ui/core'; import { LinkProps } from '@backstage/core-components'; @@ -250,7 +250,7 @@ export const EntityRefLink: (props: EntityRefLinkProps) => JSX.Element; // @public export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React_2.ReactNode; @@ -265,7 +265,7 @@ export const EntityRefLinks: ({ // @public export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; @@ -413,13 +413,8 @@ export const favoriteEntityTooltip: ( isStarred: boolean, ) => 'Remove from favorites' | 'Add to favorites'; -// @public (undocumented) -export function formatEntityRefTitle( - entityRef: Entity | EntityName, - opts?: { - defaultKind?: string; - }, -): string; +// @public @deprecated (undocumented) +export const formatEntityRefTitle: typeof humanizeEntityRef; // @public @deprecated (undocumented) export function getEntityMetadataEditUrl(entity: Entity): string | undefined; @@ -434,7 +429,7 @@ export function getEntityRelations( filter?: { kind: string; }, -): EntityName[]; +): CompoundEntityRef[]; // @public (undocumented) export function getEntitySourceLocation( @@ -442,6 +437,14 @@ export function getEntitySourceLocation( scmIntegrationsApi: ScmIntegrationRegistry, ): EntitySourceLocation | undefined; +// @public (undocumented) +export function humanizeEntityRef( + entityRef: Entity | CompoundEntityRef, + opts?: { + defaultKind?: string; + }, +): string; + // @public export function InspectEntityDialog(props: { open: boolean; @@ -449,8 +452,8 @@ export function InspectEntityDialog(props: { onClose: () => void; }): JSX.Element | null; -// @public -export function isOwnerOf(owner: Entity, owned: Entity): boolean; +// @alpha +export function isOwnerOf(owner: Entity, entity: Entity): boolean; // @public @deprecated export function loadCatalogOwnerRefs( @@ -624,12 +627,18 @@ export type UserListPickerProps = { // @public (undocumented) export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; }; // @public (undocumented) -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; }; diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index d025d08012..ec2c8126e5 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -31,7 +31,7 @@ import React, { useEffect, useMemo, useState } from 'react'; import { useEntityList } from '../../hooks/useEntityListProvider'; import { EntityOwnerFilter } from '../../filters'; import { getEntityRelations } from '../../utils'; -import { formatEntityRefTitle } from '../EntityRefLink'; +import { humanizeEntityRef } from '../EntityRefLink'; /** @public */ export type CatalogReactEntityOwnerPickerClassKey = 'input'; @@ -86,7 +86,7 @@ export const EntityOwnerPicker = () => { backendEntities .flatMap((e: Entity) => getEntityRelations(e, RELATION_OWNED_BY).map(o => - formatEntityRefTitle(o, { defaultKind: 'group' }), + humanizeEntityRef(o, { defaultKind: 'group' }), ), ) .filter(Boolean) as string[], diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx index 1e01402ddf..06a632232e 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx @@ -16,13 +16,13 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, parseEntityRef, } from '@backstage/catalog-model'; import React, { forwardRef } from 'react'; import { entityRouteRef } from '../../routes'; -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './humanize'; import { Link, LinkProps } from '@backstage/core-components'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Tooltip } from '@material-ui/core'; @@ -33,7 +33,7 @@ import { Tooltip } from '@material-ui/core'; * @public */ export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React.ReactNode; @@ -72,7 +72,7 @@ export const EntityRefLink = forwardRef( namespace = namespace?.toLocaleLowerCase('en-US') ?? DEFAULT_NAMESPACE; const routeParams = { kind, namespace, name }; - const formattedEntityRefTitle = formatEntityRefTitle( + const formattedEntityRefTitle = humanizeEntityRef( { kind, namespace, name }, { defaultKind }, ); diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx index 9be970aa56..d92cd2ec87 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import React from 'react'; import { EntityRefLink } from './EntityRefLink'; import { LinkProps } from '@backstage/core-components'; @@ -25,7 +25,7 @@ import { LinkProps } from '@backstage/core-components'; * @public */ export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts similarity index 86% rename from plugins/catalog-react/src/components/EntityRefLink/format.test.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts index b489f088fd..b541946ece 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './humanize'; describe('formatEntityRefTitle', () => { it('formats entity in default namespace', () => { @@ -30,7 +30,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:software'); }); @@ -48,7 +48,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:test/software'); }); @@ -66,7 +66,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity, { defaultKind: 'Component' }); + const title = humanizeEntityRef(entity, { defaultKind: 'Component' }); expect(title).toEqual('test/software'); }); @@ -76,7 +76,7 @@ describe('formatEntityRefTitle', () => { namespace: 'default', name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:software'); }); @@ -87,7 +87,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:test/software'); }); @@ -98,7 +98,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName, { + const title = humanizeEntityRef(entityName, { defaultKind: 'component', }); expect(title).toEqual('test/software'); diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts similarity index 85% rename from plugins/catalog-react/src/components/EntityRefLink/format.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.ts index 90100245a1..0ade1da0b9 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -16,13 +16,16 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; +/** @public @deprecated please use {@link humanizeEntityRef} instead */ +export const formatEntityRefTitle = humanizeEntityRef; + /** @public */ -export function formatEntityRefTitle( - entityRef: Entity | EntityName, +export function humanizeEntityRef( + entityRef: Entity | CompoundEntityRef, opts?: { defaultKind?: string }, ) { const defaultKind = opts?.defaultKind; diff --git a/plugins/catalog-react/src/components/EntityRefLink/index.ts b/plugins/catalog-react/src/components/EntityRefLink/index.ts index d49993feb6..50394547a0 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/index.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/index.ts @@ -18,4 +18,4 @@ export { EntityRefLink } from './EntityRefLink'; export type { EntityRefLinkProps } from './EntityRefLink'; export { EntityRefLinks } from './EntityRefLinks'; export type { EntityRefLinksProps } from './EntityRefLinks'; -export { formatEntityRefTitle } from './format'; +export { humanizeEntityRef, formatEntityRefTitle } from './humanize'; diff --git a/plugins/catalog-react/src/components/EntityTable/columns.tsx b/plugins/catalog-react/src/components/EntityTable/columns.tsx index d6c3582af6..08eb31bea3 100644 --- a/plugins/catalog-react/src/components/EntityTable/columns.tsx +++ b/plugins/catalog-react/src/components/EntityTable/columns.tsx @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, RELATION_PART_OF, } from '@backstage/catalog-model'; @@ -26,7 +26,7 @@ import { getEntityRelations } from '../../utils'; import { EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, } from '../EntityRefLink'; /** @public */ @@ -38,7 +38,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: T): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind, }) ); @@ -81,13 +81,13 @@ export const columnFactories = Object.freeze({ defaultKind?: string; filter?: { kind: string }; }): TableColumn { - function getRelations(entity: T): EntityName[] { + function getRelations(entity: T): CompoundEntityRef[] { return getEntityRelations(entity, relation, entityFilter); } function formatContent(entity: T): string { return getRelations(entity) - .map(r => formatEntityRefTitle(r, { defaultKind })) + .map(r => humanizeEntityRef(r, { defaultKind })) .join(', '); } diff --git a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx index bf6c8971b0..0423c0da44 100644 --- a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx +++ b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx @@ -33,7 +33,7 @@ import React, { useLayoutEffect, useRef, useState } from 'react'; import { useNavigate } from 'react-router'; import useAsync from 'react-use/lib/useAsync'; import { catalogApiRef } from '../../../api'; -import { formatEntityRefTitle } from '../../../components/EntityRefLink/format'; +import { humanizeEntityRef } from '../../EntityRefLink'; import { entityRouteRef } from '../../../routes'; import { EntityKindIcon } from './EntityKindIcon'; @@ -132,7 +132,7 @@ function CustomNode({ node }: DependencyGraphTypes.RenderNodeProps) { const displayTitle = node.metadata.title || (node.kind && node.metadata.name && node.metadata.namespace - ? formatEntityRefTitle({ + ? humanizeEntityRef({ kind: node.kind, name: node.metadata.name, namespace: node.metadata.namespace || '', diff --git a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts index 6420ebd5ae..c14b6523c0 100644 --- a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts @@ -16,8 +16,8 @@ import { Entity, - EntityName, - getEntityName, + CompoundEntityRef, + getCompoundEntityRef, ANNOTATION_ORIGIN_LOCATION, } from '@backstage/catalog-model'; import { catalogApiRef } from '../../api'; @@ -44,7 +44,7 @@ export type UseUnregisterEntityDialogState = | { type: 'unregister'; location: string; - colocatedEntities: EntityName[]; + colocatedEntities: CompoundEntityRef[]; unregisterLocation: () => Promise; deleteEntity: () => Promise; } @@ -141,7 +141,7 @@ export function useUnregisterEntityDialogState( return { type: 'unregister', location: locationRef!, - colocatedEntities: colocatedEntities.map(getEntityName), + colocatedEntities: colocatedEntities.map(getCompoundEntityRef), unregisterLocation, deleteEntity, }; diff --git a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx index 316a0f2c06..310deabbb7 100644 --- a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx +++ b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx @@ -55,7 +55,7 @@ const mockConfigApi = { } as Partial; const mockCatalogApi = { - getEntityByName: () => Promise.resolve(mockUser), + getEntityByRef: () => Promise.resolve(mockUser), } as Partial; const mockIdentityApi = { diff --git a/plugins/catalog-react/src/filters.ts b/plugins/catalog-react/src/filters.ts index 5d2c2e531b..9fb057a01c 100644 --- a/plugins/catalog-react/src/filters.ts +++ b/plugins/catalog-react/src/filters.ts @@ -15,7 +15,7 @@ */ import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; -import { formatEntityRefTitle } from './components/EntityRefLink'; +import { humanizeEntityRef } from './components/EntityRefLink'; import { EntityFilter, UserListFilterKind } from './types'; import { getEntityRelations } from './utils'; @@ -107,7 +107,7 @@ export class EntityOwnerFilter implements EntityFilter { filterEntity(entity: Entity): boolean { return this.values.some(v => getEntityRelations(entity, RELATION_OWNED_BY).some( - o => formatEntityRefTitle(o, { defaultKind: 'group' }) === v, + o => humanizeEntityRef(o, { defaultKind: 'group' }) === v, ), ); } diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index da3fb8f3a5..f80f858dec 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -115,7 +115,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index 27fde81cc7..e3ab09f081 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -76,7 +76,7 @@ const mockIdentityApi: Partial = { }; const mockCatalogApi: Partial = { getEntities: jest.fn().mockImplementation(async () => ({ items: entities })), - getEntityByName: async () => undefined, + getEntityByRef: async () => undefined, }; const wrapper = ({ diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx index 1bb9a649ad..4730f71a7d 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx @@ -30,13 +30,13 @@ import { loadCatalogOwnerRefs, useEntityOwnership } from './useEntityOwnership'; describe('useEntityOwnership', () => { type MockIdentityApi = jest.Mocked>; - type MockCatalogApi = jest.Mocked>; + type MockCatalogApi = jest.Mocked>; const mockIdentityApi: MockIdentityApi = { getBackstageIdentity: jest.fn(), }; const mockCatalogApi: MockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), }; const identityApi = mockIdentityApi as unknown as IdentityApi; @@ -102,11 +102,11 @@ describe('useEntityOwnership', () => { describe('loadCatalogOwnerRefs', () => { it('loads the first user from the catalog', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(user2Entity); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(user2Entity); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual(['group:default/group1']); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -114,11 +114,11 @@ describe('useEntityOwnership', () => { }); it('gracefully handles missing user', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(undefined); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual([]); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -133,7 +133,7 @@ describe('useEntityOwnership', () => { userEntityRef: 'user:default/user1', ownershipEntityRefs: ['user:default/user1', 'group:default/group1'], }); - mockCatalogApi.getEntityByName.mockResolvedValue(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValue(undefined); const { result, waitForValueToChange } = renderHook( () => useEntityOwnership(), diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.ts b/plugins/catalog-react/src/hooks/useEntityOwnership.ts index de3eca9dab..55b4e75ace 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.ts +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.ts @@ -48,7 +48,7 @@ export async function loadCatalogOwnerRefs( const primaryUserRef = identityOwnerRefs.find(ref => ref.startsWith('user:')); if (primaryUserRef) { - const entity = await catalogApi.getEntityByName( + const entity = await catalogApi.getEntityByRef( parseEntityRef(primaryUserRef), ); if (entity) { diff --git a/plugins/catalog-react/src/hooks/useOwnUser.ts b/plugins/catalog-react/src/hooks/useOwnUser.ts index 74c44bc638..211321afdf 100644 --- a/plugins/catalog-react/src/hooks/useOwnUser.ts +++ b/plugins/catalog-react/src/hooks/useOwnUser.ts @@ -34,7 +34,12 @@ export function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index 5ddabfdd2e..328a26e0f8 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; @@ -24,7 +24,9 @@ import { useCallback } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); @@ -33,8 +35,12 @@ function getEntityRef(entityOrRef: Entity | EntityName | string): string { /** @public */ export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; } { const starredEntitiesApi = useApi(starredEntitiesApiRef); @@ -44,13 +50,13 @@ export function useStarredEntities(): { ); const isStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntities.has(getEntityRef(entityOrRef)), [starredEntities], ); const toggleStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntitiesApi.toggleStarred(getEntityRef(entityOrRef)).then(), [starredEntitiesApi], ); diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx index 8ffc891092..317a0e391c 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { TestApiProvider } from '@backstage/test-utils'; import { renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; @@ -44,7 +44,7 @@ describe('useStarredEntity', () => { describe.each` title | entityOrRef ${'entity reference'} | ${'component:default/mock'} - ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as EntityName} + ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as CompoundEntityRef} ${'entity'} | ${{ apiVersion: '1', kind: 'Component', metadata: { name: 'mock' } } as Entity} `('with $title', ({ entityOrRef }) => { describe('toggleStarredEntity', () => { diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.ts b/plugins/catalog-react/src/hooks/useStarredEntity.ts index 2c647163aa..23d5677512 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntity.ts @@ -16,21 +16,25 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { useCallback, useEffect, useState } from 'react'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); } /** @public */ -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; } { diff --git a/plugins/catalog-react/src/utils/getEntityRelations.ts b/plugins/catalog-react/src/utils/getEntityRelations.ts index 45cbc6967f..13086db4ea 100644 --- a/plugins/catalog-react/src/utils/getEntityRelations.ts +++ b/plugins/catalog-react/src/utils/getEntityRelations.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { + Entity, + CompoundEntityRef, + parseEntityRef, +} from '@backstage/catalog-model'; // TODO(freben): This should be returning entity refs instead /** @@ -26,7 +30,7 @@ export function getEntityRelations( entity: Entity | undefined, relationType: string, filter?: { kind: string }, -): EntityName[] { +): CompoundEntityRef[] { let entityNames = entity?.relations ?.filter(r => r.type === relationType) diff --git a/plugins/catalog-react/src/utils/isOwnerOf.ts b/plugins/catalog-react/src/utils/isOwnerOf.ts index 3ef6b4b2a0..2b38ace247 100644 --- a/plugins/catalog-react/src/utils/isOwnerOf.ts +++ b/plugins/catalog-react/src/utils/isOwnerOf.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, stringifyEntityRef, @@ -24,18 +24,23 @@ import { import { getEntityRelations } from './getEntityRelations'; /** - * Get the related entity references. - * @public + * Returns true if the `owner` argument is a direct owner on the `entity` argument. + * + * @alpha + * @remarks + * + * Note that this ownership is not the same as using the claims in the auth-resolver, it only will take into account ownership as expressed by direct entity relations. + * It doesn't know anything about the additional groups that a user might belong to which the claims contain. */ -export function isOwnerOf(owner: Entity, owned: Entity) { +export function isOwnerOf(owner: Entity, entity: Entity) { const possibleOwners = new Set( [ ...getEntityRelations(owner, RELATION_MEMBER_OF, { kind: 'group' }), - ...(owner ? [getEntityName(owner)] : []), + ...(owner ? [getCompoundEntityRef(owner)] : []), ].map(stringifyEntityRef), ); - const owners = getEntityRelations(owned, RELATION_OWNED_BY).map( + const owners = getEntityRelations(entity, RELATION_OWNED_BY).map( stringifyEntityRef, ); diff --git a/plugins/catalog/api-report.md b/plugins/catalog/api-report.md index f314932b9a..97a36ad020 100644 --- a/plugins/catalog/api-report.md +++ b/plugins/catalog/api-report.md @@ -7,8 +7,8 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { IconComponent } from '@backstage/core-plugin-api'; import { IndexableDocument } from '@backstage/search-common'; @@ -157,9 +157,9 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } diff --git a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts index 605305882e..f4d06b0bb2 100644 --- a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts +++ b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts @@ -40,7 +40,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx index cf2eedb14c..07bd81b287 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx @@ -17,6 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, + parseEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, } from '@backstage/catalog-model'; @@ -104,11 +105,11 @@ describe('DefaultCatalogPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index 06a2395ab6..9d59d57647 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -20,7 +20,7 @@ import { RELATION_PART_OF, } from '@backstage/catalog-model'; import { - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, useEntityList, useStarredEntities, @@ -139,16 +139,16 @@ export const CatalogTable = (props: CatalogTableProps) => { return { entity, resolved: { - name: formatEntityRefTitle(entity, { + name: humanizeEntityRef(entity, { defaultKind: 'Component', }), ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), ownedByRelations, partOfSystemRelationTitle: partOfSystemRelations .map(r => - formatEntityRefTitle(r, { + humanizeEntityRef(r, { defaultKind: 'system', }), ) diff --git a/plugins/catalog/src/components/CatalogTable/columns.tsx b/plugins/catalog/src/components/CatalogTable/columns.tsx index d243be7c6f..0fb91d4f3f 100644 --- a/plugins/catalog/src/components/CatalogTable/columns.tsx +++ b/plugins/catalog/src/components/CatalogTable/columns.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; import { - formatEntityRefTitle, + humanizeEntityRef, EntityRefLink, EntityRefLinks, } from '@backstage/plugin-catalog-react'; @@ -34,7 +34,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: Entity): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind: options?.defaultKind, }) ); diff --git a/plugins/catalog/src/components/CatalogTable/types.ts b/plugins/catalog/src/components/CatalogTable/types.ts index 7eaff6faed..5bbb3af122 100644 --- a/plugins/catalog/src/components/CatalogTable/types.ts +++ b/plugins/catalog/src/components/CatalogTable/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** @public */ export interface CatalogTableRow { @@ -22,8 +22,8 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } diff --git a/plugins/code-coverage-backend/src/service/router.ts b/plugins/code-coverage-backend/src/service/router.ts index 4922f5c887..488103c835 100644 --- a/plugins/code-coverage-backend/src/service/router.ts +++ b/plugins/code-coverage-backend/src/service/router.ts @@ -18,7 +18,7 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import xmlparser from 'express-xml-bodyparser'; -import { CatalogClient } from '@backstage/catalog-client'; +import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; import { errorHandler, PluginDatabaseManager, @@ -33,10 +33,7 @@ import { aggregateCoverage, CoverageUtils } from './CoverageUtils'; import { Cobertura } from './converter/cobertura'; import { Jacoco } from './converter/jacoco'; import { Converter } from './converter'; -import { - getEntitySourceLocation, - parseEntityRef, -} from '@backstage/catalog-model'; +import { getEntitySourceLocation } from '@backstage/catalog-model'; export interface RouterOptions { config: Config; @@ -59,7 +56,7 @@ export const makeRouter = async ( await database.getClient(), ); const codecovUrl = await discovery.getExternalBaseUrl('code-coverage'); - const catalogApi = new CatalogClient({ discoveryApi: discovery }); + const catalogApi: CatalogApi = new CatalogClient({ discoveryApi: discovery }); const scm = ScmIntegrations.fromConfig(config); const router = Router(); @@ -77,8 +74,7 @@ export const makeRouter = async ( */ router.get('/report', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -100,8 +96,7 @@ export const makeRouter = async ( */ router.get('/history', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -119,8 +114,7 @@ export const makeRouter = async ( */ router.get('/file-content', async (req, res) => { const { entity, path } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -171,8 +165,7 @@ export const makeRouter = async ( */ router.post('/report', async (req, res) => { const { entity, coverageType } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } diff --git a/plugins/code-coverage-backend/src/service/types.ts b/plugins/code-coverage-backend/src/service/types.ts index c4c622bfaf..31edad88d1 100644 --- a/plugins/code-coverage-backend/src/service/types.ts +++ b/plugins/code-coverage-backend/src/service/types.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/code-coverage/src/api.ts b/plugins/code-coverage/src/api.ts index 61f920106b..081f1c7999 100644 --- a/plugins/code-coverage/src/api.ts +++ b/plugins/code-coverage/src/api.ts @@ -14,20 +14,25 @@ * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; import { JsonCodeCoverage, JsonCoverageHistory } from './types'; import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api'; export type CodeCoverageApi = { discovery: DiscoveryApi; - getCoverageForEntity: (entity: EntityName) => Promise; + getCoverageForEntity: ( + entity: CompoundEntityRef, + ) => Promise; getFileContentFromEntity: ( - entity: EntityName, + entity: CompoundEntityRef, filePath: string, ) => Promise; getCoverageHistoryForEntity: ( - entity: EntityName, + entity: CompoundEntityRef, limit?: number, ) => Promise; }; @@ -59,7 +64,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); return (await this.fetch( @@ -68,7 +73,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getFileContentFromEntity( - entityName: EntityName, + entityName: CompoundEntityRef, filePath: string, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); @@ -78,7 +83,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageHistoryForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, limit?: number, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); diff --git a/plugins/code-coverage/src/types.ts b/plugins/code-coverage/src/types.ts index c4c622bfaf..b03025b7ad 100644 --- a/plugins/code-coverage/src/types.ts +++ b/plugins/code-coverage/src/types.ts @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; + +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx index 53b225daaf..c490dd00da 100644 --- a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx +++ b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx @@ -28,6 +28,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx index 1310d9f31e..d103541333 100644 --- a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx +++ b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx index 36788ed1b6..23553b8b76 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx @@ -31,7 +31,7 @@ import { configApiRef, useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { BackstageTheme } from '@backstage/theme'; @@ -193,7 +193,7 @@ export function GroupsDiagram() { kind: catalogItem.kind, name: (catalogItem as GroupEntity).spec?.profile?.displayName || - formatEntityRefTitle(catalogItem, { defaultKind: 'Group' }), + humanizeEntityRef(catalogItem, { defaultKind: 'Group' }), }); // Edge to parent diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx index 255db061aa..99500336cf 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx index 2759afd585..f178b7de7f 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx @@ -29,6 +29,7 @@ describe('', () => { const catalogApi: jest.Mocked = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.tsx index 1b53546ccd..731852d4a7 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.tsx @@ -16,14 +16,14 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, } from '@backstage/catalog-model'; import { catalogApiRef, EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { Tooltip } from '@material-ui/core'; @@ -56,7 +56,7 @@ type FossaRow = { resolved: { name: string; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; loading: boolean; details?: FindingSummary; }; @@ -222,10 +222,10 @@ export const FossaPage = ({ return { entity, resolved: { - name: formatEntityRefTitle(entity), + name: humanizeEntityRef(entity), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), loading: summariesLoading, details: summary, diff --git a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx index 315c0d0d16..1b01820482 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx @@ -15,11 +15,7 @@ */ import { HomePageStarredEntities } from '../../plugin'; -import { - wrapInTestApp, - TestApiProvider, - MockStorageApi, -} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider } from '@backstage/test-utils'; import { starredEntitiesApiRef, MockStarredEntitiesApi, @@ -28,24 +24,18 @@ import { import { Grid } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4', - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Components/StarredEntities', decorators: [ (Story: ComponentType<{}>) => wrapInTestApp( - + , { diff --git a/plugins/home/src/templates/DefaultTemplate.stories.tsx b/plugins/home/src/templates/DefaultTemplate.stories.tsx index 5dc4ff61a0..c726294ba6 100644 --- a/plugins/home/src/templates/DefaultTemplate.stories.tsx +++ b/plugins/home/src/templates/DefaultTemplate.stories.tsx @@ -21,7 +21,7 @@ import { HomePageCompanyLogo, HomePageStarredEntities, } from '../plugin'; -import { wrapInTestApp, TestApiProvider, MockStorageApi} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider} from '@backstage/test-utils'; import { Content, Page, InfoCard } from '@backstage/core-components'; import { starredEntitiesApiRef, @@ -37,15 +37,11 @@ import { import { Grid, makeStyles } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4' - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Templates', @@ -57,7 +53,7 @@ export default { apis={[ [ starredEntitiesApiRef, - new MockStarredEntitiesApi(), + starredEntitiesApi, ], [searchApiRef, { query: () => Promise.resolve({ results: [] }) }], ]} diff --git a/plugins/jenkins-backend/README.md b/plugins/jenkins-backend/README.md index 1413a45136..443cb33c05 100644 --- a/plugins/jenkins-backend/README.md +++ b/plugins/jenkins-backend/README.md @@ -166,7 +166,7 @@ class AcmeJenkinsInfoProvider implements JenkinsInfoProvider { const PAAS_ANNOTATION = 'acme.example.com/paas-project-name'; // lookup pass-project-name from entity annotation - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/jenkins-backend/api-report.md b/plugins/jenkins-backend/api-report.md index d437a5dd75..cbc3ef4325 100644 --- a/plugins/jenkins-backend/api-report.md +++ b/plugins/jenkins-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; @@ -25,7 +25,7 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { }): DefaultJenkinsInfoProvider; // (undocumented) getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; // (undocumented) @@ -65,7 +65,7 @@ export interface JenkinsInfo { export interface JenkinsInfoProvider { // (undocumented) getInstance(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; } diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts index 62299926df..a3a5ba069d 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultJenkinsInfoProvider, @@ -160,10 +160,10 @@ describe('JenkinsConfig', () => { describe('DefaultJenkinsInfoProvider', () => { const mockCatalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any as jest.Mocked; - const entityRef: EntityName = { + const entityRef: CompoundEntityRef = { kind: 'Component', namespace: 'foo', name: 'bar', @@ -171,7 +171,7 @@ describe('DefaultJenkinsInfoProvider', () => { function configureProvider(configData: any, entityData: any) { const config = new ConfigReader(configData); - mockCatalog.getEntityByName.mockReturnValueOnce( + mockCatalog.getEntityByRef.mockReturnValueOnce( Promise.resolve(entityData as Entity), ); @@ -185,7 +185,7 @@ describe('DefaultJenkinsInfoProvider', () => { const provider = configureProvider({ jenkins: {} }, undefined); await expect(provider.getInstance({ entityRef })).rejects.toThrowError(); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); }); it('Reads simple config and annotation', async () => { @@ -207,7 +207,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toStrictEqual({ baseUrl: 'https://jenkins.example.com', crumbIssuer: undefined, @@ -243,7 +243,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -280,7 +280,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -317,7 +317,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', @@ -343,7 +343,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -369,7 +369,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -400,7 +400,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts index f822f475f1..a88dbf965e 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts @@ -17,7 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -27,7 +27,7 @@ export interface JenkinsInfoProvider { /** * The entity to get the info about. */ - entityRef: EntityName; + entityRef: CompoundEntityRef; /** * A specific job to get. This is only passed in when we know about a job name we are interested in. */ @@ -182,11 +182,11 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { } async getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise { // load entity - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/jenkins-backend/src/service/standaloneServer.ts b/plugins/jenkins-backend/src/service/standaloneServer.ts index 7f47d57ac3..f89f235667 100644 --- a/plugins/jenkins-backend/src/service/standaloneServer.ts +++ b/plugins/jenkins-backend/src/service/standaloneServer.ts @@ -18,7 +18,7 @@ import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; import { createRouter } from './router'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { JenkinsInfo } from './jenkinsInfoProvider'; export interface ServerOptions { @@ -35,7 +35,9 @@ export async function startStandaloneServer( const router = await createRouter({ logger, jenkinsInfoProvider: { - async getInstance(_: { entityRef: EntityName }): Promise { + async getInstance(_: { + entityRef: CompoundEntityRef; + }): Promise { return { baseUrl: 'https://example.com/', jobFullName: 'build-foo' }; }, }, diff --git a/plugins/jenkins/api-report.md b/plugins/jenkins/api-report.md index 5db5fd0dfe..fc9de04a7a 100644 --- a/plugins/jenkins/api-report.md +++ b/plugins/jenkins/api-report.md @@ -7,9 +7,9 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import type { EntityName } from '@backstage/catalog-model'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -48,20 +48,20 @@ export const JENKINS_ANNOTATION = 'jenkins.io/job-full-name'; export interface JenkinsApi { // Warning: (ae-forgotten-export) The symbol "Build" needs to be exported by the entry point index.d.ts getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // Warning: (ae-forgotten-export) The symbol "Project" needs to be exported by the entry point index.d.ts getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -82,20 +82,20 @@ export class JenkinsClient implements JenkinsApi { }); // (undocumented) getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // (undocumented) getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; diff --git a/plugins/jenkins/src/api/JenkinsApi.ts b/plugins/jenkins/src/api/JenkinsApi.ts index dd169de78b..04bd313227 100644 --- a/plugins/jenkins/src/api/JenkinsApi.ts +++ b/plugins/jenkins/src/api/JenkinsApi.ts @@ -19,7 +19,7 @@ import { DiscoveryApi, IdentityApi, } from '@backstage/core-plugin-api'; -import type { EntityName } from '@backstage/catalog-model'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; export const jenkinsApiRef = createApiRef({ @@ -80,7 +80,7 @@ export interface JenkinsApi { */ getProjects(options: { /** the entity whose jobs should be retrieved. */ - entity: EntityName; + entity: CompoundEntityRef; /** a filter on jobs. Currently this just takes a branch (and assumes certain structures in jenkins) */ filter: { branch?: string }; }): Promise; @@ -93,13 +93,13 @@ export interface JenkinsApi { * TODO: abstract jobFullName (so we could support differentiating between the same named job on multiple instances). */ getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -118,7 +118,7 @@ export class JenkinsClient implements JenkinsApi { } async getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string }; }): Promise { const { entity, filter } = options; @@ -157,7 +157,7 @@ export class JenkinsClient implements JenkinsApi { } async getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { @@ -182,7 +182,7 @@ export class JenkinsClient implements JenkinsApi { } async retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { diff --git a/plugins/jenkins/src/components/useBuildWithSteps.ts b/plugins/jenkins/src/components/useBuildWithSteps.ts index 2166b2a1a1..0910564384 100644 --- a/plugins/jenkins/src/components/useBuildWithSteps.ts +++ b/plugins/jenkins/src/components/useBuildWithSteps.ts @@ -19,7 +19,7 @@ import { jenkinsApiRef } from '../api'; import { useAsyncPolling } from './useAsyncPolling'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; const INTERVAL_AMOUNT = 1500; @@ -41,7 +41,7 @@ export function useBuildWithSteps({ const getBuildWithSteps = useCallback(async () => { try { - const entityName = await getEntityName(entity); + const entityName = await getCompoundEntityRef(entity); return api.getBuild({ entity: entityName, jobFullName, buildNumber }); } catch (e) { errorApi.post(e); diff --git a/plugins/jenkins/src/components/useBuilds.ts b/plugins/jenkins/src/components/useBuilds.ts index a4afeb351f..07d23cce5a 100644 --- a/plugins/jenkins/src/components/useBuilds.ts +++ b/plugins/jenkins/src/components/useBuilds.ts @@ -18,7 +18,7 @@ import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { jenkinsApiRef } from '../api'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; export enum ErrorType { CONNECTION_ERROR, @@ -33,7 +33,7 @@ export enum ErrorType { */ export function useBuilds({ branch }: { branch?: string } = {}) { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const api = useApi(jenkinsApiRef); const errorApi = useApi(errorApiRef); @@ -60,7 +60,7 @@ export function useBuilds({ branch }: { branch?: string } = {}) { } = useAsyncRetry(async () => { try { const build = await api.getProjects({ - entity: getEntityName(entity), + entity: getCompoundEntityRef(entity), filter: { branch }, }); diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 2b292cbe8b..cbf517bdca 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -25,7 +25,7 @@ import { import { useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, isOwnerOf, useEntity, } from '@backstage/plugin-catalog-react'; @@ -108,7 +108,7 @@ const getQueryParams = ( owner: Entity, selectedEntity: EntityTypeProps, ): string => { - const ownerName = formatEntityRefTitle(owner, { defaultKind: 'group' }); + const ownerName = humanizeEntityRef(owner, { defaultKind: 'group' }); const { kind, type } = selectedEntity; const filters = { kind, diff --git a/plugins/permission-node/api-report.md b/plugins/permission-node/api-report.md index 534ad45110..f6de7f4909 100644 --- a/plugins/permission-node/api-report.md +++ b/plugins/permission-node/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AllOfCriteria } from '@backstage/plugin-permission-common'; +import { AnyOfCriteria } from '@backstage/plugin-permission-common'; import { AuthorizeDecision } from '@backstage/plugin-permission-common'; import { AuthorizeQuery } from '@backstage/plugin-permission-common'; import { AuthorizeRequestOptions } from '@backstage/plugin-permission-common'; @@ -11,6 +13,7 @@ import { BackstageIdentityResponse } from '@backstage/plugin-auth-node'; import { Config } from '@backstage/config'; import express from 'express'; import { Identified } from '@backstage/plugin-permission-common'; +import { NotCriteria } from '@backstage/plugin-permission-common'; import { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import { PermissionCondition } from '@backstage/plugin-permission-common'; import { PermissionCriteria } from '@backstage/plugin-permission-common'; @@ -118,6 +121,21 @@ export type DefinitivePolicyDecision = { result: AuthorizeResult.ALLOW | AuthorizeResult.DENY; }; +// @alpha +export const isAndCriteria: ( + criteria: PermissionCriteria, +) => criteria is AllOfCriteria; + +// @alpha +export const isNotCriteria: ( + criteria: PermissionCriteria, +) => criteria is NotCriteria; + +// @alpha +export const isOrCriteria: ( + criteria: PermissionCriteria, +) => criteria is AnyOfCriteria; + // @public export const makeCreatePermissionRule: () => < TParams extends unknown[], diff --git a/plugins/permission-node/src/integration/index.ts b/plugins/permission-node/src/integration/index.ts index 978342e4ed..7702fea95b 100644 --- a/plugins/permission-node/src/integration/index.ts +++ b/plugins/permission-node/src/integration/index.ts @@ -19,3 +19,4 @@ export * from './createConditionExports'; export * from './createConditionTransformer'; export * from './createPermissionIntegrationRouter'; export * from './createPermissionRule'; +export { isAndCriteria, isOrCriteria, isNotCriteria } from './util'; diff --git a/plugins/permission-node/src/integration/util.ts b/plugins/permission-node/src/integration/util.ts index 3878e18895..9102f092c4 100644 --- a/plugins/permission-node/src/integration/util.ts +++ b/plugins/permission-node/src/integration/util.ts @@ -22,20 +22,44 @@ import { } from '@backstage/plugin-permission-common'; import { PermissionRule } from '../types'; -export const isAndCriteria = ( - filter: PermissionCriteria, -): filter is AllOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'allOf'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type allOf, + * narrowing down `criteria` to the specific type. + */ +export const isAndCriteria = ( + criteria: PermissionCriteria, +): criteria is AllOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'allOf'); -export const isOrCriteria = ( - filter: PermissionCriteria, -): filter is AnyOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'anyOf'); +/** + * Utility function used to parse a PermissionCriteria of type + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type anyOf, + * narrowing down `criteria` to the specific type. + */ +export const isOrCriteria = ( + criteria: PermissionCriteria, +): criteria is AnyOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'anyOf'); -export const isNotCriteria = ( - filter: PermissionCriteria, -): filter is NotCriteria => - Object.prototype.hasOwnProperty.call(filter, 'not'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type not, + * narrowing down `criteria` to the specific type. + */ +export const isNotCriteria = ( + criteria: PermissionCriteria, +): criteria is NotCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'not'); export const createGetRule = ( rules: PermissionRule[], diff --git a/plugins/rollbar/src/hooks/useCatalogEntity.ts b/plugins/rollbar/src/hooks/useCatalogEntity.ts index 468d0e8c50..21d22dd13c 100644 --- a/plugins/rollbar/src/hooks/useCatalogEntity.ts +++ b/plugins/rollbar/src/hooks/useCatalogEntity.ts @@ -30,7 +30,7 @@ export function useCatalogEntity() { error, loading, } = useAsync( - () => catalogApi.getEntityByName({ kind: 'Component', namespace, name }), + () => catalogApi.getEntityByRef({ kind: 'Component', namespace, name }), [catalogApi, namespace, name], ); diff --git a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts index 5fcd89841e..d095b77b4c 100644 --- a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts +++ b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, parseEntityRef, RELATION_OWNED_BY, RELATION_OWNER_OF, @@ -55,7 +55,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); if ( entity.apiVersion === 'scaffolder.backstage.io/v1beta3' && diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 445a494234..665c54fb3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -33,6 +33,7 @@ const createBitbucketCloudRepository = async (opts: { repoVisibility: 'private' | 'public'; mainBranch: string; authorization: string; + apiBaseUrl: string; }) => { const { workspace, @@ -42,6 +43,7 @@ const createBitbucketCloudRepository = async (opts: { repoVisibility, mainBranch, authorization, + apiBaseUrl, } = opts; const options: RequestInit = { @@ -61,7 +63,7 @@ const createBitbucketCloudRepository = async (opts: { let response: Response; try { response = await fetch( - `https://api.bitbucket.org/2.0/repositories/${workspace}/${repo}`, + `${apiBaseUrl}/repositories/${workspace}/${repo}`, options, ); } catch (e) { @@ -91,16 +93,14 @@ const createBitbucketCloudRepository = async (opts: { }; const createBitbucketServerRepository = async (opts: { - host: string; project: string; repo: string; description?: string; repoVisibility: 'private' | 'public'; authorization: string; - apiBaseUrl?: string; + apiBaseUrl: string; }) => { const { - host, project, repo, description, @@ -124,8 +124,7 @@ const createBitbucketServerRepository = async (opts: { }; try { - const baseUrl = apiBaseUrl ? apiBaseUrl : `https://${host}/rest/api/1.0`; - response = await fetch(`${baseUrl}/projects/${project}/repos`, options); + response = await fetch(`${apiBaseUrl}/projects/${project}/repos`, options); } catch (e) { throw new Error(`Unable to create repository, ${e}`); } @@ -314,7 +313,11 @@ export function createPublishBitbucketAction(options: { const authorization = getAuthorizationHeader( ctx.input.token - ? { host: integrationConfig.config.host, token: ctx.input.token } + ? { + host: integrationConfig.config.host, + apiBaseUrl: integrationConfig.config.apiBaseUrl, + token: ctx.input.token, + } : integrationConfig.config, ); @@ -327,7 +330,6 @@ export function createPublishBitbucketAction(options: { const { remoteUrl, repoContentsUrl } = await createMethod({ authorization, - host, workspace: workspace || '', project, repo, diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index e49c28cb74..a7148dac40 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -20,8 +20,9 @@ import { ANNOTATION_LOCATION, parseLocationRef, ANNOTATION_SOURCE_LOCATION, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, + stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, InputError, NotFoundError } from '@backstage/errors'; @@ -91,7 +92,7 @@ export function getEntityBaseUrl(entity: Entity): string | undefined { * Returns the matching template, or throws a NotFoundError if no such template existed. */ export async function findTemplate(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; token?: string; catalogApi: CatalogApi; }): Promise { @@ -106,9 +107,11 @@ export async function findTemplate(options: { throw new InputError(`Invalid kind, only 'Template' kind is supported`); } - const template = await catalogApi.getEntityByName(entityRef, { token }); + const template = await catalogApi.getEntityByRef(entityRef, { token }); if (!template) { - throw new NotFoundError(`Template ${entityRef} not found`); + throw new NotFoundError( + `Template ${stringifyEntityRef(entityRef)} not found`, + ); } return template as TemplateEntityV1beta3 | TemplateEntityV1beta2; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index fa1b649231..4041e41cf8 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -53,7 +53,7 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; const createCatalogClient = (template: any) => ({ - getEntityByName: async () => template, + getEntityByRef: async () => template, } as unknown as CatalogApi); function createDatabase(): PluginDatabaseManager { diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index a385e13534..b51f18fcc9 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -8,7 +8,6 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; -import { ComponentProps } from 'react'; import { ComponentType } from 'react'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; @@ -17,7 +16,6 @@ import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { FetchApi } from '@backstage/core-plugin-api'; import { FieldProps } from '@rjsf/core'; import { FieldValidation } from '@rjsf/core'; -import { IconButton } from '@material-ui/core'; import { JsonObject } from '@backstage/types'; import { JSONSchema7 } from 'json-schema'; import { JsonValue } from '@backstage/types'; @@ -87,12 +85,6 @@ export interface EntityTagsPickerUiOptions { kinds?: string[]; } -// Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "FavouriteTemplate" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const FavouriteTemplate: (props: Props) => JSX.Element; - // @public export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx deleted file mode 100644 index 6e8fe45b2f..0000000000 --- a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity } from '@backstage/catalog-model'; -import { useStarredEntity } from '@backstage/plugin-catalog-react'; -import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; -import Star from '@material-ui/icons/Star'; -import StarBorder from '@material-ui/icons/StarBorder'; -import React, { ComponentProps } from 'react'; - -type Props = ComponentProps & { entity: Entity }; - -const YellowStar = withStyles({ - root: { - color: '#f3ba37', - }, -})(Star); - -const WhiteBorderStar = withStyles({ - root: { - color: '#ffffff', - }, -})(StarBorder); - -const useStyles = makeStyles(theme => ({ - starButton: { - position: 'absolute', - top: theme.spacing(0.5), - right: theme.spacing(0.5), - padding: '0.25rem', - }, -})); - -export const favouriteTemplateTooltip = (isStarred: boolean) => - isStarred ? 'Remove from favorites' : 'Add to favorites'; - -export const favouriteTemplateIcon = (isStarred: boolean) => - isStarred ? : ; - -/** - * IconButton for showing if a current entity is starred and adding/removing it from the favourite entities - * @param props - MaterialUI IconButton props extended by required `entity` prop - */ -export const FavouriteTemplate = (props: Props) => { - const classes = useStyles(); - const { toggleStarredEntity, isStarredEntity } = useStarredEntity( - props.entity, - ); - return ( - toggleStarredEntity()} - > - - {favouriteTemplateIcon(isStarredEntity)} - - - ); -}; diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index f4710427c2..3a7691dd5a 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -21,6 +21,7 @@ import { } from '@backstage/integration-react'; import { EntityRefLinks, + FavoriteEntity, getEntityRelations, getEntitySourceLocation, } from '@backstage/plugin-catalog-react'; @@ -42,7 +43,6 @@ import { import WarningIcon from '@material-ui/icons/Warning'; import React from 'react'; import { selectedTemplateRouteRef } from '../../routes'; -import { FavouriteTemplate } from '../FavouriteTemplate/FavouriteTemplate'; import { Button, ItemCardHeader } from '@backstage/core-components'; import { useApi, useRouteRef } from '@backstage/core-plugin-api'; @@ -74,6 +74,12 @@ const useStyles = makeStyles(theme => ({ leftButton: { marginRight: 'auto', }, + starButton: { + position: 'absolute', + top: theme.spacing(0.5), + right: theme.spacing(0.5), + padding: '0.25rem', + }, })); const useDeprecationStyles = makeStyles(theme => ({ @@ -159,7 +165,7 @@ export const TemplateCard = ({ template, deprecated }: TemplateCardProps) => { return ( - + {deprecated && } - formatEntityRefTitle(e, { defaultKind }), + humanizeEntityRef(e, { defaultKind }), ); const onSelect = useCallback( diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx index 80f63efaf9..aded9acffa 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { - formatEntityRefTitle, + humanizeEntityRef, useOwnedEntities, } from '@backstage/plugin-catalog-react'; import { TextField } from '@material-ui/core'; @@ -50,7 +50,7 @@ export const OwnedEntityPicker = ( const { ownedEntities, loading } = useOwnedEntities(allowedKinds); const entityRefs = ownedEntities?.items - .map(e => formatEntityRefTitle(e, { defaultKind })) + .map(e => humanizeEntityRef(e, { defaultKind })) .filter(n => n); const onSelect = (_: any, value: string | null) => { diff --git a/plugins/scaffolder/src/components/index.ts b/plugins/scaffolder/src/components/index.ts index 030c485fd7..47540ff492 100644 --- a/plugins/scaffolder/src/components/index.ts +++ b/plugins/scaffolder/src/components/index.ts @@ -15,7 +15,6 @@ */ export * from './fields'; export type { RepoUrlPickerUiOptions } from './fields'; -export { FavouriteTemplate } from './FavouriteTemplate'; export { TemplateList } from './TemplateList'; export type { TemplateListProps } from './TemplateList'; export { TemplateTypePicker } from './TemplateTypePicker'; diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 6dc0256586..d4edcc6b25 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -5,6 +5,8 @@ ```ts /// +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; import { Config } from '@backstage/config'; import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/search-common'; @@ -114,7 +116,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexPrefix, }: ElasticSearchOptions): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; @@ -127,4 +129,31 @@ export class ElasticSearchSearchEngine implements SearchEngine { // (undocumented) protected translator(query: SearchQuery): ConcreteElasticSearchQuery; } + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: ElasticSearchSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + readonly indexName: string; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger_2; + elasticSearchClient: Client; +}; ``` diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 2a2dbd6f85..0a279d6b4c 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -25,6 +25,7 @@ "dependencies": { "@backstage/config": "^0.1.15", "@backstage/search-common": "^0.2.4", + "@backstage/plugin-search-backend-node": "^0.4.7", "@elastic/elasticsearch": "7.13.0", "@acuris/aws-es-connection": "^2.2.0", "aws-sdk": "^2.948.0", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 1566a35b57..467e1b4e77 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -15,7 +15,8 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { Client } from '@elastic/elasticsearch'; +import { ConfigReader } from '@backstage/config'; +import { Client, errors } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; import { ConcreteElasticSearchQuery, @@ -23,7 +24,7 @@ import { ElasticSearchSearchEngine, encodePageCursor, } from './ElasticSearchSearchEngine'; -import { ConfigReader } from '@backstage/config'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine { getTranslator() { @@ -37,6 +38,16 @@ const options = { Connection: mock.getConnection(), }; +const indexerMock = { + on: jest.fn(), + indexName: 'expected-index-name', +}; +jest.mock('./ElasticSearchSearchEngineIndexer', () => ({ + ElasticSearchSearchEngineIndexer: jest + .fn() + .mockImplementation(() => indexerMock), +})); + describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; @@ -542,23 +553,67 @@ describe('ElasticSearchSearchEngine', () => { }); }); - describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + describe('indexer', () => { + it('should get indexer', async () => { + const indexer = await testSearchEngine.getIndexer('test-index'); - // call index func and ensure the index func was invoked. - await testSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + expect(indexer).toStrictEqual(indexerMock); + expect(ElasticSearchSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + alias: 'test-index__search', + type: 'test-index', + indexPrefix: '', + indexSeparator: '-index__', + elasticSearchClient: client, + }), + ); + expect(indexerMock.on).toHaveBeenCalledWith( + 'error', + expect.any(Function), + ); + }); + + describe('onError', () => { + let errorHandler: Function; + const error = new Error('some error'); + + beforeEach(async () => { + mock.clearAll(); + await testSearchEngine.getIndexer('test-index'); + errorHandler = indexerMock.on.mock.calls[0][1]; + }); + + it('should check for and delete expected index', async () => { + const existsSpy = jest.fn().mockReturnValue('truthy value'); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check and delete HTTP requests were made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('should not delete index if none exists', async () => { + // Exists call returns 404 on no index. + const existsSpy = jest.fn().mockReturnValue( + new errors.ResponseError({ + statusCode: 404, + body: { status: 404 }, + } as unknown as any), + ); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check request was made, but no delete request was made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).not.toHaveBeenCalled(); + }); }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 48caf0d4f1..c16567fee0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -29,8 +29,8 @@ import { Client } from '@elastic/elasticsearch'; import esb from 'elastic-builder'; import { isEmpty, isNaN as nan, isNumber } from 'lodash'; import { Logger } from 'winston'; - import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; export type { ElasticSearchClientOptions }; @@ -58,12 +58,6 @@ type ElasticSearchResult = { _source: IndexableDocument; }; -function duration(startTimestamp: [number, number]): string { - const delta = process.hrtime(startTimestamp); - const seconds = delta[0] + delta[1] / 1e9; - return `${seconds.toFixed(1)}s`; -} - function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } @@ -165,67 +159,37 @@ export class ElasticSearchSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - this.logger.info( - `Started indexing ${documents.length} documents for index ${type}`, - ); - const startTimestamp = process.hrtime(); + async getIndexer(type: string) { const alias = this.constructSearchAlias(type); - const index = this.constructIndexName(type, `${Date.now()}`); - try { - const aliases = await this.elasticSearchClient.cat.aliases({ - format: 'json', - name: alias, - }); - const removableIndices = aliases.body.map( - (r: Record) => r.index, - ); + const indexer = new ElasticSearchSearchEngineIndexer({ + type, + indexPrefix: this.indexPrefix, + indexSeparator: this.indexSeparator, + alias, + elasticSearchClient: this.elasticSearchClient, + logger: this.logger, + }); - await this.elasticSearchClient.indices.create({ - index, - }); - const result = await this.elasticSearchClient.helpers.bulk({ - datasource: documents, - onDocument() { - return { - index: { _index: index }, - }; - }, - refreshOnCompletion: index, - }); - - this.logger.info( - `Indexing completed for index ${type} in ${duration(startTimestamp)}`, - result, - ); - await this.elasticSearchClient.indices.updateAliases({ - body: { - actions: [ - { remove: { index: this.constructIndexName(type, '*'), alias } }, - { add: { index, alias } }, - ], - }, - }); - - this.logger.info('Removing stale search indices', removableIndices); - if (removableIndices.length) { - await this.elasticSearchClient.indices.delete({ - index: removableIndices, - }); - } - } catch (e) { + // Attempt cleanup upon failure. + indexer.on('error', async e => { this.logger.error(`Failed to index documents for type ${type}`, e); - const response = await this.elasticSearchClient.indices.exists({ - index, - }); - const indexCreated = response.body; - if (indexCreated) { - this.logger.info(`Removing created index ${index}`); - await this.elasticSearchClient.indices.delete({ - index, + try { + const response = await this.elasticSearchClient.indices.exists({ + index: indexer.indexName, }); + const indexCreated = response.body; + if (indexCreated) { + this.logger.info(`Removing created index ${indexer.indexName}`); + await this.elasticSearchClient.indices.delete({ + index: indexer.indexName, + }); + } + } catch (error) { + this.logger.error(`Unable to clean up elastic index: ${error}`); } - } + }); + + return indexer; } async query(query: SearchQuery): Promise { @@ -268,10 +232,6 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly indexSeparator = '-index__'; - private constructIndexName(type: string, postFix: string) { - return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`; - } - private getTypeFromIndex(index: string) { return index .substring(this.indexPrefix.length) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..0867887d9a --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -0,0 +1,211 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; +import Mock from '@elastic/elasticsearch-mock'; +import { range } from 'lodash'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; + +const mock = new Mock(); +const client = new Client({ + node: 'http://localhost:9200', + Connection: mock.getConnection(), +}); + +describe('ElasticSearchSearchEngineIndexer', () => { + let indexer: ElasticSearchSearchEngineIndexer; + let bulkSpy: jest.Mock; + let catSpy: jest.Mock; + let createSpy: jest.Mock; + let aliasesSpy: jest.Mock; + let deleteSpy: jest.Mock; + + beforeEach(() => { + // Instantiate the indexer to be tested. + indexer = new ElasticSearchSearchEngineIndexer({ + type: 'some-type', + indexPrefix: '', + indexSeparator: '-index__', + alias: 'some-type-index__search', + logger: getVoidLogger(), + elasticSearchClient: client, + }); + + // Set up all requisite Elastic mocks. + mock.clearAll(); + bulkSpy = jest.fn().mockReturnValue({ took: 9, errors: false, items: [] }); + mock.add( + { + method: 'POST', + path: '/_bulk', + }, + bulkSpy, + ); + mock.add( + { + method: 'GET', + path: '/:index/_refresh', + }, + jest.fn().mockReturnValue({}), + ); + + catSpy = jest.fn().mockReturnValue([ + { + alias: 'some-type-index__search', + index: 'some-type-index__123tobedeleted', + filter: '-', + 'routing.index': '-', + 'routing.search': '-', + is_write_index: '-', + }, + ]); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + createSpy = jest.fn().mockReturnValue({ + acknowledged: true, + shards_acknowledged: true, + index: 'single_index', + }); + mock.add( + { + method: 'PUT', + path: '/:index', + }, + createSpy, + ); + + aliasesSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'POST', + path: '*', + }, + aliasesSpy, + ); + + deleteSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'DELETE', + path: '/some-type-index__123tobedeleted', + }, + deleteSpy, + ); + }); + + it('indexes documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + { + title: 'Another test', + text: 'Some more text', + location: 'test/location/2', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Older indices should have been queried for. + expect(catSpy).toHaveBeenCalled(); + + // A new index should have been created. + const createdIndex = createSpy.mock.calls[0][0].path.slice(1); + expect(createdIndex).toContain('some-type-index__'); + + // Bulk helper should have been called with documents. + const bulkBody = bulkSpy.mock.calls[0][0].body; + expect(bulkBody[0]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[1]).toStrictEqual(documents[0]); + expect(bulkBody[2]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[3]).toStrictEqual(documents[1]); + + // Alias should have been rotated. + expect(aliasesSpy).toHaveBeenCalled(); + const aliasActions = aliasesSpy.mock.calls[0][0].body.actions; + expect(aliasActions[0]).toStrictEqual({ + remove: { index: 'some-type-index__*', alias: 'some-type-index__search' }, + }); + expect(aliasActions[1]).toStrictEqual({ + add: { index: createdIndex, alias: 'some-type-index__search' }, + }); + + // Old index should be cleaned up. + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('handles bulk and batching during indexing', async () => { + const documents = range(550).map(i => ({ + title: `Hello World ${i}`, + location: `location-${i}`, + // Generate large document sizes to trigger ES bulk flushing. + text: range(2000).join(', '), + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Ensure multiple bulk requests were made. + expect(bulkSpy).toHaveBeenCalledTimes(2); + + // Ensure the first and last documents were included in the payloads. + const docLocations: string[] = [ + ...bulkSpy.mock.calls[0][0].body.map((l: any) => l.location), + ...bulkSpy.mock.calls[1][0].body.map((l: any) => l.location), + ]; + expect(docLocations).toContain('location-0'); + expect(docLocations).toContain('location-549'); + }); + + it('ignores cleanup when no existing indices exist', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + // Update initial alias cat to return nothing. + catSpy = jest.fn().mockReturnValue([]); + mock.clear({ + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Final deletion shouldn't be called. + expect(deleteSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts new file mode 100644 index 0000000000..2e4996cd2f --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Client } from '@elastic/elasticsearch'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger; + elasticSearchClient: Client; +}; + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + private received: number = 0; + private processed: number = 0; + private removableIndices: string[] = []; + + private readonly startTimestamp: [number, number]; + private readonly type: string; + public readonly indexName: string; + private readonly indexPrefix: string; + private readonly indexSeparator: string; + private readonly alias: string; + private readonly logger: Logger; + private readonly sourceStream: Readable; + private readonly elasticSearchClient: Client; + private bulkResult: Promise; + + constructor(options: ElasticSearchSearchEngineIndexerOptions) { + super({ batchSize: 100 }); + this.logger = options.logger; + this.startTimestamp = process.hrtime(); + this.type = options.type; + this.indexPrefix = options.indexPrefix; + this.indexSeparator = options.indexSeparator; + this.indexName = this.constructIndexName(`${Date.now()}`); + this.alias = options.alias; + this.elasticSearchClient = options.elasticSearchClient; + + // The ES client bulk helper supports stream-based indexing, but we have to + // supply the stream directly to it at instantiation-time. We can't supply + // this class itself, so instead, we create this inline stream instead. + this.sourceStream = new Readable({ objectMode: true }); + this.sourceStream._read = () => {}; + + // eslint-disable-next-line consistent-this + const that = this; + + // Keep a reference to the ES Bulk helper so that we can know when all + // documents have been successfully written to ES. + this.bulkResult = this.elasticSearchClient.helpers.bulk({ + datasource: this.sourceStream, + onDocument() { + that.processed++; + return { + index: { _index: that.indexName }, + }; + }, + refreshOnCompletion: that.indexName, + }); + } + + async initialize(): Promise { + this.logger.info(`Started indexing documents for index ${this.type}`); + + const aliases = await this.elasticSearchClient.cat.aliases({ + format: 'json', + name: this.alias, + }); + + this.removableIndices = aliases.body.map( + (r: Record) => r.index, + ); + + await this.elasticSearchClient.indices.create({ + index: this.indexName, + }); + } + + async index(documents: IndexableDocument[]): Promise { + await this.isReady(); + documents.forEach(document => { + this.received++; + this.sourceStream.push(document); + }); + } + + async finalize(): Promise { + // Wait for all documents to be processed. + await this.isReady(); + + // Close off the underlying stream connected to ES, indicating that no more + // documents will be written. + this.sourceStream.push(null); + + // Wait for the bulk helper to finish processing. + const result = await this.bulkResult; + + // Rotate aliases upon completion. Allow errors to bubble up so that we can + // clean up the create index. + this.logger.info( + `Indexing completed for index ${this.type} in ${duration( + this.startTimestamp, + )}`, + result, + ); + await this.elasticSearchClient.indices.updateAliases({ + body: { + actions: [ + { + remove: { index: this.constructIndexName('*'), alias: this.alias }, + }, + { add: { index: this.indexName, alias: this.alias } }, + ], + }, + }); + + // If any indices are removable, remove them. Do not bubble up this error, + // as doing so would delete the now aliased index. Log instead. + if (this.removableIndices.length) { + this.logger.info('Removing stale search indices', this.removableIndices); + try { + await this.elasticSearchClient.indices.delete({ + index: this.removableIndices, + }); + } catch (e) { + this.logger.warn(`Failed to remove stale search indices: ${e}`); + } + } + } + + /** + * Ensures that the number of documents sent over the wire to ES matches the + * number of documents this stream has received so far. This helps manage + * backpressure in other parts of the indexing pipeline. + */ + private isReady(): Promise { + return new Promise(resolve => { + const interval = setInterval(() => { + if (this.received === this.processed) { + clearInterval(interval); + resolve(); + } + }, 50); + }); + } + + private constructIndexName(postFix: string) { + return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`; + } +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index d5eee37803..19c24b37ff 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -19,3 +19,7 @@ export type { ConcreteElasticSearchQuery, ElasticSearchClientOptions, } from './ElasticSearchSearchEngine'; +export type { + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './ElasticSearchSearchEngineIndexer'; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index 8cf96de858..223141be6b 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -21,4 +21,8 @@ */ export { ElasticSearchSearchEngine } from './engines'; -export type { ElasticSearchClientOptions } from './engines'; +export type { + ElasticSearchClientOptions, + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './engines'; diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index e2edfc291a..ba8fde5f17 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/search-common'; import { Knex } from 'knex'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -28,6 +29,8 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) static create(knex: Knex): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -55,6 +58,8 @@ export interface DatabaseStore { // (undocumented) completeInsert(tx: Knex.Transaction, type: string): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -81,7 +86,7 @@ export class PgSearchEngine implements SearchEngine { database: PluginDatabaseManager; }): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) @@ -94,6 +99,28 @@ export class PgSearchEngine implements SearchEngine { translator(query: SearchQuery): ConcretePgSearchQuery; } +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: PgSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + // Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 56a3a6e643..8618bed4c8 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { range } from 'lodash'; import { DatabaseStore } from '../database'; import { ConcretePgSearchQuery, @@ -21,6 +20,13 @@ import { encodePageCursor, PgSearchEngine, } from './PgSearchEngine'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; + +jest.mock('./PgSearchEngineIndexer', () => ({ + PgSearchEngineIndexer: jest + .fn() + .mockImplementation(async () => 'the-expected-indexer'), +})); describe('PgSearchEngine', () => { const tx: any = {} as any; @@ -30,6 +36,7 @@ describe('PgSearchEngine', () => { beforeEach(() => { database = { transaction: jest.fn(), + getTransaction: jest.fn(), insertDocuments: jest.fn(), query: jest.fn(), completeInsert: jest.fn(), @@ -122,46 +129,21 @@ describe('PgSearchEngine', () => { }); }); - describe('insert', () => { - it('should insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); + describe('index', () => { + it('should instantiate indexer', async () => { + const indexer = await searchEngine.getIndexer('my-type'); - const documents = [ - { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, - { - location: 'location-2', - text: 'Hello World', - title: 'Dolor sit amet', - }, - ]; - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toHaveBeenCalledWith( - tx, - 'my-type', - documents, + // Indexer instantiated with expected args. + expect(PgSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }), ); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); - }); - it('should batch insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); - - const documents = range(350).map(i => ({ - title: `Hello World ${i}`, - text: 'Lorem Ipsum', - location: `location-${i}`, - })); - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toBeCalledTimes(4); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + // Indexer is as expected. + expect(indexer).toBe('the-expected-indexer'); }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 6fd6c571a8..ac42007401 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -15,12 +15,8 @@ */ import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-backend-node'; -import { - IndexableDocument, - SearchQuery, - SearchResultSet, -} from '@backstage/search-common'; -import { chunk } from 'lodash'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; import { DatabaseDocumentStore, DatabaseStore, @@ -77,16 +73,11 @@ export class PgSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - await this.databaseStore.transaction(async tx => { - await this.databaseStore.prepareInsert(tx); - - const batchSize = 100; - for (const documentBatch of chunk(documents, batchSize)) { - await this.databaseStore.insertDocuments(tx, type, documentBatch); - } - - await this.databaseStore.completeInsert(tx, type); + async getIndexer(type: string) { + return new PgSearchEngineIndexer({ + batchSize: 100, + type, + databaseStore: this.databaseStore, }); } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..1fc7e74fc4 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { range } from 'lodash'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +import { DatabaseStore } from '../database'; + +describe('PgSearchEngineIndexer', () => { + const tx = { + rollback: jest.fn(), + commit: jest.fn(), + } as any; + let database: jest.Mocked; + let indexer: PgSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + database = { + transaction: jest.fn().mockImplementation(fn => fn(tx)), + getTransaction: jest.fn().mockReturnValue(tx), + insertDocuments: jest.fn(), + query: jest.fn(), + completeInsert: jest.fn(), + prepareInsert: jest.fn(), + }; + indexer = new PgSearchEngineIndexer({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }); + }); + + it('should insert documents', async () => { + const documents = [ + { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, + { + location: 'location-2', + text: 'Hello World', + title: 'Dolor sit amet', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledWith( + tx, + 'my-type', + documents, + ); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + expect(tx.commit).toHaveBeenCalled(); + }); + + it('should batch insert documents', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toBeCalledTimes(4); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + }); + + it('should close out stream and bubble up error on prepare', async () => { + const expectedError = new Error('Prepare error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.prepareInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).not.toHaveBeenCalled(); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on insert', async () => { + const expectedError = new Error('Index error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.insertDocuments.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on completion', async () => { + const expectedError = new Error('Completion error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.completeInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledTimes(1); + expect(database.completeInsert).toHaveBeenCalledTimes(1); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); +}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts new file mode 100644 index 0000000000..53d040cb87 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Knex } from 'knex'; +import { DatabaseStore } from '../database'; + +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + private store: DatabaseStore; + private type: string; + private tx: Knex.Transaction | undefined; + + constructor(options: PgSearchEngineIndexerOptions) { + super({ batchSize: options.batchSize }); + this.store = options.databaseStore; + this.type = options.type; + } + + async initialize(): Promise { + this.tx = await this.store.getTransaction(); + try { + await this.store.prepareInsert(this.tx); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx.rollback(e); + throw e; + } + } + + async index(documents: IndexableDocument[]): Promise { + try { + await this.store.insertDocuments(this.tx!, this.type, documents); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx!.rollback(e); + throw e; + } + } + + async finalize(): Promise { + // Attempt to complete and commit the transaction. + try { + await this.store.completeInsert(this.tx!, this.type); + this.tx!.commit(); + } catch (e) { + // Otherwise, rollback the transaction and re-throw the error so that the + // stream can be closed and destroyed properly. + this.tx!.rollback!(e); + throw e; + } + } +} diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7994998baf..7f8e297648 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -15,3 +15,7 @@ */ export { PgSearchEngine } from './PgSearchEngine'; export type { ConcretePgSearchQuery } from './PgSearchEngine'; +export type { + PgSearchEngineIndexer, + PgSearchEngineIndexerOptions, +} from './PgSearchEngineIndexer'; diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 8c180ea019..0d3ee63ca4 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -71,6 +71,10 @@ export class DatabaseDocumentStore implements DatabaseStore { return await this.db.transaction(fn); } + async getTransaction(): Promise { + return this.db.transaction(); + } + async prepareInsert(tx: Knex.Transaction): Promise { // We create a temporary table to collect the hashes of the documents that // we expect to be in the documents table at the end. The table is deleted diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 0c0596160e..0a0dc35682 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -26,6 +26,7 @@ export interface PgSearchQuery { export interface DatabaseStore { transaction(fn: (tx: Knex.Transaction) => Promise): Promise; + getTransaction(): Promise; prepareInsert(tx: Knex.Transaction): Promise; insertDocuments( tx: Knex.Transaction, diff --git a/plugins/search-backend-node/api-report.md b/plugins/search-backend-node/api-report.md index 1929009760..37945c8902 100644 --- a/plugins/search-backend-node/api-report.md +++ b/plugins/search-backend-node/api-report.md @@ -3,30 +3,60 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { DocumentCollator } from '@backstage/search-common'; -import { DocumentDecorator } from '@backstage/search-common'; +/// + +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { DocumentDecoratorFactory } from '@backstage/search-common'; import { DocumentTypeInfo } from '@backstage/search-common'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; import { default as lunr_2 } from 'lunr'; import { QueryTranslator } from '@backstage/search-common'; +import { Readable } from 'stream'; import { SearchEngine } from '@backstage/search-common'; import { SearchQuery } from '@backstage/search-common'; import { SearchResultSet } from '@backstage/search-common'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "IndexBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta +export abstract class BatchSearchEngineIndexer extends Writable { + constructor(options: BatchSearchEngineOptions); + abstract finalize(): Promise; + abstract index(documents: IndexableDocument[]): Promise; + abstract initialize(): Promise; +} + +// @beta (undocumented) +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +// @beta (undocumented) +export type ConcreteLunrQuery = { + lunrQueryBuilder: lunr_2.Index.QueryBuilder; + documentTypes?: string[]; + pageSize: number; +}; + +// @beta +export abstract class DecoratorBase extends Transform { + constructor(); + abstract decorate( + document: IndexableDocument, + ): Promise; + abstract finalize(): Promise; + abstract initialize(): Promise; +} + +// @beta (undocumented) export class IndexBuilder { - // Warning: (ae-forgotten-export) The symbol "IndexBuilderOptions" needs to be exported by the entry point index.d.ts constructor({ logger, searchEngine }: IndexBuilderOptions); - // Warning: (ae-forgotten-export) The symbol "RegisterCollatorParameters" needs to be exported by the entry point index.d.ts addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void; - // Warning: (ae-forgotten-export) The symbol "RegisterDecoratorParameters" needs to be exported by the entry point index.d.ts - addDecorator({ decorator }: RegisterDecoratorParameters): void; + addDecorator({ factory }: RegisterDecoratorParameters): void; build(): Promise<{ scheduler: Scheduler; }>; @@ -36,32 +66,61 @@ export class IndexBuilder { getSearchEngine(): SearchEngine; } -// Warning: (ae-missing-release-tag) "LunrSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger_2; +}; + +// @beta (undocumented) +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; + +// @beta (undocumented) export class LunrSearchEngine implements SearchEngine { constructor({ logger }: { logger: Logger_2 }); // (undocumented) protected docStore: Record; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) protected logger: Logger_2; // (undocumented) protected lunrIndices: Record; // (undocumented) query(query: SearchQuery): Promise; - // Warning: (ae-forgotten-export) The symbol "LunrQueryTranslator" needs to be exported by the entry point index.d.ts - // // (undocumented) setTranslator(translator: LunrQueryTranslator): void; // (undocumented) protected translator: QueryTranslator; } -// Warning: (ae-missing-release-tag) "Scheduler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta (undocumented) +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(); + // (undocumented) + buildIndex(): lunr_2.Index; + // (undocumented) + finalize(): Promise; + // (undocumented) + getDocumentStore(): Record; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + +// @beta +export interface RegisterCollatorParameters { + defaultRefreshIntervalSeconds: number; + factory: DocumentCollatorFactory; +} + +// @beta +export interface RegisterDecoratorParameters { + factory: DocumentDecoratorFactory; +} + +// @beta (undocumented) export class Scheduler { constructor({ logger }: { logger: Logger_2 }); addToSchedule(task: Function, interval: number): void; @@ -70,4 +129,17 @@ export class Scheduler { } export { SearchEngine }; + +// @beta +export class TestPipeline { + execute(): Promise; + withDocuments(documents: IndexableDocument[]): TestPipeline; + static withSubject(subject: Readable | Transform | Writable): TestPipeline; +} + +// @beta +export type TestPipelineResult = { + error: unknown; + documents: IndexableDocument[]; +}; ``` diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index dd8ef5599a..2c644e357d 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -23,10 +23,12 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/errors": "^0.2.2", "@backstage/search-common": "^0.2.4", - "winston": "^3.2.1", + "@types/lunr": "^2.3.3", + "lodash": "^4.17.21", "lunr": "^2.3.9", - "@types/lunr": "^2.3.3" + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-common": "^0.11.0", diff --git a/plugins/search-backend-node/src/IndexBuilder.test.ts b/plugins/search-backend-node/src/IndexBuilder.test.ts index 0465f2701c..95a2e1d439 100644 --- a/plugins/search-backend-node/src/IndexBuilder.test.ts +++ b/plugins/search-backend-node/src/IndexBuilder.test.ts @@ -16,35 +16,37 @@ import { getVoidLogger } from '@backstage/backend-common'; import { - DocumentCollator, - DocumentDecorator, - IndexableDocument, + DocumentCollatorFactory, + DocumentDecoratorFactory, } from '@backstage/search-common'; +import { Readable, Transform } from 'stream'; import { IndexBuilder } from './IndexBuilder'; import { LunrSearchEngine, SearchEngine } from './index'; -class TestDocumentCollator implements DocumentCollator { +class TestDocumentCollatorFactory implements DocumentCollatorFactory { readonly type: string = 'anything'; - async execute(): Promise { - return []; + async getCollator(): Promise { + const collator = new Readable({ objectMode: true }); + collator._read = () => {}; + return collator; } } -class TypedDocumentCollator extends TestDocumentCollator { +class TypedDocumentCollatorFactory extends TestDocumentCollatorFactory { readonly type = 'an-expected-type'; } -class TestDocumentDecorator implements DocumentDecorator { - async execute(documents: IndexableDocument[]) { - return documents; +class TestDocumentDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator(): Promise { + return new Transform(); } } -class TypedDocumentDecorator extends TestDocumentDecorator { +class TypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['an-expected-type']; } -class DifferentlyTypedDocumentDecorator extends TestDocumentDecorator { +class DifferentlyTypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['not-the-expected-type']; } @@ -64,13 +66,13 @@ describe('IndexBuilder', () => { describe('addCollator', () => { it('adds a collator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const collatorSpy = jest.spyOn(testCollator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Build the index and ensure the collator was invoked. @@ -84,19 +86,19 @@ describe('IndexBuilder', () => { describe('addDecorator', () => { it('adds a decorator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const testDecorator = new TestDocumentDecorator(); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = new TestDocumentDecoratorFactory(); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -110,27 +112,20 @@ describe('IndexBuilder', () => { it('adds a type-specific decorator', async () => { jest.useFakeTimers(); - const testCollator = new TypedDocumentCollator(); - const testDecorator = new TypedDocumentDecorator(); - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TypedDocumentCollatorFactory(); + const testDecoratorFactory = new TypedDocumentDecoratorFactory(); + jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for the same type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -140,31 +135,24 @@ describe('IndexBuilder', () => { // wait for async decorator execution await Promise.resolve(); expect(decoratorSpy).toHaveBeenCalled(); - expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); it('adds a type-specific decorator that should not be called', async () => { - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - const testCollator = new TestDocumentCollator(); - const testDecorator = new DifferentlyTypedDocumentDecorator(); - const collatorSpy = jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = + new DifferentlyTypedDocumentDecoratorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for a different type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was not invoked. diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 92adb03d7d..e0a059e303 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -15,32 +15,31 @@ */ import { - DocumentCollator, - DocumentDecorator, + DocumentCollatorFactory, + DocumentDecoratorFactory, DocumentTypeInfo, - IndexableDocument, SearchEngine, } from '@backstage/search-common'; +import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './index'; import { + IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; interface CollatorEnvelope { - collate: DocumentCollator; + factory: DocumentCollatorFactory; refreshInterval: number; } -type IndexBuilderOptions = { - searchEngine: SearchEngine; - logger: Logger; -}; - +/** + * @beta + */ export class IndexBuilder { private collators: Record; - private decorators: Record; + private decorators: Record; private documentTypes: Record; private searchEngine: SearchEngine; private logger: Logger; @@ -66,18 +65,18 @@ export class IndexBuilder { * given refresh interval. */ addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void { this.logger.info( - `Added ${collator.constructor.name} collator for type ${collator.type}`, + `Added ${factory.constructor.name} collator factory for type ${factory.type}`, ); - this.collators[collator.type] = { + this.collators[factory.type] = { refreshInterval: defaultRefreshIntervalSeconds, - collate: collator, + factory, }; - this.documentTypes[collator.type] = { - visibilityPermission: collator.visibilityPermission, + this.documentTypes[factory.type] = { + visibilityPermission: factory.visibilityPermission, }; } @@ -86,18 +85,18 @@ export class IndexBuilder { * the decorator, it will be applied to documents from all known collators, * otherwise it will only be applied to documents of the given types. */ - addDecorator({ decorator }: RegisterDecoratorParameters): void { - const types = decorator.types || ['*']; + addDecorator({ factory }: RegisterDecoratorParameters): void { + const types = factory.types || ['*']; this.logger.info( - `Added decorator ${decorator.constructor.name} to types ${types.join( + `Added decorator ${factory.constructor.name} to types ${types.join( ', ', )}`, ); types.forEach(type => { if (this.decorators.hasOwnProperty(type)) { - this.decorators[type].push(decorator); + this.decorators[type].push(factory); } else { - this.decorators[type] = [decorator]; + this.decorators[type] = [factory]; } }); } @@ -111,46 +110,43 @@ export class IndexBuilder { Object.keys(this.collators).forEach(type => { scheduler.addToSchedule(async () => { - // Collate, Decorate, Index. - const decorators: DocumentDecorator[] = ( - this.decorators['*'] || [] - ).concat(this.decorators[type] || []); - - this.logger.debug( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, + // Instantiate the collator. + const collator = await this.collators[type].factory.getCollator(); + this.logger.info( + `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`, ); - let documents: IndexableDocument[]; - try { - documents = await this.collators[type].collate.execute(); - } catch (e) { - this.logger.error( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`, - ); - return; - } + // Instantiate all relevant decorators. + const decorators: Transform[] = await Promise.all( + (this.decorators['*'] || []) + .concat(this.decorators[type] || []) + .map(async factory => { + const decorator = await factory.getDecorator(); + this.logger.info( + `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`, + ); + return decorator; + }), + ); - for (let i = 0; i < decorators.length; i++) { - this.logger.debug( - `Decorating ${type} documents via ${decorators[i].constructor.name}`, - ); - try { - documents = await decorators[i].execute(documents); - } catch (e) { - this.logger.error( - `Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`, - ); - return; - } - } + // Instantiate the indexer. + const indexer = await this.searchEngine.getIndexer(type); - if (!documents || documents.length === 0) { - this.logger.debug(`No documents for type "${type}" to index`); - return; - } + // Compose collator/decorators/indexer into a pipeline + return new Promise(done => { + pipeline([collator, ...decorators, indexer], error => { + if (error) { + this.logger.error( + `Collating documents for ${type} failed: ${error}`, + ); + } else { + this.logger.info(`Collating documents for ${type} succeeded`); + } - // pushing documents to index to a configured search engine. - await this.searchEngine.index(type, documents); + // Signal index pipeline completion! + done(); + }); + }); }, this.collators[type].refreshInterval * 1000); }); diff --git a/plugins/search-backend-node/src/Scheduler.ts b/plugins/search-backend-node/src/Scheduler.ts index 3e356aa6aa..6debaa3dcd 100644 --- a/plugins/search-backend-node/src/Scheduler.ts +++ b/plugins/search-backend-node/src/Scheduler.ts @@ -26,6 +26,9 @@ type TaskEnvelope = { * TODO: coordination, error handling */ +/** + * @beta + */ export class Scheduler { private logger: Logger; private schedule: TaskEnvelope[]; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index cbe48ef5a7..88a3ba9a24 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -16,28 +16,61 @@ import { getVoidLogger } from '@backstage/backend-common'; import lunr from 'lunr'; -import { SearchEngine } from '@backstage/search-common'; +import { IndexableDocument, SearchEngine } from '@backstage/search-common'; import { ConcreteLunrQuery, LunrSearchEngine, decodePageCursor, encodePageCursor, } from './LunrSearchEngine'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; /** * Just used to test the default translator shipped with LunrSearchEngine. */ -class LunrSearchEngineForTranslatorTests extends LunrSearchEngine { +class LunrSearchEngineForTests extends LunrSearchEngine { + getDocStore() { + return this.docStore; + } + setDocStore(docStore: Record) { + this.docStore = docStore; + } + getLunrIndices() { + return this.lunrIndices; + } getTranslator() { return this.translator; } } +const indexerMock = { + on: jest.fn(), + buildIndex: jest.fn(), + getDocumentStore: jest.fn(), +}; +jest.mock('./LunrSearchEngineIndexer', () => ({ + LunrSearchEngineIndexer: jest.fn().mockImplementation(() => indexerMock), +})); + +const getActualIndexer = (engine: SearchEngine, index: string) => { + (LunrSearchEngineIndexer as unknown as jest.Mock).mockImplementationOnce( + () => { + const ActualIndexer = jest.requireActual( + './LunrSearchEngineIndexer', + ).LunrSearchEngineIndexer; + return new ActualIndexer(); + }, + ); + return engine.getIndexer(index); +}; + describe('LunrSearchEngine', () => { let testLunrSearchEngine: SearchEngine; beforeEach(() => { testLunrSearchEngine = new LunrSearchEngine({ logger: getVoidLogger() }); + jest.clearAllMocks(); }); describe('translator', () => { @@ -65,7 +98,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -107,7 +140,7 @@ describe('LunrSearchEngine', () => { }); it('should have default offset and limit', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -148,7 +181,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with 1 filter', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -193,7 +226,7 @@ describe('LunrSearchEngine', () => { }); it('should handle single-item array filter as scalar value', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -224,7 +257,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with multiple filters', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -273,7 +306,7 @@ describe('LunrSearchEngine', () => { }); it('should throw if translated query references missing field', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -334,7 +367,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -359,7 +398,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -392,7 +437,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -424,7 +475,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -456,7 +513,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -489,7 +552,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -522,7 +591,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -560,7 +635,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -604,8 +685,21 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 1 document each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer1 = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer1) + .withDocuments(mockDocuments) + .execute(); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); + // Perform search query scoped to "test-index-2" with a filter on the field "extraField" const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -642,7 +736,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -695,8 +795,20 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 2 documents each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); // Perform search query scoped to "test-index-2" const mockedSearchResult = await testLunrSearchEngine.query({ @@ -734,7 +846,13 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -767,7 +885,10 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index'); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -793,22 +914,46 @@ describe('LunrSearchEngine', () => { }); describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testLunrSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + it('should get indexer', async () => { + const indexer = await testLunrSearchEngine.getIndexer('test-index'); + expect(LunrSearchEngineIndexer).toHaveBeenCalled(); + expect(indexer.on).toHaveBeenCalledWith('close', expect.any(Function)); + }); - // call index func and ensure the index func was invoked. - await testLunrSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + it('should manage indices and docs on close', async () => { + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + + // Set up an inspectable search engine to pre-set some data. + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methds called by close handler. + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({ + 'new-location': doc, + }); + + // Get the indexer and invoke its close handler. + await inspectableSearchEngine.getIndexer('test-index'); + const onClose = indexerMock.on.mock.calls[0][1] as Function; + onClose(); + + // Ensure mocked methods were called. + expect(indexerMock.buildIndex).toHaveBeenCalled(); + expect(indexerMock.getDocumentStore).toHaveBeenCalled(); + + // Ensure the lunr index was written to the search engine. + expect(inspectableSearchEngine.getLunrIndices()).toStrictEqual({ + 'test-index': 'expected-index', + }); + + // Ensure documents are merged into the existing store. + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + 'new-location': doc, + }); }); }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index ea51ffe8e1..394cfffac8 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -23,7 +23,11 @@ import { } from '@backstage/search-common'; import lunr from 'lunr'; import { Logger } from 'winston'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +/** + * @beta + */ export type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; documentTypes?: string[]; @@ -35,8 +39,14 @@ type LunrResultEnvelope = { type: string; }; -type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ export class LunrSearchEngine implements SearchEngine { protected lunrIndices: Record = {}; protected docStore: Record; @@ -124,30 +134,17 @@ export class LunrSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - const lunrBuilder = new lunr.Builder(); + async getIndexer(type: string) { + const indexer = new LunrSearchEngineIndexer(); - lunrBuilder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); - lunrBuilder.searchPipeline.add(lunr.stemmer); - - // Make this lunr index aware of all relevant fields. - Object.keys(documents[0]).forEach(field => { - lunrBuilder.field(field); + indexer.on('close', () => { + // Once the stream is closed, build the index and store the documents in + // memory for later retrieval. + this.lunrIndices[type] = indexer.buildIndex(); + this.docStore = { ...this.docStore, ...indexer.getDocumentStore() }; }); - // Set "location" field as reference field - lunrBuilder.ref('location'); - - documents.forEach((document: IndexableDocument) => { - // Add document to Lunar index - lunrBuilder.add(document); - // Store documents in memory to be able to look up document using the ref during query time - // This is not how you should implement your SearchEngine implementation! Do not copy! - this.docStore[document.location] = document; - }); - - // "Rotate" the index by simply overwriting any existing index of the same name. - this.lunrIndices[type] = lunrBuilder.build(); + return indexer; } async query(query: SearchQuery): Promise { diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..fbb6b153d0 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import lunr from 'lunr'; +import { range } from 'lodash'; +import { TestPipeline } from '../test-utils'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; + +const lunrBuilderAddSpy = jest.fn(); +const lunrBuilderRefSpy = jest.fn(); +const lunrBuilderFieldSpy = jest.fn(); +const lunrBuilderPipelineAddSpy = jest.fn(); +const lunrBuilderSearchPipelineAddSpy = jest.fn(); + +jest.mock('lunr', () => { + const actualLunr = jest.requireActual('lunr'); + return { + ...actualLunr, + Builder: jest.fn().mockImplementation(() => { + const actualBuilder = new actualLunr.Builder(); + actualBuilder.add = lunrBuilderAddSpy; + actualBuilder.ref = lunrBuilderRefSpy; + actualBuilder.field = lunrBuilderFieldSpy; + actualBuilder.pipeline.add = lunrBuilderPipelineAddSpy; + actualBuilder.searchPipeline.add = lunrBuilderSearchPipelineAddSpy; + return actualBuilder; + }), + }; +}); + +describe('LunrSearchEngineIndexer', () => { + let indexer: LunrSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + indexer = new LunrSearchEngineIndexer(); + }); + + it('should index documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(lunrBuilderAddSpy).toHaveBeenCalledWith(documents[0]); + }); + + it('should index documents in bulk', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + expect(lunrBuilderAddSpy).toHaveBeenCalledTimes(350); + }); + + it('should initialize schema', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + extra: 'field', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Builder ref should be set to location (and only once). + expect(lunrBuilderRefSpy).toHaveBeenCalledTimes(1); + expect(lunrBuilderRefSpy).toHaveBeenLastCalledWith('location'); + + // Builder fields should be based on document fields. + expect(lunrBuilderFieldSpy).toHaveBeenCalledTimes(4); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('title'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('text'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('location'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('extra'); + }); + + it('should configure lunr pipeline', async () => { + expect(lunrBuilderSearchPipelineAddSpy).toHaveBeenLastCalledWith( + lunr.stemmer, + ); + expect(lunrBuilderPipelineAddSpy).toHaveBeenCalledWith( + ...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer], + ); + }); +}); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts new file mode 100644 index 0000000000..01c95f07d4 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import lunr from 'lunr'; +import { BatchSearchEngineIndexer } from '../indexing'; + +/** + * @beta + */ +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + private schemaInitialized = false; + private builder: lunr.Builder; + private docStore: Record = {}; + + constructor() { + super({ batchSize: 100 }); + + this.builder = new lunr.Builder(); + this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); + this.builder.searchPipeline.add(lunr.stemmer); + } + + // No async initialization required. + async initialize(): Promise {} + async finalize(): Promise {} + + async index(documents: IndexableDocument[]): Promise { + if (!this.schemaInitialized) { + // Make this lunr index aware of all relevant fields. + Object.keys(documents[0]).forEach(field => { + this.builder.field(field); + }); + + // Set "location" field as reference field + this.builder.ref('location'); + + this.schemaInitialized = true; + } + + documents.forEach(document => { + // Add document to Lunar index + this.builder.add(document); + + // Store documents in memory to be able to look up document using the ref during query time + // This is not how you should implement your SearchEngine implementation! Do not copy! + this.docStore[document.location] = document; + }); + } + + buildIndex() { + return this.builder.build(); + } + + getDocumentStore() { + return this.docStore; + } +} diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 7e6fb86bd4..0d710eadc2 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -15,4 +15,8 @@ */ export { LunrSearchEngine } from './LunrSearchEngine'; -export type { ConcreteLunrQuery } from './LunrSearchEngine'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, +} from './LunrSearchEngine'; +export type { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index d49ef195f2..2edab1b01d 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,6 +23,18 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, + LunrSearchEngineIndexer, +} from './engines'; +export type { + IndexBuilderOptions, + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; +export * from './indexing'; +export * from './test-utils'; /** * @deprecated Import from @backstage/search-common instead diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..b692ce3aff --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; + +const indexSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteBatchIndexer extends BatchSearchEngineIndexer { + async index(documents: IndexableDocument[]): Promise { + return indexSpy(documents); + } + async initialize(): Promise { + return initializeSpy(); + } + async finalize(): Promise { + return finalizeSpy(); + } +} + +describe('BatchSearchEngineIndexer', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + await TestPipeline.withSubject(indexer) + .withDocuments([document, document, document]) + .execute(); + expect(indexSpy).toHaveBeenCalledTimes(3); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + indexer.write(document); + }); + }); + + it('should call index according to batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert the batches. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(2); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document, document]); + expect(indexSpy).toHaveBeenNthCalledWith(2, [document]); + done(); + }); + + // Write batchSize + 1 documents and end the stream. + indexer.write(document); + indexer.write(document); + indexer.write(document); + indexer.end(); + }); + }); + + it('should call index without exceeding batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert that it still wrote. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(1); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document]); + done(); + }); + + // Write batchSize - 1 documents and end the stream. + indexer.write(document); + indexer.end(); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('index error'); + indexSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts new file mode 100644 index 0000000000..b368dd8184 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Writable } from 'stream'; + +/** + * @beta + */ +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +/** + * Base class encapsulating batch-based stream processing. Useful as a base + * class for search engine indexers. + * @beta + */ +export abstract class BatchSearchEngineIndexer extends Writable { + private batchSize: number; + private currentBatch: IndexableDocument[] = []; + private initialized: Promise; + + constructor(options: BatchSearchEngineOptions) { + super({ objectMode: true }); + this.batchSize = options.batchSize; + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Receives an array of indexable documents (of size this.batchSize) which + * should be written to the search engine. This method won't be called again + * at least until it resolves. + */ + public abstract index(documents: IndexableDocument[]): Promise; + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates batch stream write logic. + * @internal + */ + async _write( + doc: IndexableDocument, + _e: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + this.currentBatch.push(doc); + if (this.currentBatch.length < this.batchSize) { + done(); + return; + } + + try { + await this.index(this.currentBatch); + this.currentBatch = []; + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + // Index any remaining documents. + if (this.currentBatch.length) { + await this.index(this.currentBatch); + this.currentBatch = []; + } + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts new file mode 100644 index 0000000000..3b045dcddf --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { DecoratorBase } from './DecoratorBase'; +import { TestPipeline } from '../test-utils'; + +const decorateSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteDecorator extends DecoratorBase { + public initialize(): Promise { + return initializeSpy(); + } + public decorate( + document: IndexableDocument, + ): Promise { + return decorateSpy(document); + } + public finalize(): Promise { + return finalizeSpy(); + } +} + +describe('DecoratorBase', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + decorateSpy.mockImplementation(doc => ({ + ...doc, + transformed: true, + })); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(documents.length).toBe(3); + expect((documents[0] as unknown as any).transformed).toBe(true); + expect((documents[1] as unknown as any).transformed).toBe(true); + expect((documents[2] as unknown as any).transformed).toBe(true); + }); + + it('should allow filtering', async () => { + decorateSpy.mockResolvedValue(undefined); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(0); + }); + + it('should allow fanning', async () => { + decorateSpy.mockImplementation(doc => { + return [doc, doc]; + }); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(6); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteDecorator(); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + decorator.write(document); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('decorate error'); + decorateSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.ts new file mode 100644 index 0000000000..382ce443e3 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Transform } from 'stream'; + +/** + * Base class encapsulating simple async transformations. Useful as a base + * class for Backstage search decorators. + * @beta + */ +export abstract class DecoratorBase extends Transform { + private initialized: Promise; + + constructor() { + super({ objectMode: true }); + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Receives a single indexable document. In your decorate method, you can: + * + * - Resolve `undefined` to indicate the record should be omitted. + * - Resolve a single modified document, which could contain new fields, + * edited fields, or removed fields. + * - Resolve an array of indexable documents, if the purpose if the decorator + * is to convert one document into multiple derivative documents. + */ + public abstract decorate( + document: IndexableDocument, + ): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates simple transform stream logic. + * @internal + */ + async _transform( + document: IndexableDocument, + _: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + try { + const decorated = await this.decorate(document); + + // If undefined was returned, omit the record and move on. + if (decorated === undefined) { + done(); + return; + } + + // If an array of documents was given, push them all. + if (Array.isArray(decorated)) { + decorated.forEach(doc => { + this.push(doc); + }); + done(); + return; + } + + // Otherwise, just push the decorated document. + this.push(decorated); + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/index.ts b/plugins/search-backend-node/src/indexing/index.ts new file mode 100644 index 0000000000..e86235df16 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +export { DecoratorBase } from './DecoratorBase'; +export type { BatchSearchEngineOptions } from './BatchSearchEngineIndexer'; diff --git a/plugins/search-backend-node/src/test-utils/TestPipeline.ts b/plugins/search-backend-node/src/test-utils/TestPipeline.ts new file mode 100644 index 0000000000..c0c2bf5f89 --- /dev/null +++ b/plugins/search-backend-node/src/test-utils/TestPipeline.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { pipeline, Readable, Transform, Writable } from 'stream'; + +/** + * Object resolved after a test pipeline is executed. + * @beta + */ +export type TestPipelineResult = { + /** + * If an error was emitted by the pipeline, it will be set here. + */ + error: unknown; + + /** + * A list of documents collected at the end of the pipeline. If the subject + * under test is an indexer, this will be an empty array (because your + * indexer should have received the documents instead). + */ + documents: IndexableDocument[]; +}; + +/** + * Test utility for Backstage Search collators, decorators, and indexers. + * @beta + */ +export class TestPipeline { + private collator?: Readable; + private decorator?: Transform; + private indexer?: Writable; + + private constructor({ + collator, + decorator, + indexer, + }: { + collator?: Readable; + decorator?: Transform; + indexer?: Writable; + }) { + this.collator = collator; + this.decorator = decorator; + this.indexer = indexer; + } + + /** + * Provide the collator, decorator, or indexer to be tested. + */ + static withSubject(subject: Readable | Transform | Writable) { + if (subject instanceof Transform) { + return new TestPipeline({ decorator: subject }); + } + + if (subject instanceof Readable) { + return new TestPipeline({ collator: subject }); + } + + if (subject instanceof Writable) { + return new TestPipeline({ indexer: subject }); + } + + throw new Error( + 'Unknown test subject: are you passing a readable, writable, or transform stream?', + ); + } + + /** + * Provide documents for testing decorators and indexers. + */ + withDocuments(documents: IndexableDocument[]): TestPipeline { + if (this.collator) { + throw new Error('Cannot provide documents when testing a collator.'); + } + + // Set a naive readable stream that just pushes all given documents. + this.collator = new Readable({ objectMode: true }); + this.collator._read = () => {}; + process.nextTick(() => { + documents.forEach(document => { + this.collator!.push(document); + }); + this.collator!.push(null); + }); + + return this; + } + + /** + * Execute the test pipeline so that you can make assertions about the result + * or behavior of the given test subject. + */ + async execute(): Promise { + const documents: IndexableDocument[] = []; + if (!this.collator) { + throw new Error( + 'Cannot execute pipeline without a collator or documents', + ); + } + + // If we are here and there is no indexer, we are testing a collator or a + // decorator. Set up a naive writable that captures documents in memory. + if (!this.indexer) { + this.indexer = new Writable({ objectMode: true }); + this.indexer._write = (document: IndexableDocument, _, done) => { + documents.push(document); + done(); + }; + } + + return new Promise(done => { + const pipes: (Readable | Transform | Writable)[] = [this.collator!]; + if (this.decorator) { + pipes.push(this.decorator); + } + pipes.push(this.indexer!); + + pipeline(pipes, error => { + done({ + error, + documents, + }); + }); + }); + } +} diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts b/plugins/search-backend-node/src/test-utils/index.ts similarity index 79% rename from plugins/scaffolder/src/components/FavouriteTemplate/index.ts rename to plugins/search-backend-node/src/test-utils/index.ts index 5955e88c01..185d25433b 100644 --- a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts +++ b/plugins/search-backend-node/src/test-utils/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,5 @@ * limitations under the License. */ -export * from './FavouriteTemplate'; +export { TestPipeline } from './TestPipeline'; +export type { TestPipelineResult } from './TestPipeline'; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index df83357f6c..68ccfe4c7e 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -14,10 +14,24 @@ * limitations under the License. */ -import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; +import { + DocumentCollatorFactory, + DocumentDecoratorFactory, + SearchEngine, +} from '@backstage/search-common'; +import { Logger } from 'winston'; + +/** + * @beta + */ +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger; +}; /** * Parameters required to register a collator. + * @beta */ export interface RegisterCollatorParameters { /** @@ -26,17 +40,18 @@ export interface RegisterCollatorParameters { defaultRefreshIntervalSeconds: number; /** - * The collator class responsible for returning all documents of the given type. + * The class responsible for returning the document collator of the given type. */ - collator: DocumentCollator; + factory: DocumentCollatorFactory; } /** * Parameters required to register a decorator + * @beta */ export interface RegisterDecoratorParameters { /** - * The decorator class responsible for appending or modifying documents of the given type(s). + * The class responsible for returning the decorator which appends, modifies, or filters documents. */ - decorator: DocumentDecorator; + factory: DocumentDecoratorFactory; } diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts index 0a7b45e164..137ffc24f1 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts @@ -62,7 +62,7 @@ describe('AuthorizedSearchEngine', () => { setTranslator: () => { throw new Error('Function not implemented. 1'); }, - index: () => { + getIndexer: () => { throw new Error('Function not implemented.2'); }, query: mockedQuery, diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts index d0488af00d..82ed061196 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts @@ -25,7 +25,6 @@ import { } from '@backstage/plugin-permission-common'; import { DocumentTypeInfo, - IndexableDocument, QueryRequestOptions, QueryTranslator, SearchEngine, @@ -35,6 +34,7 @@ import { } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; +import { Writable } from 'stream'; export function decodePageCursor(pageCursor?: string): { page: number } { if (!pageCursor) { @@ -78,8 +78,8 @@ export class AuthorizedSearchEngine implements SearchEngine { this.searchEngine.setTranslator(translator); } - async index(type: string, documents: IndexableDocument[]): Promise { - this.searchEngine.index(type, documents); + async getIndexer(type: string): Promise { + return this.searchEngine.getIndexer(type); } async query( diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts index 0a94250aeb..bdf46a240b 100644 --- a/plugins/search-backend/src/service/router.test.ts +++ b/plugins/search-backend/src/service/router.test.ts @@ -105,7 +105,7 @@ describe('createRouter', () => { beforeAll(async () => { const logger = getVoidLogger(); mockSearchEngine = { - index: jest.fn(), + getIndexer: jest.fn(), setTranslator: jest.fn(), query: jest.fn(), }; diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index b13802358f..b86677ab90 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -27,7 +27,7 @@ import { Logger } from 'winston'; import { DateTime } from 'luxon'; import { PersistenceContext } from './persistence/persistenceContext'; import { - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -99,8 +99,10 @@ export async function createRouter< }); router.post('/checks/run', async (req, res) => { - const { checks, entities }: { checks: string[]; entities: EntityName[] } = - req.body; + const { + checks, + entities, + }: { checks: string[]; entities: CompoundEntityRef[] } = req.body; const tasks = entities.map(async entity => { const entityTriplet = typeof entity === 'string' ? entity : stringifyEntityRef(entity); diff --git a/plugins/tech-insights/api-report.md b/plugins/tech-insights/api-report.md index 79854e505f..deadd06010 100644 --- a/plugins/tech-insights/api-report.md +++ b/plugins/tech-insights/api-report.md @@ -9,7 +9,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { BulkCheckResponse } from '@backstage/plugin-tech-insights-common'; import { CheckResult } from '@backstage/plugin-tech-insights-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { default as React_2 } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -52,11 +52,14 @@ export interface TechInsightsApi { ) => CheckResultRenderer | undefined; // (undocumented) runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; // (undocumented) - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; } // @public diff --git a/plugins/tech-insights/src/api/TechInsightsApi.ts b/plugins/tech-insights/src/api/TechInsightsApi.ts index b1287a62dc..0a6bb937a6 100644 --- a/plugins/tech-insights/src/api/TechInsightsApi.ts +++ b/plugins/tech-insights/src/api/TechInsightsApi.ts @@ -21,7 +21,7 @@ import { } from '@backstage/plugin-tech-insights-common'; import { Check } from './types'; import { CheckResultRenderer } from '../components/CheckResultRenderer'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * {@link @backstage/core-plugin-api#ApiRef} for the {@link TechInsightsApi} @@ -45,9 +45,12 @@ export interface TechInsightsApi { description?: string, ) => CheckResultRenderer | undefined; getAllChecks(): Promise; - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; } diff --git a/plugins/tech-insights/src/api/TechInsightsClient.ts b/plugins/tech-insights/src/api/TechInsightsClient.ts index 5e29749854..5b9af9557f 100644 --- a/plugins/tech-insights/src/api/TechInsightsClient.ts +++ b/plugins/tech-insights/src/api/TechInsightsClient.ts @@ -22,7 +22,7 @@ import { import { Check } from './types'; import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { ResponseError } from '@backstage/errors'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { CheckResultRenderer, @@ -74,7 +74,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runChecks( - entityParams: EntityName, + entityParams: CompoundEntityRef, checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); @@ -102,7 +102,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 5d106cfafb..155f096614 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -3,9 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; import { Knex } from 'knex'; @@ -15,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/techdocs-common'; import { PublisherBase } from '@backstage/techdocs-common'; +import { Readable } from 'stream'; import { TechDocsDocument } from '@backstage/techdocs-common'; import { TokenManager } from '@backstage/backend-common'; // @public export function createRouter(options: RouterOptions): Promise; -// @public -export class DefaultTechDocsCollator implements DocumentCollator { +// @public @deprecated +export class DefaultTechDocsCollator { // (undocumented) protected applyArgsToFormat( format: string, @@ -41,6 +45,27 @@ export class DefaultTechDocsCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + config: Config, + options: TechDocsCollatorFactoryOptions, + ): DefaultTechDocsCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public +export interface DocsBuildStrategy { + // (undocumented) + shouldBuild(params: ShouldBuildParameters): Promise; +} + // @public export type OutOfTheBoxDeploymentOptions = { preparers: PreparerBuilder; @@ -51,6 +76,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -60,6 +86,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -67,6 +94,22 @@ export type RouterOptions = | RecommendedDeploymentOptions | OutOfTheBoxDeploymentOptions; +// @public +export type ShouldBuildParameters = { + entity: Entity; +}; + +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger_2; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + // @public export type TechDocsCollatorOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 7e8241bfea..20100f4848 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@backstage/cli": "^0.14.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/dockerode": "^3.3.0", "msw": "^0.35.0", diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 12e7df1e9e..570da95092 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -25,10 +25,18 @@ export type { RouterOptions, RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, + DocsBuildStrategy, + ShouldBuildParameters, } from './service'; -export { DefaultTechDocsCollator } from './search'; -export type { TechDocsCollatorOptions } from './search'; +export { + DefaultTechDocsCollator, + DefaultTechDocsCollatorFactory, +} from './search'; +export type { + TechDocsCollatorFactoryOptions, + TechDocsCollatorOptions, +} from './search'; /** * @deprecated Use directly from @backstage/techdocs-common diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts index d9f75767a7..887dc60b1a 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts @@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [ owner: 'someone', }, }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'test-entity', - description: 'The expected description', - }, - spec: { - type: 'some-type', - lifecycle: 'experimental', - }, - }, ]; describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => { diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 73c030cc51..d36e1c0ae5 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -24,7 +24,6 @@ import { RELATION_OWNED_BY, stringifyEntityRef, } from '@backstage/catalog-model'; -import { DocumentCollator } from '@backstage/search-common'; import fetch from 'node-fetch'; import unescape from 'lodash/unescape'; import { Logger } from 'winston'; @@ -69,8 +68,10 @@ type EntityInfo = { * A search collator responsible for gathering and transforming TechDocs documents. * * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultTechDocsCollatorFactory` instead. */ -export class DefaultTechDocsCollator implements DocumentCollator { +export class DefaultTechDocsCollator { public readonly type: string = 'techdocs'; public readonly visibilityPermission = catalogEntityReadPermission; diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts new file mode 100644 index 0000000000..fafaae71b9 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + getVoidLogger, + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; + +const logger = getVoidLogger(); + +const mockSearchDocIndex = { + config: { + lang: ['en'], + min_search_length: 3, + prebuild_index: false, + separator: '[\\s\\-]+', + }, + docs: [ + { + location: '', + text: 'docs docs docs', + title: 'Home', + }, + { + location: 'local-development/', + text: 'Docs for first subtitle', + title: 'Local development', + }, + { + location: 'local-development/#development', + text: 'Docs for sub-subtitle', + title: 'Development', + }, + ], +}; + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity with Docs!', + name: 'test-entity-with-docs', + description: 'Documented description', + annotations: { + 'backstage.io/techdocs-ref': './', + }, + }, + spec: { + type: 'dog', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultTechDocsCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + logger: getVoidLogger(), + tokenManager: mockTokenManager, + }; + + it('has expected type', () => { + const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('techdocs'); + }); + + describe('getCollator', () => { + let factory: DefaultTechDocsCollatorFactory; + let collator: Readable; + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + + worker.use( + rest.get( + 'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json', + (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)), + ), + rest.get('http://test-backend/entities', (req, res, ctx) => { + // Imitate offset/limit pagination. + const offset = parseInt( + req.url.searchParams.get('offset') || '0', + 10, + ); + const limit = parseInt( + req.url.searchParams.get('limit') || '500', + 10, + ); + + // Limit 50 corresponds to a case testing pagination. + if (limit === 50) { + // Return 50 copies of invalid entities on the first request. + if (offset === 0) { + return res(ctx.status(200), ctx.json(Array(50).fill({}))); + } + // Then just the regular 2 on the second. + return res(ctx.status(200), ctx.json(expectedEntities)); + } + return res( + ctx.status(200), + ctx.json(expectedEntities.slice(offset, limit + offset)), + ); + }), + ); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog and tech docs services', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs'); + expect(documents).toHaveLength(mockSearchDocIndex.docs.length); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind.toLocaleLowerCase('en-US'), + name: entity.metadata.name, + }); + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + logger, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + location: '/software/test-entity-with-docs', + }); + }); + + it('paginates through catalog entities using batchSize', async () => { + // A parallelismLimit of 1 is a catalog limit of 50 per request. Code + // above in the /entities handler ensures valid entities are only + // returned on the second page. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + parallelismLimit: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // Only 1 entity with TechDocs configured multipled by 3 pages. + expect(documents).toHaveLength(3); + }); + + describe('with legacyPathCasing configuration', () => { + beforeEach(async () => { + const legacyConfig = new ConfigReader({ + techdocs: { + legacyUseCaseSensitiveTripletPaths: true, + }, + }); + factory = DefaultTechDocsCollatorFactory.fromConfig( + legacyConfig, + options, + ); + collator = await factory.getCollator(); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind, + name: entity.metadata.name, + }); + }); + }); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts new file mode 100644 index 0000000000..181f032b54 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts @@ -0,0 +1,254 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + CATALOG_FILTER_EXISTS, +} from '@backstage/catalog-client'; +import { + Entity, + parseEntityRef, + RELATION_OWNED_BY, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { TechDocsDocument } from '@backstage/techdocs-common'; +import unescape from 'lodash/unescape'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +interface MkSearchIndexDoc { + title: string; + text: string; + location: string; +} + +/** + * Options to configure the TechDocs collator factory + * + * @public + */ +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + +type EntityInfo = { + name: string; + namespace: string; + kind: string; +}; + +/** + * A search collator factory responsible for gathering and transforming + * TechDocs documents. + * + * @public + */ +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'techdocs'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private discovery: PluginEndpointDiscovery; + private locationTemplate: string; + private readonly logger: Logger; + private readonly catalogClient: CatalogApi; + private readonly tokenManager: TokenManager; + private readonly parallelismLimit: number; + private readonly legacyPathCasing: boolean; + + private constructor(options: TechDocsCollatorFactoryOptions) { + this.discovery = options.discovery; + this.locationTemplate = + options.locationTemplate || '/docs/:namespace/:kind/:name/:path'; + this.logger = options.logger; + this.catalogClient = + options.catalogClient || + new CatalogClient({ discoveryApi: options.discovery }); + this.parallelismLimit = options.parallelismLimit ?? 10; + this.legacyPathCasing = options.legacyPathCasing ?? false; + this.tokenManager = options.tokenManager; + } + + static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) { + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing }); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const limit = pLimit(this.parallelismLimit); + const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs'); + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. The batchSize is calculated as a factor of the given + // parallelism limit to simplify configuration. + const batchSize = this.parallelismLimit * 50; + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': + CATALOG_FILTER_EXISTS, + }, + fields: [ + 'kind', + 'namespace', + 'metadata.annotations', + 'metadata.name', + 'metadata.title', + 'metadata.namespace', + 'spec.type', + 'spec.lifecycle', + 'relations', + ], + limit: batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === batchSize; + entitiesRetrieved += entities.length; + + const docPromises = entities + .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) + .map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const searchIndexResponse = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + const searchIndex = await searchIndexResponse.json(); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + title: unescape(doc.title), + text: unescape(doc.text || ''), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', + { + ...entityInfo, + path: doc.location, + }, + ), + path: doc.location, + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); + yield* (await Promise.all(docPromises)).flat(); + } + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted; + } + + private static constructDocsIndexUrl( + techDocsBaseUrl: string, + entityInfo: { kind: string; namespace: string; name: string }, + ) { + return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`; + } + + private static handleEntityInfoCasing( + legacyPaths: boolean, + entityInfo: EntityInfo, + ): EntityInfo { + return legacyPaths + ? entityInfo + : Object.entries(entityInfo).reduce((acc, [key, value]) => { + return { ...acc, [key]: value.toLocaleLowerCase('en-US') }; + }, {} as EntityInfo); + } +} + +function getSimpleEntityOwnerString(entity: Entity): string { + if (entity.relations) { + const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY); + if (owner) { + const { name } = parseEntityRef(owner.targetRef); + return name; + } + } + return ''; +} diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index fbbd23b964..68e3b4edc7 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; +export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; +export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting these in a future release. + */ +export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator'; diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts index 678a4b0539..19684486dc 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts @@ -16,11 +16,11 @@ import { CachedEntityLoader } from './CachedEntityLoader'; import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; describe('CachedEntityLoader', () => { const catalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any; const cache: jest.Mocked = { @@ -28,7 +28,7 @@ describe('CachedEntityLoader', () => { set: jest.fn(), } as any; - const entityName: EntityName = { + const entityName: CompoundEntityRef = { kind: 'component', namespace: 'default', name: 'test', @@ -53,7 +53,7 @@ describe('CachedEntityLoader', () => { it('writes entities to cache', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); @@ -71,12 +71,12 @@ describe('CachedEntityLoader', () => { const result = await loader.load(entityName, token); expect(result).toEqual(entity); - expect(catalog.getEntityByName).not.toBeCalled(); + expect(catalog.getEntityByRef).not.toBeCalled(); }); it('does not cache missing entites', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); const result = await loader.load(entityName, token); @@ -86,7 +86,7 @@ describe('CachedEntityLoader', () => { it('uses entity ref as cache key for anonymous users', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, undefined); @@ -103,7 +103,7 @@ describe('CachedEntityLoader', () => { setTimeout(() => resolve(undefined), 10000); }), ); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts index 81c6ec7431..424d9b9541 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts @@ -17,7 +17,7 @@ import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -37,17 +37,17 @@ export class CachedEntityLoader { } async load( - entityName: EntityName, + entityRef: CompoundEntityRef, token: string | undefined, ): Promise { - const cacheKey = this.getCacheKey(entityName, token); + const cacheKey = this.getCacheKey(entityRef, token); let result = await this.getFromCache(cacheKey); if (result) { return result; } - result = await this.catalog.getEntityByName(entityName, { token }); + result = await this.catalog.getEntityByRef(entityRef, { token }); if (result) { this.cache.set(cacheKey, result, { ttl: 5000 }); @@ -66,7 +66,7 @@ export class CachedEntityLoader { } private getCacheKey( - entityName: EntityName, + entityName: CompoundEntityRef, token: string | undefined, ): string { const key = ['catalog', stringifyEntityRef(entityName)]; diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts new file mode 100644 index 0000000000..84bd960f8f --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DefaultDocsBuildStrategy } from './DocsBuildStrategy'; +import { ConfigReader } from '@backstage/config'; + +const MockedConfigReader = ConfigReader as jest.MockedClass< + typeof ConfigReader +>; + +jest.mock('@backstage/config'); + +describe('DefaultDocsBuildStrategy', () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + }, + }; + + const config = new ConfigReader({}); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('shouldBuild', () => { + it('should return true when techdocs.build is set to local', async () => { + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); + + MockedConfigReader.prototype.getString.mockReturnValue('local'); + + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); + + expect(result).toBe(true); + }); + + it('should return false when techdocs.build is set to external', async () => { + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); + + MockedConfigReader.prototype.getString.mockReturnValue('external'); + + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); + + expect(result).toBe(false); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts new file mode 100644 index 0000000000..42a16234f9 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; + +/** + * Parameters passed to the shouldBuild method on the DocsBuildStrategy interface + * + * @public + */ +export type ShouldBuildParameters = { + entity: Entity; +}; + +/** + * A strategy for when to build TechDocs locally, and when to skip building TechDocs (allowing for an external build) + * + * @public + */ +export interface DocsBuildStrategy { + shouldBuild(params: ShouldBuildParameters): Promise; +} + +export class DefaultDocsBuildStrategy { + private readonly config: Config; + + private constructor(config: Config) { + this.config = config; + } + + static fromConfig(config: Config): DefaultDocsBuildStrategy { + return new DefaultDocsBuildStrategy(config); + } + + async shouldBuild(_: ShouldBuildParameters): Promise { + return this.config.getString('techdocs.builder') === 'local'; + } +} diff --git a/plugins/techdocs-backend/src/service/index.ts b/plugins/techdocs-backend/src/service/index.ts index 0065e33a2a..7355a34e32 100644 --- a/plugins/techdocs-backend/src/service/index.ts +++ b/plugins/techdocs-backend/src/service/index.ts @@ -20,3 +20,7 @@ export type { RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, } from './router'; +export type { + DocsBuildStrategy, + ShouldBuildParameters, +} from './DocsBuildStrategy'; diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts index 43e9641efb..c7cda203bb 100644 --- a/plugins/techdocs-backend/src/service/router.test.ts +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -38,6 +38,7 @@ import { RouterOptions, } from './router'; import { TechDocsCache } from '../cache'; +import { DocsBuildStrategy } from './DocsBuildStrategy'; jest.mock('@backstage/catalog-client'); jest.mock('@backstage/config'); @@ -120,6 +121,9 @@ describe('createRouter', () => { const cache: jest.Mocked = { getClient: jest.fn(), }; + const docsBuildStrategy: jest.Mocked = { + shouldBuild: jest.fn(), + }; const outOfTheBoxOptions = { preparers, generators, @@ -128,6 +132,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; const recommendedOptions = { publisher, @@ -135,6 +140,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; beforeEach(() => { @@ -181,10 +187,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -198,10 +204,10 @@ describe('createRouter', () => { expect(response.status).toBe(304); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -210,7 +216,7 @@ describe('createRouter', () => { expect(response.status).toBe(500); expect(response.text).toMatch( - /Invalid configuration\. 'techdocs\.builder' was set to 'local' but no 'preparer' was provided to the router initialization/, + /Invalid configuration\. docsBuildStrategy\.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization./, ); expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0); @@ -219,7 +225,7 @@ describe('createRouter', () => { it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -244,7 +250,7 @@ describe('createRouter', () => { it('should return on updated', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { @@ -297,10 +303,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -322,10 +328,10 @@ data: {"updated":false} ); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -337,7 +343,7 @@ data: {"updated":false} expect(response.get('content-type')).toBe('text/event-stream'); expect(response.text).toEqual( `event: error -data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization." +data: "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization." `, ); @@ -348,7 +354,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -376,7 +382,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should return an event-stream', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 5877dda099..73f212d049 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -35,6 +35,10 @@ import { ScmIntegrations } from '@backstage/integration'; import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; import { createCacheMiddleware, TechDocsCache } from '../cache'; import { CachedEntityLoader } from './CachedEntityLoader'; +import { + DefaultDocsBuildStrategy, + DocsBuildStrategy, +} from './DocsBuildStrategy'; /** * Required dependencies for running TechDocs in the "out-of-the-box" @@ -51,6 +55,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; // TODO: Make database required when we're implementing database stuff. config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -65,6 +70,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -99,6 +105,8 @@ export async function createRouter( const router = Router(); const { publisher, config, logger, discovery } = options; const catalogClient = new CatalogClient({ discoveryApi: discovery }); + const docsBuildStrategy = + options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config); // Entities are cached to optimize the /static/docs request path, which can be called many times // when loading a single techdocs page. @@ -210,10 +218,13 @@ export async function createRouter( responseHandler = createEventStream(res); } - // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' - // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline - // of the repository) is responsible for building and publishing documentation to the storage provider - if (config.getString('techdocs.builder') !== 'local') { + // By default, techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to + // 'local'. If set to 'external', it will assume that an external process (e.g. CI/CD pipeline + // of the repository) is responsible for building and publishing documentation to the storage provider. + // Altering the implementation of the injected docsBuildStrategy allows for more complex behaviours, based on + // either config or the properties of the entity (e.g. annotations, labels, spec fields etc.). + const shouldBuild = await docsBuildStrategy.shouldBuild({ entity }); + if (!shouldBuild) { // However, if caching is enabled, take the opportunity to check and // invalidate stale cache entries. if (cache) { @@ -244,7 +255,7 @@ export async function createRouter( responseHandler.error( new Error( - "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization.", + "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.", ), ); }); diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 917c1d5d85..f7ef6e984a 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -7,11 +7,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { CSSProperties } from '@material-ui/styles'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FetchApi } from '@backstage/core-plugin-api'; import { IdentityApi } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; @@ -89,7 +89,7 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; @@ -161,7 +161,7 @@ export const Reader: (props: ReaderProps) => JSX.Element; // @public export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -187,9 +187,11 @@ export type TabsConfig = TabConfig[]; export interface TechDocsApi { getApiOrigin(): Promise; // (undocumented) - getEntityMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; // (undocumented) - getTechDocsMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -208,8 +210,10 @@ export class TechDocsClient implements TechDocsApi { discoveryApi: DiscoveryApi; // (undocumented) getApiOrigin(): Promise; - getEntityMetadata(entityId: EntityName): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -297,7 +301,7 @@ export const TechDocsReaderPageHeader: ( // @public export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; @@ -315,7 +319,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; @@ -324,7 +328,7 @@ export const TechDocsSearch: (props: TechDocsSearchProps) => JSX.Element; // @public export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; @@ -348,18 +352,18 @@ export interface TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; // (undocumented) - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } @@ -384,18 +388,18 @@ export class TechDocsStorageClient implements TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index 8dc0e5d30b..d08e914736 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -17,7 +17,7 @@ import { createDevApp } from '@backstage/dev-utils'; import { NotFoundError } from '@backstage/errors'; import React from 'react'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Reader, SyncResult, @@ -82,7 +82,10 @@ function createPage({ }); } - async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) { + async syncEntityDocs( + _: CompoundEntityRef, + logHandler?: (line: string) => void, + ) { if (syncDocsDelay) { for (let i = 0; i < 10; i++) { setTimeout( diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 4566bbf028..b349431b0d 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsEntityMetadata, TechDocsMetadata } from './types'; import { createApiRef } from '@backstage/core-plugin-api'; @@ -55,14 +55,14 @@ export interface TechDocsStorageApi { getApiOrigin(): Promise; getStorageUrl(): Promise; getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; } @@ -77,6 +77,8 @@ export interface TechDocsApi { * Set to techdocs.requestUrl as the URL for techdocs-backend API. */ getApiOrigin(): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; - getEntityMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; } diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index eac5da0b6b..f260f00547 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { DiscoveryApi, @@ -62,7 +62,9 @@ export class TechDocsClient implements TechDocsApi { * * @param entityId - Object containing entity data like name, namespace, etc. */ - async getTechDocsMetadata(entityId: EntityName): Promise { + async getTechDocsMetadata( + entityId: CompoundEntityRef, + ): Promise { const { kind, namespace, name } = entityId; const apiOrigin = await this.getApiOrigin(); @@ -84,7 +86,7 @@ export class TechDocsClient implements TechDocsApi { * @param entityId - Object containing entity data like name, namespace, etc. */ async getEntityMetadata( - entityId: EntityName, + entityId: CompoundEntityRef, ): Promise { const { kind, namespace, name } = entityId; @@ -149,7 +151,10 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @returns HTML content of the docs page as string * @throws Throws error when the page is not found. */ - async getEntityDocs(entityId: EntityName, path: string): Promise { + async getEntityDocs( + entityId: CompoundEntityRef, + path: string, + ): Promise { const { kind, namespace, name } = entityId; const storageUrl = await this.getStorageUrl(); @@ -190,7 +195,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @throws Throws error on error from sync endpoint in Techdocs Backend */ async syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler: (line: string) => void = () => {}, ): Promise { const { kind, namespace, name } = entityId; @@ -243,7 +248,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { async getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise { const { kind, namespace, name } = entityId; diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 33f5e0a08a..14fdb88395 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -45,7 +45,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: () => Promise.resolve(), + getEntityByRef: () => Promise.resolve(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx index a8fd472f7c..16d8d48bc3 100644 --- a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx @@ -33,7 +33,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx index b875ba4e98..883c496191 100644 --- a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx +++ b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx @@ -20,7 +20,7 @@ import useCopyToClipboard from 'react-use/lib/useCopyToClipboard'; import { useRouteRef, useApi, configApiRef } from '@backstage/core-plugin-api'; import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { rootDocsRouteRef } from '../../../routes'; @@ -76,7 +76,7 @@ export const DocsTable = (props: DocsTableProps) => { }), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), }, }; diff --git a/plugins/techdocs/src/home/components/Tables/types.ts b/plugins/techdocs/src/home/components/Tables/types.ts index c076d4c12d..6c06526d07 100644 --- a/plugins/techdocs/src/home/components/Tables/types.ts +++ b/plugins/techdocs/src/home/components/Tables/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** * Generic representing the metadata structure for a docs table row. @@ -26,6 +26,6 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx index e9c3a55b15..2088dbf4fc 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx @@ -31,7 +31,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx index c8149f1ff2..d2f0266af6 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx @@ -232,7 +232,12 @@ function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 4042585b6d..6aa5c99be4 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -34,7 +34,7 @@ import { alpha, } from '@material-ui/core'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, configApiRef } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; @@ -68,7 +68,7 @@ import { useReaderState } from './useReaderState'; * @public */ export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -95,7 +95,7 @@ const TechDocsReaderContext = createContext( const TechDocsReaderProvider = ({ children, entityRef, -}: PropsWithChildren<{ entityRef: EntityName }>) => { +}: PropsWithChildren<{ entityRef: CompoundEntityRef }>) => { const { '*': path } = useParams(); const { kind, namespace, name } = entityRef; const value = useReaderState(kind, namespace, name, path); @@ -116,7 +116,7 @@ const TechDocsReaderProvider = ({ * @internal */ export const withTechDocsReaderProvider = - (Component: ComponentType, entityRef: EntityName) => + (Component: ComponentType, entityRef: CompoundEntityRef) => (props: T) => ( @@ -157,7 +157,9 @@ const headings: TypographyHeadingsKeys[] = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']; * todo: Make public or stop exporting (see others: "altReaderExperiments") * @internal */ -export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { +export const useTechDocsReaderDom = ( + entityRef: CompoundEntityRef, +): Element | null => { const navigate = useNavigate(); const theme = useTheme(); const techdocsStorageApi = useApi(techdocsStorageApiRef); @@ -322,7 +324,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { --md-source-version-icon: url('data:image/svg+xml;charset=utf-8,'); --md-version-icon: url('data:image/svg+xml;charset=utf-8,'); } - + :host > * { /* CODE */ --md-code-fg-color: ${theme.palette.text.primary}; @@ -438,7 +440,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-main__inner { margin-top: 0; } - + .md-sidebar { height: calc(100% - 100px); position: fixed; @@ -450,13 +452,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-sidebar--secondary { right: ${theme.spacing(3)}px; } - + .md-content { max-width: calc(100% - 16rem * 2); margin-left: 16rem; margin-bottom: 50px; } - + .md-footer { position: fixed; bottom: 0px; @@ -471,7 +473,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-dialog { background-color: unset; } - + @media screen and (max-width: 76.1875em) { .md-nav { transition: none !important; @@ -567,7 +569,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { }), injectCss({ // Typeset - css: ` + css: ` .md-typeset { font-size: var(--md-typeset-font-size); } @@ -600,11 +602,11 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset .md-content__button { color: var(--md-default-fg-color); } - + .md-typeset hr { border-bottom: 0.05rem dotted ${theme.palette.divider}; } - + .md-typeset details { font-size: var(--md-typeset-font-size) !important; } @@ -621,7 +623,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset details[open] > summary:after { transform: rotate(90deg) translateX(-50%) !important; } - + .md-typeset blockquote { color: var(--md-default-fg-color--light); border-left: 0.2rem solid var(--md-default-fg-color--light); @@ -667,13 +669,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .highlight .md-clipboard:after { content: unset; } - + .highlight .nx { color: ${isDarkTheme ? '#ff53a3' : '#ec407a'}; } /* CODE HILITE */ - .codehilite .gd { + .codehilite .gd { background-color: ${ isDarkTheme ? 'rgba(248,81,73,0.65)' : '#fdd' }; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx index d03ea2ef61..184ab24b6d 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx @@ -21,12 +21,12 @@ import useAsync from 'react-use/lib/useAsync'; import { techdocsApiRef } from '../../api'; import { LegacyTechDocsPage } from './LegacyTechDocsPage'; import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, useApp } from '@backstage/core-plugin-api'; import { Page } from '@backstage/core-components'; /** - * Helper function that gives the children of {@link TechDocsReaderPage} acccess to techdocs and entity metadata + * Helper function that gives the children of {@link TechDocsReaderPage} access to techdocs and entity metadata * * @public */ @@ -37,7 +37,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx index 928833efed..5867641c48 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx @@ -19,7 +19,7 @@ import CodeIcon from '@material-ui/icons/Code'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Header, HeaderLabel } from '@backstage/core-components'; -import { EntityName, RELATION_OWNED_BY } from '@backstage/catalog-model'; +import { CompoundEntityRef, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { EntityRefLink, EntityRefLinks, @@ -35,7 +35,7 @@ import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; * @public */ export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; diff --git a/plugins/techdocs/src/reader/components/useRawPage.ts b/plugins/techdocs/src/reader/components/useRawPage.ts index 61b04a5a82..311f8554a9 100644 --- a/plugins/techdocs/src/reader/components/useRawPage.ts +++ b/plugins/techdocs/src/reader/components/useRawPage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { AsyncState } from 'react-use/lib/useAsync'; import { techdocsStorageApiRef } from '../../api'; @@ -22,7 +22,7 @@ import { useApi } from '@backstage/core-plugin-api'; export type RawPage = { content: string; path: string; - entityId: EntityName; + entityId: CompoundEntityRef; }; export function useRawPage( diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts index f3906b7b91..2e3b853e42 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsStorageApi } from '../../api'; import type { Transformer } from './transformer'; type AddBaseUrlOptions = { techdocsStorageApi: TechDocsStorageApi; - entityId: EntityName; + entityId: CompoundEntityRef; path: string; }; diff --git a/plugins/techdocs/src/search/components/TechDocsSearch.tsx b/plugins/techdocs/src/search/components/TechDocsSearch.tsx index c817fb3217..30cc692b7c 100644 --- a/plugins/techdocs/src/search/components/TechDocsSearch.tsx +++ b/plugins/techdocs/src/search/components/TechDocsSearch.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { SearchContextProvider, useSearch } from '@backstage/plugin-search'; import { makeStyles, @@ -42,7 +42,7 @@ const useStyles = makeStyles({ * @public */ export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; diff --git a/plugins/todo-backend/api-report.md b/plugins/todo-backend/api-report.md index 71094e03f8..f3f2af3fb9 100644 --- a/plugins/todo-backend/api-report.md +++ b/plugins/todo-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; import { ScmIntegrations } from '@backstage/integration'; @@ -19,7 +19,7 @@ export function createTodoParser(options?: TodoParserOptions): TodoParser; // @public (undocumented) export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { diff --git a/plugins/todo-backend/src/service/TodoReaderService.test.ts b/plugins/todo-backend/src/service/TodoReaderService.test.ts index 985687c6bb..943462bd63 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.test.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.test.ts @@ -44,6 +44,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { const mock = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -54,7 +55,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { getEntityFacets: jest.fn(), }; if (entity) { - mock.getEntityByName.mockReturnValue(entity); + mock.getEntityByRef.mockReturnValue(entity); } return mock; } @@ -93,7 +94,7 @@ describe('TodoReaderService', () => { offset: 0, limit: 10, }); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); @@ -304,7 +305,7 @@ describe('TodoReaderService', () => { message: 'Entity not found, component:default/my-component', }), ); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); diff --git a/plugins/todo-backend/src/service/TodoReaderService.ts b/plugins/todo-backend/src/service/TodoReaderService.ts index 86f53057da..4397f34045 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.ts @@ -66,7 +66,7 @@ export class TodoReaderService implements TodoService { throw new InputError('Entity filter is required to list TODOs'); } const token = options?.token; - const entity = await this.catalogClient.getEntityByName(req.entity, { + const entity = await this.catalogClient.getEntityByRef(req.entity, { token, }); if (!entity) { diff --git a/plugins/todo-backend/src/service/router.ts b/plugins/todo-backend/src/service/router.ts index a0e6a6ec94..33356a83d6 100644 --- a/plugins/todo-backend/src/service/router.ts +++ b/plugins/todo-backend/src/service/router.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { CompoundEntityRef, parseEntityRef } from '@backstage/catalog-model'; import { InputError } from '@backstage/errors'; import express from 'express'; import Router from 'express-promise-router'; @@ -52,7 +52,7 @@ export async function createRouter( if (entityRef && typeof entityRef !== 'string') { throw new InputError(`entity query must be a string`); } - let entity: EntityName | undefined = undefined; + let entity: CompoundEntityRef | undefined = undefined; if (entityRef) { try { entity = parseEntityRef(entityRef); diff --git a/plugins/todo-backend/src/service/types.ts b/plugins/todo-backend/src/service/types.ts index 90ee6bc913..88d2f2b340 100644 --- a/plugins/todo-backend/src/service/types.ts +++ b/plugins/todo-backend/src/service/types.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TodoItem } from '../lib'; /** @public */ export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts index 6635dd8b22..afe2c1a1bb 100644 --- a/scripts/api-extractor.ts +++ b/scripts/api-extractor.ts @@ -213,6 +213,7 @@ const NO_WARNING_PACKAGES = [ 'packages/errors', 'packages/integration', 'packages/integration-react', + 'packages/search-common', 'packages/techdocs-common', 'packages/test-utils', 'packages/theme', @@ -235,6 +236,7 @@ const NO_WARNING_PACKAGES = [ 'plugins/scaffolder-backend-module-rails', 'plugins/scaffolder-backend-module-yeoman', 'plugins/scaffolder-common', + 'plugins/search-backend-node', 'plugins/techdocs-backend', 'plugins/tech-insights', 'plugins/tech-insights-backend', diff --git a/scripts/prepare-release.js b/scripts/prepare-release.js index 5cc888fd66..be82e19344 100755 --- a/scripts/prepare-release.js +++ b/scripts/prepare-release.js @@ -28,10 +28,6 @@ const execFile = promisify(execFileCb); // All of these are considered to be main-line release branches const MAIN_BRANCHES = ['master', 'origin/master', 'changeset-release/master']; -// This prefix is used for patch branches, followed by the release version WITH a 'v' prefix -// For example, `patch/v1.2.0` -const PATCH_BRANCH_PREFIX = 'patch/'; - const DEPENDENCY_TYPES = [ 'dependencies', 'devDependencies', @@ -39,84 +35,25 @@ const DEPENDENCY_TYPES = [ 'peerDependencies', ]; -/** - * Returns the most recent release version on the main branch that is not a pre-release. - */ -async function getPreviousReleaseVersion(repo) { - // TODO(Rugvip): Figure out which field to sort by to avoid manual sort after - const { stdout: tagsStr } = await execFile( - 'git', - ['tag', '--list', 'v*', '--merged=HEAD'], - { shell: true, cwd: repo.root.dir }, - ); - const tags = tagsStr.trim().split(/\r\n|\n/); - const [latestTag] = semver.rsort(tags).filter(t => !semver.prerelease(t)); - return latestTag; -} - -/** - * Finds the tip of the patch branch of a given release version. - * Returns undefined if no patch branch exists. - */ -async function findTipOfPatchBranch(repo, release) { - try { - await execFile('git', ['fetch', 'origin', PATCH_BRANCH_PREFIX + release], { - shell: true, - cwd: repo.root.dir, - }); - } catch (error) { - if (error.stderr?.match(/fatal: couldn't find remote ref/i)) { - return undefined; - } - throw error; - } - const { stdout: refStr } = await execFile('git', ['rev-parse', 'FETCH_HEAD']); - return refStr.trim(); -} - -/** - * Returns a map of packages to their versions for any package version - * in that does not match the current version in the working directory. - */ -async function detectPatchVersionsForRef(repo, ref) { - const patchVersions = new Map(); - - for (const pkg of repo.packages) { - const pkgJsonPath = path.join( - path.relative(repo.root.dir, pkg.dir), - 'package.json', - ); - const { stdout: pkgJsonStr } = await execFile('git', [ - 'show', - `${ref}:${pkgJsonPath}`, - ]); - if (pkgJsonStr) { - const releasePkgJson = JSON.parse(pkgJsonStr); - const pkgJson = pkg.packageJson; - if (releasePkgJson.name !== pkgJson.name) { - throw new Error( - `Mismatched package name at ${pkg.dir}, ${releasePkgJson.name} !== ${pkgJson.name}`, - ); - } - if (releasePkgJson.version !== pkgJson.version) { - patchVersions.set(pkgJson.name, releasePkgJson.version); - } - } - } - - return patchVersions; -} - /** * Bumps up the versions of packages to account for * the base versions that are set in .changeset/patched.json. * This may be needed when we have made emergency releases. */ -async function applyPatchVersions(repo, patchVersions) { +async function updatePatchVersions() { + const patchedJsonPath = path.resolve('.changeset', 'patched.json'); + const { currentReleaseVersion } = await fs.readJson(patchedJsonPath); + if (Object.keys(currentReleaseVersion).length === 0) { + console.log('No currentReleaseVersion overrides found, skipping.'); + return; + } + + const { packages } = await getPackages(path.resolve('.')); + const pendingVersionBumps = new Map(); - for (const [name, version] of patchVersions) { - const pkg = repo.packages.find(p => p.packageJson.name === name); + for (const [name, version] of Object.entries(currentReleaseVersion)) { + const pkg = packages.find(p => p.packageJson.name === name); if (!pkg) { throw new Error(`Package ${name} not found`); } @@ -144,7 +81,7 @@ async function applyPatchVersions(repo, patchVersions) { }); } - for (const { dir, packageJson } of [repo.root, ...repo.packages]) { + for (const { dir, packageJson } of packages) { let hasChanges = false; if (pendingVersionBumps.has(packageJson.name)) { @@ -180,44 +117,20 @@ async function applyPatchVersions(repo, patchVersions) { }); } } -} -/** - * Detects any patched packages version since the most recent release on - * the main branch, and then bumps all packages in the repo accordingly. - */ -async function updatePackageVersions(repo) { - const previousRelease = await getPreviousReleaseVersion(repo); - console.log(`Found release version: ${previousRelease}`); - - const patchRef = await findTipOfPatchBranch(repo, previousRelease); - if (patchRef) { - console.log(`Tip of the patch branch: ${patchRef}`); - - const patchVersions = await detectPatchVersionsForRef(repo, patchRef); - if (patchVersions.size > 0) { - console.log( - `Found ${patchVersions.size} packages that were patched since the last release`, - ); - for (const [name, version] of patchVersions) { - console.log(` ${name}: ${version}`); - } - - await applyPatchVersions(repo, patchVersions); - } else { - console.log('No packages were patched since the last release'); - } - } else { - console.log('No patch branch found'); - } + await fs.writeJSON( + patchedJsonPath, + { currentReleaseVersion: {} }, + { spaces: 2, encoding: 'utf8' }, + ); } /** * Returns the mode and tag that is currently set * in the .changeset/pre.json file */ -async function getPreInfo(repo) { - const pre = path.join(repo.root.dir, '.changeset', 'pre.json'); +async function getPreInfo(rootPath) { + const pre = path.join(rootPath, '.changeset', 'pre.json'); if (!(await fs.pathExists(pre))) { return { mode: undefined, tag: undefined }; } @@ -226,30 +139,26 @@ async function getPreInfo(repo) { return { mode, tag }; } -/** - * Returns the name of the current git branch - */ -async function getCurrentBranch(repo) { - const { stdout } = await execFile( - 'git', - ['rev-parse', '--abbrev-ref', 'HEAD'], - { cwd: repo.root.dir, shell: true }, - ); - return stdout.trim(); -} - /** * Bumps the release version in the root package.json. * * This takes into account whether we're in pre-release mode or on a patch branch. */ -async function updateBackstageReleaseVersion(repo, type) { - const { mode: preMode, tag: preTag } = await getPreInfo(repo); +async function updateBackstageReleaseVersion() { + const rootPath = path.resolve(__dirname, '..'); + const branchName = await execFile( + 'git', + ['rev-parse', '--abbrev-ref', 'HEAD'], + { shell: true }, + ).then(({ stdout }) => stdout.trim()); + const { mode: preMode, tag: preTag } = await getPreInfo(rootPath); - const { version: currentVersion } = repo.root.packageJson; + const packagePath = path.join(rootPath, 'package.json'); + const package = await fs.readJson(packagePath); + const { version: currentVersion } = package; let nextVersion; - if (type === 'minor') { + if (MAIN_BRANCHES.includes(branchName)) { if (preMode === 'pre') { if (semver.prerelease(currentVersion)) { nextVersion = semver.inc(currentVersion, 'pre', preTag); @@ -261,7 +170,7 @@ async function updateBackstageReleaseVersion(repo, type) { } else { nextVersion = semver.inc(currentVersion, 'minor'); } - } else if (type === 'patch') { + } else { if (preMode) { throw new Error(`Unexpected pre mode ${preMode} on branch ${branchName}`); } @@ -269,9 +178,9 @@ async function updateBackstageReleaseVersion(repo, type) { } await fs.writeJson( - path.join(repo.root.dir, 'package.json'), + packagePath, { - ...repo.root.packageJson, + ...package, version: nextVersion, }, { spaces: 2, encoding: 'utf8' }, @@ -279,17 +188,8 @@ async function updateBackstageReleaseVersion(repo, type) { } async function main() { - const repo = await getPackages(__dirname); - const branchName = await getCurrentBranch(repo); - const isMainBranch = MAIN_BRANCHES.includes(branchName); - - console.log(`Current branch: ${branchName}`); - if (isMainBranch) { - console.log('Main release, updating package versions'); - await updatePackageVersions(repo); - } - - await updateBackstageReleaseVersion(repo, isMainBranch ? 'minor' : 'patch'); + await updatePatchVersions(); + await updateBackstageReleaseVersion(); } main().catch(error => { diff --git a/yarn.lock b/yarn.lock index 90dd0207c2..2b13c60201 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6726,15 +6726,23 @@ eslint-utils "^3.0.0" "@typescript-eslint/parser@^5.9.0": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.9.1.tgz#b114011010a87e17b3265ca715e16c76a9834cef" - integrity sha512-PLYO0AmwD6s6n0ZQB5kqPgfvh73p0+VqopQQLuNfi7Lm0EpfKyDalchpVwkE+81k5HeiRrTV/9w1aNHzjD7C4g== + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.13.0.tgz#0394ed8f2f849273c0bf4b811994d177112ced5c" + integrity sha512-GdrU4GvBE29tm2RqWOM0P5QfCtgCyN4hXICj/X9ibKED16136l9ZpoJvCL5pSKtmJzA+NRDzQ312wWMejCVVfg== dependencies: - "@typescript-eslint/scope-manager" "5.9.1" - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/typescript-estree" "5.9.1" + "@typescript-eslint/scope-manager" "5.13.0" + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/typescript-estree" "5.13.0" debug "^4.3.2" +"@typescript-eslint/scope-manager@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.13.0.tgz#cf6aff61ca497cb19f0397eea8444a58f46156b6" + integrity sha512-T4N8UvKYDSfVYdmJq7g2IPJYCRzwtp74KyDZytkR4OL3NRupvswvmJQJ4CX5tDSurW2cvCc1Ia1qM7d0jpa7IA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + "@typescript-eslint/scope-manager@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.0.tgz#02dfef920290c1dcd7b1999455a3eaae7a1a3117" @@ -6743,14 +6751,6 @@ "@typescript-eslint/types" "5.9.0" "@typescript-eslint/visitor-keys" "5.9.0" -"@typescript-eslint/scope-manager@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.1.tgz#6c27be89f1a9409f284d95dfa08ee3400166fe69" - integrity sha512-8BwvWkho3B/UOtzRyW07ffJXPaLSUKFBjpq8aqsRvu6HdEuzCY57+ffT7QoV4QXJXWSU1+7g3wE4AlgImmQ9pQ== - dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - "@typescript-eslint/type-utils@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.9.0.tgz#fd5963ead04bc9b7af9c3a8e534d8d39f1ce5f93" @@ -6760,15 +6760,28 @@ debug "^4.3.2" tsutils "^3.21.0" +"@typescript-eslint/types@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.13.0.tgz#da1de4ae905b1b9ff682cab0bed6b2e3be9c04e5" + integrity sha512-LmE/KO6DUy0nFY/OoQU0XelnmDt+V8lPQhh8MOVa7Y5k2gGRd6U9Kp3wAjhB4OHg57tUO0nOnwYQhRRyEAyOyg== + "@typescript-eslint/types@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.0.tgz#e5619803e39d24a03b3369506df196355736e1a3" integrity sha512-mWp6/b56Umo1rwyGCk8fPIzb9Migo8YOniBGPAQDNC6C52SeyNGN4gsVwQTAR+RS2L5xyajON4hOLwAGwPtUwg== -"@typescript-eslint/types@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.1.tgz#1bef8f238a2fb32ebc6ff6d75020d9f47a1593c6" - integrity sha512-SsWegWudWpkZCwwYcKoDwuAjoZXnM1y2EbEerTHho19Hmm+bQ56QG4L4jrtCu0bI5STaRTvRTZmjprWlTw/5NQ== +"@typescript-eslint/typescript-estree@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.13.0.tgz#b37c07b748ff030a3e93d87c842714e020b78141" + integrity sha512-Q9cQow0DeLjnp5DuEDjLZ6JIkwGx3oYZe+BfcNuw/POhtpcxMTy18Icl6BJqTSd+3ftsrfuVb7mNHRZf7xiaNA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + debug "^4.3.2" + globby "^11.0.4" + is-glob "^4.0.3" + semver "^7.3.5" + tsutils "^3.21.0" "@typescript-eslint/typescript-estree@5.9.0": version "5.9.0" @@ -6783,18 +6796,13 @@ semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/typescript-estree@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.9.1.tgz#d5b996f49476495070d2b8dd354861cf33c005d6" - integrity sha512-gL1sP6A/KG0HwrahVXI9fZyeVTxEYV//6PmcOn1tD0rw8VhUWYeZeuWHwwhnewnvEMcHjhnJLOBhA9rK4vmb8A== +"@typescript-eslint/visitor-keys@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.13.0.tgz#f45ff55bcce16403b221ac9240fbeeae4764f0fd" + integrity sha512-HLKEAS/qA1V7d9EzcpLFykTePmOQqOFim8oCvhY3pZgQ8Hi38hYpHd9e5GN6nQBFQNecNhws5wkS9Y5XIO0s/g== dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - debug "^4.3.2" - globby "^11.0.4" - is-glob "^4.0.3" - semver "^7.3.5" - tsutils "^3.21.0" + "@typescript-eslint/types" "5.13.0" + eslint-visitor-keys "^3.0.0" "@typescript-eslint/visitor-keys@5.9.0": version "5.9.0" @@ -6804,14 +6812,6 @@ "@typescript-eslint/types" "5.9.0" eslint-visitor-keys "^3.0.0" -"@typescript-eslint/visitor-keys@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.9.1.tgz#f52206f38128dd4f675cf28070a41596eee985b7" - integrity sha512-Xh37pNz9e9ryW4TVdwiFzmr4hloty8cFj8GTWMXh3Z8swGwyQWeCcNgF0hm6t09iZd6eiZmIf4zHedQVP6TVtg== - dependencies: - "@typescript-eslint/types" "5.9.1" - eslint-visitor-keys "^3.0.0" - "@vscode/sqlite3@^5.0.7": version "5.0.7" resolved "https://registry.npmjs.org/@vscode/sqlite3/-/sqlite3-5.0.7.tgz#358df36bb0e9e735c54785e3e4b9b2dce1d32895" @@ -11548,9 +11548,9 @@ escodegen@^2.0.0: source-map "~0.6.1" eslint-config-prettier@^8.3.0: - version "8.3.0" - resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz#f7471b20b6fe8a9a9254cc684454202886a2dd7a" - integrity sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew== + version "8.5.0" + resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz#5a81680ec934beca02c7b1a61cf8ca34b66feab1" + integrity sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q== eslint-formatter-friendly@^7.0.0: version "7.0.0" @@ -16251,9 +16251,9 @@ ldap-filter@^0.3.3: assert-plus "^1.0.0" ldapjs@^2.2.0: - version "2.3.1" - resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.1.tgz#04136815fb1f21d692ac87fab5961a04d86e8b04" - integrity sha512-kf0tHHLrpwKaBAQOhYHXgdeh2PkFuCCxWgLb1MRn67ZQVo787D2pij3mmHVZx193GIdM8xcfi8HF6AIYYnj0fQ== + version "2.3.2" + resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.2.tgz#a599d081519f70462941cc33a50e9354c32f35b7" + integrity sha512-FU+GR/qbQ96WUZ2DUb7FzaEybYvv3240wTVPcbsdELB3o4cK92zGVjntsh68siVkLeCmlCcsd/cIQzyGXSS7LA== dependencies: abstract-logging "^2.0.0" asn1 "^0.2.4"