diff --git a/.changeset/beige-snails-drum.md b/.changeset/beige-snails-drum.md new file mode 100644 index 0000000000..3476c9383b --- /dev/null +++ b/.changeset/beige-snails-drum.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-model': patch +--- + +Move `@types/json-schema` to be a dev dependency diff --git a/.changeset/brown-dryers-serve.md b/.changeset/brown-dryers-serve.md new file mode 100644 index 0000000000..40c02538b8 --- /dev/null +++ b/.changeset/brown-dryers-serve.md @@ -0,0 +1,10 @@ +--- +'@backstage/catalog-client': minor +--- + +**BREAKING**: Removed the old deprecated request/response types: + +- `CatalogEntitiesRequest` - please use `GetEntitiesRequest` instead +- `CatalogEntityAncestorsRequest` - please use `GetEntityAncestorsRequest` instead +- `CatalogEntityAncestorsResponse` - please use `GetEntityAncestorsResponse` instead +- `CatalogListResponse` - please use `GetEntitiesResponse` instead diff --git a/.changeset/chilled-dolls-agree.md b/.changeset/chilled-dolls-agree.md new file mode 100644 index 0000000000..750307bf53 --- /dev/null +++ b/.changeset/chilled-dolls-agree.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**Breaking**: Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-backend/alpha` instead of `@backstage/plugin-catalog-backend`. + +- `catalogConditions` +- `createCatalogPolicyDecision` +- `permissionRules` +- `createCatalogPermissionRule` diff --git a/.changeset/chilled-items-trade.md b/.changeset/chilled-items-trade.md new file mode 100644 index 0000000000..b069885498 --- /dev/null +++ b/.changeset/chilled-items-trade.md @@ -0,0 +1,13 @@ +--- +'@backstage/backend-common': minor +--- + +**BREAKING**: The connection string for `redis` cache store now requires a protocol prefix. + +```diff +backend: + cache: + store: redis +- connection: user:pass@cache.example.com:6379 ++ connection: redis://user:pass@cache.example.com:6379 +``` diff --git a/.changeset/dull-months-knock.md b/.changeset/dull-months-knock.md new file mode 100644 index 0000000000..3b957518c2 --- /dev/null +++ b/.changeset/dull-months-knock.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +Added a new interface that allows for customization of when to build techdocs diff --git a/.changeset/fluffy-trees-occur.md b/.changeset/fluffy-trees-occur.md new file mode 100644 index 0000000000..6e41a61069 --- /dev/null +++ b/.changeset/fluffy-trees-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add config option `groupExpand` to allow expanding a single relationship diff --git a/.changeset/gentle-icons-vanish.md b/.changeset/gentle-icons-vanish.md new file mode 100644 index 0000000000..27ea78863b --- /dev/null +++ b/.changeset/gentle-icons-vanish.md @@ -0,0 +1,20 @@ +--- +'@backstage/plugin-api-docs': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-badges-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-rollbar': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `getEntityByRef` instead of `getEntityByName` in the catalog client diff --git a/.changeset/giant-taxis-drop.md b/.changeset/giant-taxis-drop.md new file mode 100644 index 0000000000..41f8a91d40 --- /dev/null +++ b/.changeset/giant-taxis-drop.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-common': minor +--- + +Remove resourceType property from catalogEntityCreatePermission. Resource type refers to the type of resources whose resourceRefs should be passed along with authorize requests, to allow conditional responses for that resource type. Since creation does not correspond to an entity (as the entity does not exist at the time of authorization), the resourceRef should not be included on the permission. diff --git a/.changeset/gorgeous-actors-shave.md b/.changeset/gorgeous-actors-shave.md new file mode 100644 index 0000000000..bcf5f43018 --- /dev/null +++ b/.changeset/gorgeous-actors-shave.md @@ -0,0 +1,12 @@ +--- +'@backstage/create-app': patch +--- + +Update import location of catalogEntityCreatePermission. + +To apply this change to an existing app, make the following change to `packages/app/src/App.tsx`: + +```diff +-import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common'; ++import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha'; +``` diff --git a/.changeset/gorgeous-boats-hide.md b/.changeset/gorgeous-boats-hide.md new file mode 100644 index 0000000000..5c60240433 --- /dev/null +++ b/.changeset/gorgeous-boats-hide.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-explore': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-org': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-techdocs': patch +--- + +- **DEPRECATION**: Deprecated `formatEntityRefTitle` in favor of the new `humanizeEntityRef` method instead. Please migrate to using the new method instead. diff --git a/.changeset/grumpy-apes-repeat.md b/.changeset/grumpy-apes-repeat.md new file mode 100644 index 0000000000..bc47b0d422 --- /dev/null +++ b/.changeset/grumpy-apes-repeat.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +DockerContainerRunner.runContainer now automatically removes the container when its execution terminates diff --git a/.changeset/honest-students-clean.md b/.changeset/honest-students-clean.md new file mode 100644 index 0000000000..ca8db28343 --- /dev/null +++ b/.changeset/honest-students-clean.md @@ -0,0 +1,7 @@ +--- +'@backstage/integration': minor +'@backstage/backend-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +ensure `apiBaseUrl` being set for Bitbucket integrations, replace hardcoded defaults diff --git a/.changeset/large-dancers-learn.md b/.changeset/large-dancers-learn.md new file mode 100644 index 0000000000..9f25990924 --- /dev/null +++ b/.changeset/large-dancers-learn.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +- **BREAKING**: Removed the `FavouriteTemplate` export in favor of the `FavoriteEntity` from `@backstage/plugin-catalog-react`. Please migrate any usages to that component instead if you are creating your own `TemplateCard` page. diff --git a/.changeset/lemon-needles-applaud.md b/.changeset/lemon-needles-applaud.md new file mode 100644 index 0000000000..a77d3fd33c --- /dev/null +++ b/.changeset/lemon-needles-applaud.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATED**: The `results` export, and instead adding `processingResult` with the same shape and purpose. diff --git a/.changeset/long-weeks-thank.md b/.changeset/long-weeks-thank.md new file mode 100644 index 0000000000..a6c3a0719f --- /dev/null +++ b/.changeset/long-weeks-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Internal restructuring to collect the various provider files in a `modules` folder while waiting to be externalized diff --git a/.changeset/metal-months-tie.md b/.changeset/metal-months-tie.md new file mode 100644 index 0000000000..890c7e8b5d --- /dev/null +++ b/.changeset/metal-months-tie.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Deprecated `favoriteEntityTooltip` and `favoriteEntityIcon` since the utility value is very low. diff --git a/.changeset/nasty-pets-join.md b/.changeset/nasty-pets-join.md new file mode 100644 index 0000000000..1ff07274cf --- /dev/null +++ b/.changeset/nasty-pets-join.md @@ -0,0 +1,9 @@ +--- +'@backstage/catalog-model': patch +--- + +**DEPRECATION**: Deprecated the `EntityName` type, and added the better-named `CompoundEntityRef` to replace it. + +**DEPRECATION**: Deprecated the `getEntityName` function, and added the better-named `getCompoundEntityRef` to replace it. + +Please switch over to using the new symbols, as the old ones may be removed in a future release. diff --git a/.changeset/new-foxes-matter.md b/.changeset/new-foxes-matter.md new file mode 100644 index 0000000000..234dd78451 --- /dev/null +++ b/.changeset/new-foxes-matter.md @@ -0,0 +1,7 @@ +--- +'@backstage/core-app-api': minor +'@backstage/core-plugin-api': minor +'@backstage/test-utils': minor +--- + +**BREAKING**: Removed the deprecated `get` method from `StorageAPI` and its implementations, this method has been replaced by the `snapshot` method. The return value from snapshot no longer includes `newValue` which has been replaced by `value`. For getting notified when a value changes, use `observe$`. diff --git a/.changeset/nice-dragons-collect.md b/.changeset/nice-dragons-collect.md new file mode 100644 index 0000000000..b3d9efefd9 --- /dev/null +++ b/.changeset/nice-dragons-collect.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-permission-common': patch +--- + +Add api doc comments to `Permission` type properties. diff --git a/.changeset/nice-walls-reply.md b/.changeset/nice-walls-reply.md new file mode 100644 index 0000000000..f6d1fb7562 --- /dev/null +++ b/.changeset/nice-walls-reply.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-common': minor +--- + +**Breaking**: Mark permission-related exports as alpha. This means that the exports below should now be imported from `@backstage/plugin-catalog-common/alpha` instead of `@backstage/plugin-catalog-common`. + +- `RESOURCE_TYPE_CATALOG_ENTITY` +- `catalogEntityReadPermission` +- `catalogEntityCreatePermission` +- `catalogEntityDeletePermission` +- `catalogEntityRefreshPermission` +- `catalogLocationReadPermission` +- `catalogLocationCreatePermission` +- `catalogLocationDeletePermission` diff --git a/.changeset/ninety-kids-drop.md b/.changeset/ninety-kids-drop.md new file mode 100644 index 0000000000..ed5cf6330b --- /dev/null +++ b/.changeset/ninety-kids-drop.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-react': minor +--- + +Removed some previously deprecated `routeRefs` as follows: + +- **BREAKING**: Removed `entityRoute` in favor of `entityRouteRef`. +- **BREAKING**: Removed the previously deprecated `rootRoute` and `catalogRouteRef`. If you want to refer to the catalog index page from a public plugin you now need to use an `ExternalRouteRef` instead. For private plugins it is possible to take the shortcut of referring directly to `catalogPlugin.routes.indexPage` instead. diff --git a/.changeset/old-waves-wash.md b/.changeset/old-waves-wash.md new file mode 100644 index 0000000000..34ba0760b5 --- /dev/null +++ b/.changeset/old-waves-wash.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +'@backstage/plugin-catalog-backend-module-ldap': patch +'@backstage/plugin-catalog-backend-module-msgraph': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Use the new `processingResult` export from the catalog backend diff --git a/.changeset/olive-glasses-approve.md b/.changeset/olive-glasses-approve.md new file mode 100644 index 0000000000..e74b7ef485 --- /dev/null +++ b/.changeset/olive-glasses-approve.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Export FetchUrlReader to facilitate more flexible configuration of the backend. diff --git a/.changeset/olive-lobsters-rescue.md b/.changeset/olive-lobsters-rescue.md new file mode 100644 index 0000000000..37a7712782 --- /dev/null +++ b/.changeset/olive-lobsters-rescue.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +**BREAKING**: Removed export of `GithubSession` and `SamlSession` which are only used internally. diff --git a/.changeset/orange-crews-explain.md b/.changeset/orange-crews-explain.md new file mode 100644 index 0000000000..4b831392eb --- /dev/null +++ b/.changeset/orange-crews-explain.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': minor +--- + +**BREAKING**: Removed the deprecated `app.` template variables from the `index.html` templating. These should be replaced by using `config.getString("app.")` instead. diff --git a/.changeset/plenty-tables-mix.md b/.changeset/plenty-tables-mix.md new file mode 100644 index 0000000000..5dffdf6946 --- /dev/null +++ b/.changeset/plenty-tables-mix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Deprecated the `BitbucketRepositoryParser` type. diff --git a/.changeset/polite-houses-wink.md b/.changeset/polite-houses-wink.md new file mode 100644 index 0000000000..3e177f5383 --- /dev/null +++ b/.changeset/polite-houses-wink.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +- **BREAKING**: The `isOwnerOf` function has been marked as `@alpha` and is now only available via the `@backstage/plugin-catalog-react/alpha` import. The limitations of this function with regards to only supporting direct relations have also been documented. diff --git a/.changeset/polite-poems-nail.md b/.changeset/polite-poems-nail.md new file mode 100644 index 0000000000..c3eb97fce0 --- /dev/null +++ b/.changeset/polite-poems-nail.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': minor +--- + +Marked `useEntityPermission` as alpha since the underlying permission framework is under active development. diff --git a/.changeset/poor-hounds-beam.md b/.changeset/poor-hounds-beam.md new file mode 100644 index 0000000000..925370c1b0 --- /dev/null +++ b/.changeset/poor-hounds-beam.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The CLI now bundles both version 16 and 17 of the patched `@hot-loader/react-dom` dependency, and selects the appropriate one based on what version of `react-dom` is installed within the app. diff --git a/.changeset/popular-items-tan.md b/.changeset/popular-items-tan.md new file mode 100644 index 0000000000..b91773a5d8 --- /dev/null +++ b/.changeset/popular-items-tan.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-permission-node': patch +--- + +Export some utility functions for parsing PermissionCriteria + +`isAndCriteria`, `isOrCriteria`, `isNotCriteria` are now exported. diff --git a/.changeset/quick-mugs-pay.md b/.changeset/quick-mugs-pay.md new file mode 100644 index 0000000000..f2d3a00eb8 --- /dev/null +++ b/.changeset/quick-mugs-pay.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add documentation for config options `userGroupMemberSearch` and `groupSearch` diff --git a/.changeset/rare-insects-punch.md b/.changeset/rare-insects-punch.md new file mode 100644 index 0000000000..d22b851abd --- /dev/null +++ b/.changeset/rare-insects-punch.md @@ -0,0 +1,10 @@ +--- +'@backstage/app-defaults': minor +'@backstage/core-app-api': minor +'@backstage/core-components': minor +'@backstage/core-plugin-api': minor +'@backstage/test-utils': minor +'@backstage/plugin-user-settings': minor +--- + +**BREAKING**: Removed deprecated `auth0AuthApiRef`, `oauth2ApiRef`, `samlAuthApiRef` and `oidcAuthApiRef` as these APIs are too generic to be useful. Instructions for how to migrate can be found at [https://backstage.io/docs/api/deprecations#generic-auth-api-refs](https://backstage.io/docs/api/deprecations#generic-auth-api-refs). diff --git a/.changeset/real-kids-hide.md b/.changeset/real-kids-hide.md new file mode 100644 index 0000000000..188a4061c9 --- /dev/null +++ b/.changeset/real-kids-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**Breaking**: Removed `entityRef` from `CatalogProcessorRelationResult`. The field is not used by the catalog and relation information is already available inside the `reation` property. diff --git a/.changeset/red-chefs-beam.md b/.changeset/red-chefs-beam.md new file mode 100644 index 0000000000..5cadccb0aa --- /dev/null +++ b/.changeset/red-chefs-beam.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-airbrake': patch +'@backstage/plugin-airbrake-backend': patch +--- + +The Airbrake plugin installation instructions have been updated to work better and conform to how the frontend and backend plugins are supposed to be integrated into a Backstage instance. diff --git a/.changeset/rotten-windows-worry.md b/.changeset/rotten-windows-worry.md new file mode 100644 index 0000000000..a1ab353ce3 --- /dev/null +++ b/.changeset/rotten-windows-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/catalog-client': patch +--- + +**DEPRECATION**: Deprecated `getEntityByName` from `CatalogApi` and added `getEntityByRef` instead, which accepts both string and compound ref forms. diff --git a/.changeset/search-blankly-have-a-nice-life.md b/.changeset/search-blankly-have-a-nice-life.md new file mode 100644 index 0000000000..63dc6051bd --- /dev/null +++ b/.changeset/search-blankly-have-a-nice-life.md @@ -0,0 +1,18 @@ +--- +'@backstage/plugin-search-backend-node': minor +'@backstage/search-common': minor +--- + +**BREAKING** + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +The concepts of `Collator` and `Decorator` have been replaced with readable and +transform object streams (respectively), as well as factory classes to +instantiate them. Accordingly, the `SearchEngine.index()` method has also been +replaced with a `getIndexer()` factory method that resolves to a writable +object stream. + +Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-done-me-no-favor.md b/.changeset/search-done-me-no-favor.md new file mode 100644 index 0000000000..0b4e24c4a8 --- /dev/null +++ b/.changeset/search-done-me-no-favor.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-pg': minor +--- + +**BREAKING** + +The `PgSearchEngine` implements the new stream-based indexing process expected +by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-just-smile-politely.md b/.changeset/search-just-smile-politely.md new file mode 100644 index 0000000000..a7831155da --- /dev/null +++ b/.changeset/search-just-smile-politely.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-techdocs-backend': patch +--- + +A `DefaultTechDocsCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultTechDocsCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for necessary changes to your search backend plugin configuration. diff --git a/.changeset/search-like-a-bank-teller.md b/.changeset/search-like-a-bank-teller.md new file mode 100644 index 0000000000..a9b0330eb2 --- /dev/null +++ b/.changeset/search-like-a-bank-teller.md @@ -0,0 +1,43 @@ +--- +'@backstage/create-app': patch +--- + +The Backstage Search Platform's indexing process has been rewritten as a stream +pipeline in order to improve efficiency and performance on large document sets. + +To take advantage of this, upgrade to the latest version of +`@backstage/plugin-search-backend-node`, as well as any backend plugins whose +collators you are using. Then, make the following changes to your +`/packages/backend/src/plugins/search.ts` file: + +```diff +-import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +-import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; ++import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; ++import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + +// ... + + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultCatalogCollator.fromConfig(config, { discovery }), ++ factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, +- collator: DefaultTechDocsCollator.fromConfig(config, { ++ factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); +``` + +If you've written custom collators, decorators, or search engines in your +Backstage backend instance, you will need to re-implement them as readable, +transform, and writable streams respectively (including factory classes for +instantiating them). [A how-to guide for refactoring](https://backstage.io/docs/features/search/how-to-guides#rewriting-alpha-style-collators-for-beta) +existing implementations is available. diff --git a/.changeset/search-selfless-cold-composed.md b/.changeset/search-selfless-cold-composed.md new file mode 100644 index 0000000000..7e7dafd421 --- /dev/null +++ b/.changeset/search-selfless-cold-composed.md @@ -0,0 +1,12 @@ +--- +'@backstage/plugin-search-backend-module-elasticsearch': minor +--- + +**BREAKING** + +The `ElasticSearchSearchEngine` implements the new stream-based indexing +process expected by the latest `@backstage/search-backend-node`. + +When updating to this version, you must also update to the latest version of +`@backstage/search-backend-node`. Check [this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for further details. diff --git a/.changeset/search-throw-me-a-right.md b/.changeset/search-throw-me-a-right.md new file mode 100644 index 0000000000..32f4e82fe4 --- /dev/null +++ b/.changeset/search-throw-me-a-right.md @@ -0,0 +1,13 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +A `DefaultCatalogCollatorFactory`, which works with the new stream-based +search indexing subsystem, is now available. The `DefaultCatalogCollator` will +continue to be available for those unable to upgrade to the stream-based +`@backstage/search-backend-node` (and related packages), however it is now +marked as deprecated and will be removed in a future version. + +To upgrade this plugin and the search indexing subsystem in one go, check +[this upgrade guide](https://backstage.io/docs/features/search/how-to-guides#how-to-migrate-from-search-alpha-to-beta) +for necessary changes to your search backend plugin configuration. diff --git a/.changeset/serious-mayflies-thank.md b/.changeset/serious-mayflies-thank.md new file mode 100644 index 0000000000..b3c5e31527 --- /dev/null +++ b/.changeset/serious-mayflies-thank.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +**Breaking**: Removed optional `handleError()` from `CatalogProcessor`. This optional method is never called by the catalog processing engine and can therefore be removed. diff --git a/.changeset/silver-boxes-flash.md b/.changeset/silver-boxes-flash.md new file mode 100644 index 0000000000..9f30c1a34b --- /dev/null +++ b/.changeset/silver-boxes-flash.md @@ -0,0 +1,9 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +**BREAKING**: Moved **DefaultStarredEntitiesApi** to `@backstage/plugin-catalog`. If you were using this in tests, you can use the new `MockStarredEntitiesApi` from `@backstage/plugin-catalog-react` instead. + +Fixed a risky behavior where `DefaultStarredEntitiesApi` forwarded values to observers that were later mutated. + +Removed the `isStarred` method from `DefaultStarredEntitiesApi`, as it is not part of the `StarredEntitiesApi`. diff --git a/.changeset/small-brooms-retire.md b/.changeset/small-brooms-retire.md new file mode 100644 index 0000000000..2aa722a666 --- /dev/null +++ b/.changeset/small-brooms-retire.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-techdocs': patch +--- + +Removed usage of deprecated favorite utility methods. diff --git a/.changeset/small-hornets-dress.md b/.changeset/small-hornets-dress.md new file mode 100644 index 0000000000..474c3011bd --- /dev/null +++ b/.changeset/small-hornets-dress.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-common': patch +--- + +**DEPRECATION**: Moved the `CatalogEntityDocument` to `@backstage/plugin-catalog-common` and deprecated the export from `@backstage/plugin-catalog-backend`. + +A new `type` field has also been added to `CatalogEntityDocument` as a replacement for `componentType`, which is now deprecated. Both fields are still present and should be set to the same value in order to avoid issues with indexing. + +Any search customizations need to be updated to use this new `type` field instead, including any custom frontend filters, custom frontend result components, custom search decorators, or non-default Catalog collator implementations. diff --git a/.changeset/sour-eggs-kick.md b/.changeset/sour-eggs-kick.md new file mode 100644 index 0000000000..f796b173fb --- /dev/null +++ b/.changeset/sour-eggs-kick.md @@ -0,0 +1,10 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-react': patch +--- + +Added the following deprecations to the `catalog-react` package: + +- **DEPRECATION**: `useEntity` will now warn if the entity has not yet been loaded, and will soon throw errors instead. If you're using the default implementation of `EntityLayout` and `EntitySwitch` then these components will ensure that there is an entity loaded before rendering children. If you're implementing your own `EntityLayout` or `EntitySwitch` or something that operates outside or adjacent to them, then use `useAsyncEntity`. + +- **DEPRECATION**: the `loading`, `error` and `refresh` properties that are returned from `useEntity` have been deprecated, and are available on `useAsyncEntity` instead. diff --git a/.changeset/spicy-onions-kiss.md b/.changeset/spicy-onions-kiss.md new file mode 100644 index 0000000000..777e233b88 --- /dev/null +++ b/.changeset/spicy-onions-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +- Replaced usage of the deprecated and now removed `rootRoute` and `catalogRouteRef`s from the `catalog-react` package diff --git a/.changeset/strong-suns-hope.md b/.changeset/strong-suns-hope.md new file mode 100644 index 0000000000..876f5f1618 --- /dev/null +++ b/.changeset/strong-suns-hope.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': minor +--- + +Removing the `EntityName` path for the `useEntityOwnership` as it has never worked correctly. Please pass in an entire `Entity` instead. diff --git a/.changeset/swift-roses-hug.md b/.changeset/swift-roses-hug.md new file mode 100644 index 0000000000..0e6d76ed87 --- /dev/null +++ b/.changeset/swift-roses-hug.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +**BREAKING**: Removing the exports of the raw components that back the `CustomFieldExtensions`. diff --git a/.changeset/tame-lions-know.md b/.changeset/tame-lions-know.md new file mode 100644 index 0000000000..7f304b22ef --- /dev/null +++ b/.changeset/tame-lions-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-app-api': minor +--- + +**BREAKING**: Removed the deprecated `GithubAuth.normalizeScopes` method. diff --git a/.changeset/tasty-poems-raise.md b/.changeset/tasty-poems-raise.md new file mode 100644 index 0000000000..fbd81423b6 --- /dev/null +++ b/.changeset/tasty-poems-raise.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': minor +--- + +**BREAKING**: The `TokenFactory.issueToken` used by custom sign-in resolvers now ensures that the sub claim given is a full entity reference of the format `:/`. Any existing custom sign-in resolver functions that do not supply a full entity reference must be updated. diff --git a/.changeset/tidy-hairs-sip.md b/.changeset/tidy-hairs-sip.md new file mode 100644 index 0000000000..512447236d --- /dev/null +++ b/.changeset/tidy-hairs-sip.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend-module-cookiecutter': patch +--- + +Fixed bug where existing cookiecutter.json file is not used. diff --git a/.changeset/tidy-jokes-dream.md b/.changeset/tidy-jokes-dream.md new file mode 100644 index 0000000000..aa9c84fc4b --- /dev/null +++ b/.changeset/tidy-jokes-dream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +add `userExpand` config option to allow expanding a single relationship diff --git a/.changeset/tiny-jobs-hunt.md b/.changeset/tiny-jobs-hunt.md new file mode 100644 index 0000000000..8cf9db20b8 --- /dev/null +++ b/.changeset/tiny-jobs-hunt.md @@ -0,0 +1,5 @@ +--- +'@backstage/techdocs-common': patch +--- + +Modify techdocs builder to automatically append techdocs-core plugin to mkdocs.yaml file if it is missing. Adds an optional configuration item if this plugin needs to be omitted. diff --git a/.changeset/tricky-students-promise.md b/.changeset/tricky-students-promise.md new file mode 100644 index 0000000000..b9c6150a4a --- /dev/null +++ b/.changeset/tricky-students-promise.md @@ -0,0 +1,22 @@ +--- +'@backstage/catalog-client': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-graph': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-catalog-react': patch +'@backstage/plugin-code-coverage': patch +'@backstage/plugin-code-coverage-backend': patch +'@backstage/plugin-fossa': patch +'@backstage/plugin-jenkins': patch +'@backstage/plugin-jenkins-backend': patch +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-tech-insights': patch +'@backstage/plugin-tech-insights-backend': patch +'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs-backend': patch +'@backstage/plugin-todo-backend': patch +--- + +Use `CompoundEntityRef` instead of `EntityName`, and `getCompoundEntityRef` instead of `getEntityName`, from `@backstage/catalog-model`. diff --git a/.changeset/two-lobsters-hammer.md b/.changeset/two-lobsters-hammer.md new file mode 100644 index 0000000000..412901fb41 --- /dev/null +++ b/.changeset/two-lobsters-hammer.md @@ -0,0 +1,6 @@ +--- +'@backstage/core-components': patch +'@backstage/plugin-catalog-react': patch +--- + +Updated usage of `StorageApi` to use `snapshot` method instead of `get` diff --git a/.changeset/weak-news-reply.md b/.changeset/weak-news-reply.md new file mode 100644 index 0000000000..4a7a38cef5 --- /dev/null +++ b/.changeset/weak-news-reply.md @@ -0,0 +1,23 @@ +--- +'@backstage/catalog-model': minor +--- + +**BREAKING**: The following changes are all breaking changes. + +Removed `EDIT_URL_ANNOTATION` and `VIEW_URL_ANNOTATION`, `LOCATION_ANNOTATION`, `ORIGIN_LOCATION_ANNOTATION`, `LOCATION_ANNOTATION`, `SOURCE_LOCATION_ANNOTATION`. All of these constants have been prefixed with ANNOTATION to be easier to find meaning `SOURCE_LOCATION_ANNOTATION` is available as `ANNOTATION_SOURCE_LOCATION`. + +Removed `parseLocationReference`, replaced by `parseLocationRef`. + +Removed `stringifyLocationReference`, replaced by `stringifyLocationRef`. + +Removed `Location` type which has been moved to `catalog-client`. + +Removed `ENTITY_DEFAULT_NAMESPACE`, replaced by `DEFAULT_NAMESPACE`. + +Removed `compareEntityToRef` compare using `stringifyEntityRef` instead. + +Removed `JSONSchema` type which should be imported from `json-schema` package instead. + +Removed utility methods: `entityHasChanges`, `generateEntityEtag`, `generateEntityUid`, `generateUpdatedEntity`. + +Removed `ENTITY_META_GENERATED_FIELDS` and `EntityRefContext`. diff --git a/.changeset/wild-dolphins-lick.md b/.changeset/wild-dolphins-lick.md new file mode 100644 index 0000000000..cdd5827b00 --- /dev/null +++ b/.changeset/wild-dolphins-lick.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +**DEPRECATION**: Deprecated the `RefreshIntervalFunction` and `createRandomRefreshInterval` in favour of the `ProcessingIntervalFunction` and `createRandomProcessingInterval` type and method respectively. Please migrate to use the new names. + +**DEPRECATION**: Deprecated the `setRefreshInterval` and `setRefreshIntervalSeconds` methods on the `CatalogBuilder` for the new `setProcessingInterval` and `setProcessingIntervalSeconds` methods. Please migrate to use the new names. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 6359493d4b..c7971f6c52 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -20,7 +20,6 @@ Autoscaling autoselect Avro aws -backrub backported backporting Bigtable @@ -39,8 +38,6 @@ Changesets chanwit Chanwit ci -cisphobia -cissexist classname cli cloudbuild @@ -68,7 +65,6 @@ css Datadog dataflow dayjs -deadnaming debounce Debounce declaratively @@ -171,7 +167,6 @@ Minikube Minio misconfiguration misconfigured -misgendering mkdocs Mkdocs monorepo @@ -213,6 +208,7 @@ parallelization Patrik Peloton performant +Performant plantuml Platformize Podman diff --git a/README.md b/README.md index 1078631926..e78800ef84 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # [Backstage](https://backstage.io) +_During March 7 to March 11 the maintainers will be taking part in Spotify's annual hack week. Development will continue as usual, but expect a slower pace for discussions and PR reviews. Why not take this opportunity to [build a plugin](https://backstage.io/docs/plugins/)?_ + [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![CNCF Status](https://img.shields.io/badge/cncf%20status-sandbox-blue.svg)](https://www.cncf.io/projects) [![Main CI Build](https://github.com/backstage/backstage/workflows/Main%20Master%20Build/badge.svg)](https://github.com/backstage/backstage/actions?query=workflow%3A%22Main+Master+Build%22) diff --git a/contrib/search/README.md b/contrib/search/README.md new file mode 100644 index 0000000000..9a43a05c4c --- /dev/null +++ b/contrib/search/README.md @@ -0,0 +1,3 @@ +# Search + +Contributions/extensions to the Search plugin diff --git a/contrib/search/confluence/ConfluenceCollator.md b/contrib/search/confluence/ConfluenceCollator.md new file mode 100644 index 0000000000..f636ae88ea --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator.md @@ -0,0 +1,95 @@ +ConfluenceCollator.ts reference + +```ts +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch'; + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + const ConfluenceUrlBase = + 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api'; + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {}; + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + Authorization: `Basic ${process.env.CONFLUENCE_TOKEN}`, + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson; + } + + async function getSpaces(): Promise { + const data = await getConfluenceData( + `${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`, + ); + let spacesList = []; + if (data['results']) { + const results = data['results']; + for (const result of results) { + spacesList.push(result['key']); + } + } + return spacesList; + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = []; + for (var space of spaces) { + let next = true; + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}`; + while (next) { + const data = await getConfluenceData(requestUrl); + if (data['results']) { + const results = data['results']; + for (const result of results) { + documentsList.push(result['_links']['self']); + } + if (data['_links']['next']) { + requestUrl = data['_links']['base'] + data['_links']['next']; + } else { + next = false; + } + } else { + break; + } + } + } + return documentsList; + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = []; + for (var documentUrl of documents) { + const data = await getConfluenceData( + documentUrl + '?expand=body.storage', + ); + if (data['status'] && data['status'] == 'current') { + const documentMetaData = { + title: data['title'], + text: data['body']['storage']['value'], + location: data['_links']['base'] + data['_links']['webui'], + }; + documentInfo.push(documentMetaData); + } + } + return documentInfo; + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList; + } +} +``` diff --git a/contrib/search/confluence/ConfluenceResultListItem.md b/contrib/search/confluence/ConfluenceResultListItem.md new file mode 100644 index 0000000000..ba7c4f91a9 --- /dev/null +++ b/contrib/search/confluence/ConfluenceResultListItem.md @@ -0,0 +1,59 @@ +ConfluenceResultListItem.tsx reference + +```tsx +import React from 'react'; +import { Link } from '@backstage/core-components'; +import { IndexableDocument } from '@backstage/search-common'; +import { + Divider, + ListItem, + ListItemIcon, + ListItemText, +} from '@material-ui/core'; + +type Props = { + result: IndexableDocument; +}; + +export const ConfluenceResultListItem = ({ result }: Props) => { + // Remove html tags from document text before displaying + const chars = []; + let isTag = false; + for (const c of result.text.substring(0, 500)) { + if (c === '<') { + isTag = true; + continue; + } + if (c === '>') { + isTag = false; + chars.push(' '); + continue; + } + if (!isTag) { + chars.push(c); + } + } + const excerpt = + chars.join('').substring(0, 80) + (result.text.length > 80 ? '...' : ''); + + return ( + + + + + + + + + + ); +}; +``` diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md new file mode 100644 index 0000000000..80beb9c169 --- /dev/null +++ b/contrib/search/confluence/README.md @@ -0,0 +1,32 @@ +# Confluence + +These files help you add Confluence as a source to the Backstage Search plugin. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app. +Then, add the following code to your packages/app/src/components/search/SearchPage.tsx: + +```tsx +import { ConfluenceResultListItem } from './ConfluenceResultListItem'; +``` + +```tsx +case 'confluence': + return ( + + ); +``` + +and the following to packages/backend/src/plugins/search.ts: + +```ts +import { ConfluenceCollator } from './search/ConfluenceCollator'; +``` + +```ts +indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + collator: new ConfluenceCollator(), +}); +``` diff --git a/docs/auth/identity-resolver.md b/docs/auth/identity-resolver.md index a33351e938..3b355470d1 100644 --- a/docs/auth/identity-resolver.md +++ b/docs/auth/identity-resolver.md @@ -27,6 +27,8 @@ sign-in resolvers and set them for any of the Authentication providers inside `@backstage/plugin-auth-backend` plugin. ```ts +import { DEFAULT_NAMESPACE, stringifyEntityRef } from '@backstage/catalog-model'; + export default async function createPlugin({ ... }: PluginEnvironment): Promise { @@ -38,22 +40,31 @@ export default async function createPlugin({ resolver: async ({ profile: { email } }, ctx) => { // Call a custom validator function that checks that the email is // valid and on our own company's domain, and throws an Error if it - // isn't + // isn't. + // TODO: Implement this function validateEmail(email); // List of entity references that denote the identity and // membership of the user - const ent = []; + const ent: string[] = []; // Let's use the username in the email ID as the user's default // unique identifier inside Backstage. const [id] = email.split('@'); - ent.push(`User:default/${id}`) + ent.push(stringifyEntityRef({ + kind: 'User', + namespace: DEFAULT_NAMESPACE, + name: id, + })); // Let's call the internal LDAP provider to get a list of groups // that the user belongs to, and add those to the list as well const ldapGroups = await getLdapGroups(email); - ldapGroups.forEach(group => ent.push(`Group:default/${group}`)) + ldapGroups.forEach(group => ent.push(stringifyEntityRef({ + kind: 'Group', + namespace: DEFAULT_NAMESPACE, + name: group, + }))); // Issue the token containing the entity claims const token = await ctx.tokenIssuer.issueToken({ diff --git a/docs/features/search/concepts.md b/docs/features/search/concepts.md index 8cb17b12e6..052f71376f 100644 --- a/docs/features/search/concepts.md +++ b/docs/features/search/concepts.md @@ -54,13 +54,14 @@ An index is a collection of such documents of a given type. ### Collators You need to be able to search something! Collators are the way to define what -can be searched. Specifically, they're classes which return documents conforming -to a minimum set of fields (including a document title, location, and text), but -which can contain any other fields as defined by the collator itself. One -collator is responsible for defining and collecting documents of a type. +can be searched. Specifically, they're readable object streams of documents that +conform to a minimum set of fields (including a document title, location, and +text), but which can contain any other fields as defined by the collator itself. +One collator is responsible for defining and collecting documents of a type. -Some plugins, like the Catalog Backend, provide so-called "default" collators -which you can use out-of-the-box to start searching across Backstage quickly. +Some plugins, like the Catalog Backend, provide so-called "default" collator +factories which you can use out-of-the-box to start searching across Backstage +quickly. ### Decorators @@ -68,9 +69,15 @@ Sometimes you want to add extra information to a set of documents in your search index that the collator may not be aware of. For example, the Software Catalog knows about software entities, but it may not know about their usage or quality. -Decorators are classes which can add extra fields to pre-collated documents. -This extra metadata could then be used to bias search results or otherwise -improve the search experience in your Backstage instance. +Decorators are transform streams which sit between a collator (read stream) and +an indexer (write stream) during the indexing process. It can be used to add +extra fields to documents as they are being collated and indexed. This extra +metadata could then be used to bias search results or otherwise improve the +search experience in your Backstage instance. + +In addition to adding extra metadata, decorators (like any transform stream) can +also be used to remove metadata, filter out, or even add extra documents at +index-time. ### The Scheduler diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index a3c42c682c..ac36d64673 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -48,10 +48,10 @@ const app = createApp({ ## How to index TechDocs documents The TechDocs plugin has supported integrations to Search, meaning that it -provides a default collator ready to be used. +provides a default collator factory ready to be used. The purpose of this guide is to walk you through how to register the -[DefaultTechDocsCollator](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts) +[DefaultTechDocsCollatorFactory](https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts) in your App, so that you can get TechDocs documents indexed. If you have been through the @@ -60,18 +60,19 @@ you should have the `packages/backend/src/plugins/search.ts` file available. If so, you can go ahead and follow this guide - if not, start by going through the getting started guide. -1. Import the DefaultTechDocsCollator from `@backstage/plugin-techdocs-backend`. +1. Import the `DefaultTechDocsCollatorFactory` from + `@backstage/plugin-techdocs-backend`. ```typescript -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; ``` -2. Register the DefaultTechDocsCollator with the IndexBuilder. +2. Register the `DefaultTechDocsCollatorFactory` with the IndexBuilder. ```typescript indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, @@ -131,3 +132,264 @@ indexBuilder.addCollator({ As shown above, you can add a catalog entity filter to narrow down what catalog entities are indexed by the search engine. + +## How to migrate from Search Alpha to Beta + +For the purposes of this guide, Search Beta version is defined as: + +- **Search Plugin**: At least `v0.x.y` +- **Search Backend Plugin**: At least `v0.x.y` +- **Search Backend Node**: At least `v0.x.y` + +In the Beta version, the Search Platform's indexing process has been rewritten +as a stream pipeline in order to improve efficiency and performance on large +sets of documents. + +If you've not yet extended the Search Platform with custom code, and have +instead taken advantage of default collators, decorators, and search engines +provided by existing plugins, the migration process is fairly straightforward: + +1. Upgrade to at least version `0.x.y` of + `@backstage/plugin-search-backend-node`, as well as any backend plugins whose + collators you are using (e.g. at least version `0.x.y` of + `@backstage/plugin-catalog-backend` and/or version `0.x.y` of + `@backstage/plugin-techdocs-backend`). +2. Then, make the following changes to your + `/packages/backend/src/plugins/search.ts` file: + + ```diff + -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; + -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; + +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; + +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; + // ... + const indexBuilder = new IndexBuilder({ logger, searchEngine }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultCatalogCollator.fromConfig(config, { discovery }), + + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery }), + }); + indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + - collator: DefaultTechDocsCollator.fromConfig(config, { + + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery, + logger, + }), + }); + ``` + +Any custom collators, decorators, or search engine implementations will require +minor refactoring. Continue on for details. + +### Rewriting alpha-style collators for beta + +In alpha versions of the Backstage Search Platform, collators were classes that +implemented an `execute` method which resolved an `IndexableDocument` array. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an [object-mode][obj-mode] `Readable` stream where each +object pushed onto the stream is of type `IndexableDocument`. Instances of this +stream are instantiated by a factory class conforming to the +`DocumentCollatorFactory` interface. + +The optimal conversion strategy will vary depending on the collator's logic, but +the simplest conversion can follow a process like this: + +1. Rename your collator class to something like `YourCollatorFactory` and update + it to implement `DocumentCollatorFactory` instead of `DocumentCollator`. +2. Update its `execute` method so that it resolves + `AsyncGenerator` instead of `YourIndexableDocument[]`. +3. Implement `DocumentCollatorFactory`'s `getCollator` method which resolves to + `Readable.from(this.execute())` (which is a utility for creating [readable + streams][read-stream] from [async generators][async-gen]). + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { Readable } from 'stream'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async *execute(): AsyncGenerator { + const widgets = await this.client.getWidgets(); + for (const widget of widgets) { + yield { + title: widget.name, + location: widget.url, + text: widget.description, + }; + } + } + getCollator() { + return Readable.from(this.execute()); + } +} +``` + +Note: it may be possible to simplify your collator dramatically! If your custom +collator was previously using streams under the hood (for example, by reading +newline delimited JSON from a local or remote file), you could just expose the +stream directly via a simple factory class: + +```ts +import { DocumentCollatorFactory } from '@backstage/plugin-search-backend-node'; +import { createReadStream } from 'fs'; +import { parse } from '@jsonlines/core'; +export class YourCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'your-type'; + async getCollator() { + const parseStream = parse(); + return createReadStream('./documents.ndjson').pipe(parseStream); + } +} +``` + +### Rewriting alpha-style decorators for beta + +In alpha versions of the Backstage Search Platform, decorators were classes that +implemented an `execute` method which took an `IndexableDocument` array as an +argument, and resolved a modified array of the same type. + +In beta versions, the logic encapsulated by the aforementioned `execute` method +is contained within an object-mode `Transform` stream which reads objects of +type `IndexableDocument`, and writes objects of a conforming type. Similar to +collators, instances of this stream are instantiated by a factory class +conforming to the `DocumentDecoratorFactory` interface. + +Although you can choose to implement a `Transform` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a `DecoratorBase` class +in order to simplify the developer experience. With this base class, all that's +needed is to transfer your old decorator class logic into the base class' three +methods (`initialize`, `decorate`, and `finalize`), and implement the factory +class that instantiates the stream: + +```ts +import { DecoratorBase } from '@backstage/plugin-search-backend-node'; +export class YourDecorator extends DecoratorBase { + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async decorate( + document: YourIndexableDocument, + ): Promise { + // Perform transformation logic here. + return document; + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator() { + return new YourDecorator(); + } +} +``` + +Note the return type of the `decorate` method and how each can be used to +different effect. + +- By resolving a single `YourIndexableDocument` object, your decorator can be + used to make simple transformations: + + ```ts + class BooleanWidgetCoolnessDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple, 1:1 transformation. + widget.isCool = widget.isCool === 'true' ? true : false; + return widget; + } + } + ``` + +- By resolving `undefined`, your decorator can filter out documents which + shouldn't be in the index: + + ```ts + class OnlyCoolWidgetsDecorator extends DecoratorBase { + async decorator(widget) { + // Perform a simple filter operation. + return widget.isCool ? widget : undefined; + } + } + ``` + +- By resolving an array of `YourIndexableDocument` objects, you can generate + multiple documents based on the content of one: + + ```ts + class WidgetByVariantDecorator extends DecoratorBase { + async decorator(widget) { + // Generate one widget doc per widget variant. + return widget.variants.map(variant => { + // Each widget doc is the given widget plus a "variant" property + // pulled from a widget.variants string array. + return { + ...widget, + variant, + }; + }); + } + } + ``` + +In alpha versions, a decorator had access to every `IndexableDocument` +simultaneously. This is no longer possible in beta versions (precisely to make +the indexing process more efficient and performant). You will need to modify +your decorator's logic so that it does not need access to every document at +once. + +### Rewriting alpha-style search engines for beta + +Search Engines are responsible for both querying and indexing documents to an +underlying search engine technology. While the search engine query interface +didn't change between alpha and beta versions, the indexing half of the +interface _did_ change. + +In alpha versions of the Backstage Search Platform, a search engine implemented +an `index` method which took a `type` and an `IndexableDocument` array and was +responsible for writing these documents to the underlying search engine. + +In beta versions, the logic encapsulated by the aforementioned `index` method is +contained within an object-mode `Writable` stream which expects objects of type +`IndexableDocument`. On the search engine class itself, the `index` method is +replaced with a `getIndexer` factory method which still takes the `type`, but +resolves an instance of the aforementioned `Writable` stream. + +Although you can choose to implement a `Writable` stream from scratch, the +`@backstage/plugin-search-backend-node` package provides a +`BatchSearchEngineIndexer` class in order to simplify the developer experience. +With this base class, which collects documents in batches of a configurable size +on your behalf, all that's needed is to transfer your old `index` method logic +into the base class' three methods (`initialize`, `index`, and `finalize`), and +implement the factory method that instantiates the stream: + +```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { SearchEngine } from '@backstage/search-common'; +export class YourSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor({ type }: { type: string }) { + // Customize the number of documents passed to the index method per batch. + super({ batchSize: 500 }); + // An imaginary search engine indexing client. + this.index = new SomeSearchEngineIndex({ indexName: type }); + } + async initialize() { + // Setup logic. Performed once before any documents are consumed. + } + async index(documents: IndexableDocument[]) { + await this.index.batchOf(documents); + } + async finalize() { + // Teardown logic. Performed once after all documents have been consumed. + } +} +export class YourSearchEngine implements SearchEngine { + async getIndexer(type: string) { + return new YourSearchEngineIndexer({ type }); + } +} +``` + +[obj-mode]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_object_mode +[read-stream]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_readable_streams +[async-gen]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators diff --git a/docs/features/software-catalog/external-integrations.md b/docs/features/software-catalog/external-integrations.md index 59caf9475c..9f592e7cca 100644 --- a/docs/features/software-catalog/external-integrations.md +++ b/docs/features/software-catalog/external-integrations.md @@ -57,7 +57,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/providers/types.ts) +[`EntityProvider`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/provider.ts) subclass that can be added to this catalog builder. Let's make a simple provider that can refresh a set of entities based on a @@ -355,7 +355,7 @@ The recommended way of instantiating the catalog backend classes is to use the `CatalogBuilder`, as illustrated in the [example backend here](https://github.com/backstage/backstage/blob/master/packages/backend/src/plugins/catalog.ts). We will create a new -[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/ingestion/processors/types.ts) +[`CatalogProcessor`](https://github.com/backstage/backstage/blob/master/plugins/catalog-backend/src/api/processor.ts) subclass that can be added to this catalog builder. It is up to you where you put the code for this new processor class. For quick diff --git a/docs/features/techdocs/cli.md b/docs/features/techdocs/cli.md index a5dc8c3c54..2a11e1181a 100644 --- a/docs/features/techdocs/cli.md +++ b/docs/features/techdocs/cli.md @@ -130,6 +130,8 @@ Options: if not found. --etag A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json. + --omitTechdocsCoreMkdocsPlugin An option to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. -v --verbose Enable verbose output. (default: false) -h, --help display help for command ``` diff --git a/docs/features/techdocs/concepts.md b/docs/features/techdocs/concepts.md index 408f092ebd..e34f58bc70 100644 --- a/docs/features/techdocs/concepts.md +++ b/docs/features/techdocs/concepts.md @@ -46,6 +46,24 @@ between `techdocs-backend` and the storage) [TechDocs Backend](https://github.com/backstage/backstage/tree/master/plugins/techdocs-backend) +## TechDocs Build Strategy + +To accommodate more complex logic surrounding whether or not to build TechDocs, the TechDocs backend +supports selecting a Build Strategy. +The Build Strategy is responsible for deciding whether the documentation requested should be built locally +by the TechDocs backend or not. +Customization of the Build Strategy allows for more complex behaviour regarding whether the TechDocs backend +is responsible for building TechDocs, whether an external process is responsible, or whether a combination +of local builds and an external process is responsible, on an entity-by-entity basis. + +The default Build Strategy results in the TechDocs backend building documentation locally if the +`techdocs.builder` configuration option is set to `'local'`, and skipping any building otherwise. +However any logic that satisfies the Build Strategy interface can be implemented, using the Backstage +config as well as the entity being processed to make a decision. + +For an example of how the Build Strategy can be used to implement a 'hybrid' build model, refer to +the [How to implement a hybrid build strategy](./how-to-guides.md#how-to-implement-a-hybrid-build-strategy) guide. + ## TechDocs Container The TechDocs container is a Docker container available at diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 6317ae365b..3edc55574f 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -37,12 +37,22 @@ techdocs: pullImage: true + mkdocs: + # (Optional) techdocs.generator.omitTechdocsCoreMkdocsPlugin can be used to disable automatic addition of techdocs-core plugin to the mkdocs.yaml files. + # Defaults to false, which means that the techdocs-core plugin is always added to the mkdocs file. + omitTechdocsCorePlugin: false + # techdocs.builder can be either 'local' or 'external. - # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage - # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. - # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to generate and publish. In this case of 'external', - # we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). This is the "Recommended" setup of - # the architecture. Read more here https://backstage.io/docs/features/techdocs/architecture + # Using the default build strategy, if builder is set to 'local' and you open a TechDocs page, + # techdocs-backend will try to generate the docs, publish to storage and show the generated docs afterwords. + # This is the "Basic" setup of the TechDocs Architecture. + # Using the default build strategy, if builder is set to 'external' (or anything other than 'local'), techdocs-backend + # will only fetch the docs and will NOT try to generate and publish. + # In this case, we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). + # This is the "Recommended" setup of the architecture. + # Note that custom build strategies may alter this behaviour. + # Read more about the "Basic" and "Recommended" setups here https://backstage.io/docs/features/techdocs/architecture + # Read more about build strategies here: https://backstage.io/docs/features/techdocs/concepts#techdocs-build-strategy builder: 'local' diff --git a/docs/features/techdocs/creating-and-publishing.md b/docs/features/techdocs/creating-and-publishing.md index 383c3ae6d2..f63ff43b7f 100644 --- a/docs/features/techdocs/creating-and-publishing.md +++ b/docs/features/techdocs/creating-and-publishing.md @@ -79,6 +79,9 @@ plugins: - techdocs-core ``` +> Note - The plugins section above is optional. Backstage automatically adds the `techdocs-core` plugin to the +> mkdocs file if it is missing. This functionality can be turned off with a [configuration option](./configuration.md) in Backstage. + Update your component's entity description by adding the following lines to its `catalog-info.yaml` in the root of its repository: diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index 863b4b3ecb..56ca2252d0 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -538,3 +538,58 @@ Done! Now you have a support of the following diagrams along with mermaid: - `Vega` - `Vega-Lite` - `WaveDrom` + +## How to implement a hybrid build strategy + +One limitation of the [Recommended deployment](./architecture.md#recommended-deployment) is that +the experience for users requires modifying their CI/CD process to publish +their TechDocs. For some users, this may be unnecessary, and provides a barrier +to entry for onboarding users to Backstage. However, a purely local TechDocs +build restricts TechDocs creators to using the tooling provided in Backstage, +as well as the plugins and features provided in the Backstage-included `mkdocs` +installation. + +To accommodate both of these use-cases, users can implement a custom [Build Strategy](./concepts.md#techdocs-build-strategy) +with logic to encode which TechDocs should be built locally, and which will be +built externally. + +To achieve this hybrid build model: + +1. In your Backstage instance's `app-config.yaml`, set `techdocs.builder` to + `'local'`. This ensures that Backstage will build docs for users who want the + 'out-of-the-box' experience. +2. Configure external storage of TechDocs as normal for a production deployment. + This allows Backstage to publish documentation to your storage, as well as + allowing other users to publish documentation from their CI/CD pipelines. +3. Create a custom build strategy, that implements the `DocsBuildStrategy` interface, + and which implements your custom logic for determining whether to build docs for + a given entity. + For example, to only build docs when an entity has the `company.com/techdocs-builder` + annotation set to `'local'`: + + ```typescript + export class AnnotationBasedBuildStrategy { + private readonly config: Config; + + constructor(config: Config) { + this.config = config; + } + + async shouldBuild(_: Entity): Promise { + return ( + this.entity.metadata?.annotations?.['company.com/techdocs-builder'] === + 'local' + ); + } + } + ``` + +4. Pass an instance of this Build Strategy as the `docsBuildStrategy` parameter of the + TechDocs backend `createRouter` method. + +Users should now be able to choose to have their documentation built and published by +the TechDocs backend by adding the `company.com/techdocs-builder` annotation to their +entity. If the value of this annotation is `'local'`, the TechDocs backend will build +and publish the documentation for them. If the value of the `company.com/techdocs-builder` +annotation is anything other than `'local'`, the user is responsible for publishing +documentation to the appropriate location in the TechDocs external storage. diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index 75f0606752..f57efc1255 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -552,3 +552,36 @@ The following is an excerpt of a typical setup of an isomorphic library package: }, "files": ["dist"], ``` + +## Experimental Type Build + +The Backstage CLI has an experimental feature where multiple different type definition files can be generated for different release stages. The release stages are marked in the [TSDoc](https://tsdoc.org/) for each individual export, using either `@public`, `@alpha`, or `@beta`. Rather than just building a single `index.d.ts` file, the build process will instead output `index.d.ts`, `index.beta.d.ts`, and `index.alpha.d.ts`. Each of these files will have exports from more unstable release stages stripped, meaning that `index.d.ts` will omit all exports marked with `@alpha` or `@beta`, while `index.beta.d.ts` will omit all exports marked with `@alpha`. + +This feature is aimed at projects that publish to package registries and wish to maintain different levels of API stability within each package. There is no need to use this within a single monorepo, as it has no effect due to only applying to built and published packages. + +In order for the experimental type build to work, `@microsoft/api-extractor` must be installed in your project, as it is an optional peer dependency of the Backstage CLI. There are then three steps that need to be taken for each package where you want to enable this feature: + +- Add the `--experimental-type-build` flag to the `"build"` script of the package. +- Add either one or both of `"alphaTypes"` and `"betaTypes"` to the `"publishConfig"` of the package: + ```json + "publishConfig": { + ... + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts", + "betaTypes": "dist/index.beta.d.ts" + }, + ``` +- Add either one or both of `"alpha"` and `"beta"` to the `"files"` of the package: + ```json + "files": [ + "dist", + "alpha", + "beta" + ] + ``` + +Once this setup is complete, users of the published packages will only be able to access the stable API via the main package entry point, for example `@acme/my-plugin`. Exports marked with `@alpha` or `@beta` will only be available via the `/alpha` entry point, for example `@acme/my-plugin/alpha`, and exports marked with `@beta` will only be available via `/beta`. This does not apply within the monorepo that contains the package. There all exports still have to be imported via the main entry point. + +Note that these different entry points are only separated during type checking. At runtime they all share the same code which contains the exports from all releases stages. + +An example of this setup can be seen in the [`@backstage/catalog-model`](https://github.com/backstage/backstage/blob/da0675bf9f28ed1460f03635a22d3c26abd14707/packages/catalog-model/package.json#L14) package, which has enabled `alpha` type exports. With this setup, exports marked as `@alpha` are only available for import via `@backstage/catalog-model/alpha`. The `@backstage/catalog-model` package currently does not have any exports marked as `@beta`, or a `/beta` entry point. diff --git a/docs/overview/architecture-overview.md b/docs/overview/architecture-overview.md index 358dc53852..5b6b81cf10 100644 --- a/docs/overview/architecture-overview.md +++ b/docs/overview/architecture-overview.md @@ -310,7 +310,7 @@ backend: backend: cache: store: redis - connection: user:pass@cache.example.com:6379 + connection: redis://user:pass@cache.example.com:6379 ``` Contributions supporting other cache stores are welcome! diff --git a/docs/tutorials/switching-sqlite-postgres.md b/docs/tutorials/switching-sqlite-postgres.md index fafd2ad586..8a7f90b21f 100644 --- a/docs/tutorials/switching-sqlite-postgres.md +++ b/docs/tutorials/switching-sqlite-postgres.md @@ -43,9 +43,11 @@ backend: + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + # https://node-postgres.com/features/ssl -+ #ssl: require # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) -+ #ca: # if you have a CA file and want to verify it you can uncomment this section -+ #$file: /ca/server.crt ++ # you can set the sslmode configuration option via the `PGSSLMODE` environment variable ++ # see https://www.postgresql.org/docs/current/libpq-ssl.html Table 33.1. SSL Mode Descriptions (e.g. require) ++ # ssl: ++ # ca: # if you have a CA file and want to verify it you can uncomment this section ++ # $file: /ca/server.crt ``` If you have an `app-config.local.yaml` for local development, a similar update diff --git a/package.json b/package.json index e7cb42c1c6..1882d400b4 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,7 @@ ] }, "resolutions": { - "**/@graphql-codegen/cli/**/ws": "^7.4.6", - "**/@roadiehq/**/@backstage/plugin-catalog": "*", - "**/@roadiehq/**/@backstage/catalog-model": "*" + "**/@graphql-codegen/cli/**/ws": "^7.4.6" }, "version": "0.69.0", "dependencies": { diff --git a/packages/app-defaults/src/defaults/apis.ts b/packages/app-defaults/src/defaults/apis.ts index f26c298fb1..3f5cfc1c58 100644 --- a/packages/app-defaults/src/defaults/apis.ts +++ b/packages/app-defaults/src/defaults/apis.ts @@ -21,16 +21,13 @@ import { ErrorAlerter, GoogleAuth, GithubAuth, - OAuth2, OktaAuth, GitlabAuth, - Auth0Auth, MicrosoftAuth, BitbucketAuth, OAuthRequestManager, WebStorage, UrlPatternDiscovery, - SamlAuth, OneLoginAuth, UnhandledErrorForwarder, AtlassianAuth, @@ -49,16 +46,12 @@ import { oauthRequestApiRef, googleAuthApiRef, githubAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, gitlabAuthApiRef, - auth0AuthApiRef, microsoftAuthApiRef, storageApiRef, configApiRef, - samlAuthApiRef, oneloginAuthApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, atlassianAuthApiRef, } from '@backstage/core-plugin-api'; @@ -197,46 +190,6 @@ export const apis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: auth0AuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - Auth0Auth.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: oauth2ApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: samlAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, configApi }) => - SamlAuth.create({ - discoveryApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: oneloginAuthApiRef, deps: { @@ -251,25 +204,6 @@ export const apis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: oidcAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider: { - id: 'oidc', - title: 'Your Identity Provider', - icon: () => null, - }, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: bitbucketAuthApiRef, deps: { diff --git a/packages/app/src/components/catalog/EntityPage.test.tsx b/packages/app/src/components/catalog/EntityPage.test.tsx index b68b9e9895..68ce5fb497 100644 --- a/packages/app/src/components/catalog/EntityPage.test.tsx +++ b/packages/app/src/components/catalog/EntityPage.test.tsx @@ -16,15 +16,14 @@ import { EntityLayout } from '@backstage/plugin-catalog'; import { - DefaultStarredEntitiesApi, EntityProvider, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { githubActionsApiRef } from '@backstage/plugin-github-actions'; import { permissionApiRef } from '@backstage/plugin-permission-react'; import { MockPermissionApi, - MockStorageApi, renderInTestApp, TestApiProvider, } from '@backstage/test-utils'; @@ -59,12 +58,7 @@ describe('EntityPage Test', () => { diff --git a/packages/app/src/identityProviders.ts b/packages/app/src/identityProviders.ts index 49204d2b5e..5f8c0faf47 100644 --- a/packages/app/src/identityProviders.ts +++ b/packages/app/src/identityProviders.ts @@ -19,27 +19,12 @@ import { gitlabAuthApiRef, oktaAuthApiRef, githubAuthApiRef, - samlAuthApiRef, microsoftAuthApiRef, oneloginAuthApiRef, - oauth2ApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, } from '@backstage/core-plugin-api'; export const providers = [ - { - id: 'oidc-auth-provider', - title: 'Oidc', - message: 'Sign In using OpenId Connect', - apiRef: oidcAuthApiRef, - }, - { - id: 'oauth2-auth-provider', - title: 'OAuth 2.0', - message: 'Sign In using OAuth 2.0', - apiRef: oauth2ApiRef, - }, { id: 'google-auth-provider', title: 'Google', @@ -70,12 +55,6 @@ export const providers = [ message: 'Sign In using Okta', apiRef: oktaAuthApiRef, }, - { - id: 'saml-auth-provider', - title: 'SAML', - message: 'Sign In using SAML', - apiRef: samlAuthApiRef, - }, { id: 'onelogin-auth-provider', title: 'OneLogin', diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 9b963f3fc9..4ed9bb1514 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -234,6 +234,21 @@ export type ErrorHandlerOptions = { logClientErrors?: boolean; }; +// @public +export class FetchUrlReader implements UrlReader { + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree(): Promise; + // (undocumented) + readUrl(url: string, options?: ReadUrlOptions): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export type FromReadableArrayOptions = Array<{ data: Readable; diff --git a/packages/backend-common/config.d.ts b/packages/backend-common/config.d.ts index 57645d9fda..0c3e3132c0 100644 --- a/packages/backend-common/config.d.ts +++ b/packages/backend-common/config.d.ts @@ -137,7 +137,7 @@ export interface Config { | { store: 'redis'; /** - * A redis connection string in the form `user:pass@host:port`. + * A redis connection string in the form `redis://user:pass@host:port`. * @secret */ connection: string; diff --git a/packages/backend-common/src/cache/CacheManager.test.ts b/packages/backend-common/src/cache/CacheManager.test.ts index 660aa82bb0..c6cad2e49b 100644 --- a/packages/backend-common/src/cache/CacheManager.test.ts +++ b/packages/backend-common/src/cache/CacheManager.test.ts @@ -195,14 +195,13 @@ describe('CacheManager', () => { }); it('returns a Redis client when configured', () => { - const redisHostAndPort = '127.0.0.1:6379'; - const expectedHost = `redis://${redisHostAndPort}`; + const redisConnection = 'redis://127.0.0.1:6379'; const manager = CacheManager.fromConfig( new ConfigReader({ backend: { cache: { store: 'redis', - connection: redisHostAndPort, + connection: redisConnection, }, }, }), @@ -218,7 +217,7 @@ describe('CacheManager', () => { expect(mockCacheCalls[0][0].store).toBeInstanceOf(KeyvRedis); const redis = KeyvRedis as jest.Mock; const mockRedisCalls = redis.mock.calls.splice(-1); - expect(mockRedisCalls[0][0]).toEqual(expectedHost); + expect(mockRedisCalls[0][0]).toEqual(redisConnection); }); describe('connection errors', () => { diff --git a/packages/backend-common/src/cache/CacheManager.ts b/packages/backend-common/src/cache/CacheManager.ts index d63ac6d940..d8652e42d8 100644 --- a/packages/backend-common/src/cache/CacheManager.ts +++ b/packages/backend-common/src/cache/CacheManager.ts @@ -133,7 +133,7 @@ export class CacheManager { return new Keyv({ namespace: pluginId, ttl: defaultTtl, - store: new KeyvRedis(`redis://${this.connection}`), + store: new KeyvRedis(this.connection), }); } diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index ea8e4626be..371e11b364 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -270,22 +270,6 @@ describe('BitbucketUrlReader', () => { expect(response.etag).toBe('12ab34cd56ef'); }); - - it('should throw error when apiBaseUrl is missing', () => { - expect(() => { - /* eslint-disable no-new */ - new BitbucketUrlReader( - new BitbucketIntegration( - readBitbucketIntegrationConfig( - new ConfigReader({ - host: 'bitbucket.mycompany.net', - }), - ), - ), - { treeResponseFactory }, - ); - }).toThrowError('must configure an explicit apiBaseUrl'); - }); }); describe('search hosted', () => { diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 2006637545..e22137ea69 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -62,14 +62,9 @@ export class BitbucketUrlReader implements UrlReader { private readonly integration: BitbucketIntegration, private readonly deps: { treeResponseFactory: ReadTreeResponseFactory }, ) { - const { host, apiBaseUrl, token, username, appPassword } = - integration.config; + const { host, token, username, appPassword } = integration.config; - if (!apiBaseUrl) { - throw new Error( - `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, - ); - } else if (!token && username && !appPassword) { + if (!token && username && !appPassword) { throw new Error( `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, ); diff --git a/packages/backend-common/src/reading/index.ts b/packages/backend-common/src/reading/index.ts index 2c3394e4fc..9c29bc93fe 100644 --- a/packages/backend-common/src/reading/index.ts +++ b/packages/backend-common/src/reading/index.ts @@ -19,6 +19,7 @@ export { BitbucketUrlReader } from './BitbucketUrlReader'; export { GithubUrlReader } from './GithubUrlReader'; export { GitlabUrlReader } from './GitlabUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { FetchUrlReader } from './FetchUrlReader'; export type { FromReadableArrayOptions, ReaderFactory, diff --git a/packages/backend-common/src/util/DockerContainerRunner.test.ts b/packages/backend-common/src/util/DockerContainerRunner.test.ts index 8833e4518b..3208679d1f 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.test.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.test.ts @@ -115,6 +115,7 @@ describe('DockerContainerRunner', () => { Env: envVarsArray, WorkingDir: workingDir, HostConfig: { + AutoRemove: true, Binds: expect.arrayContaining([ `${path.join(rootDir, 'input')}:/input`, `${path.join(rootDir, 'output')}:/output`, @@ -207,6 +208,7 @@ describe('DockerContainerRunner', () => { logStream, expect.objectContaining({ HostConfig: { + AutoRemove: true, Binds: [], }, Volumes: {}, diff --git a/packages/backend-common/src/util/DockerContainerRunner.ts b/packages/backend-common/src/util/DockerContainerRunner.ts index 522328c2ec..4d4684c814 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.ts @@ -105,6 +105,7 @@ export class DockerContainerRunner implements ContainerRunner { await this.dockerClient.run(imageName, args, logStream, { Volumes, HostConfig: { + AutoRemove: true, Binds, }, ...(workingDir ? { WorkingDir: workingDir } : {}), diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index 4b5363e73d..a90b47cd81 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -18,7 +18,7 @@ import { useHotCleanup, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; import { createRouter } from '@backstage/plugin-search-backend'; import { ElasticSearchSearchEngine } from '@backstage/plugin-search-backend-module-elasticsearch'; import { PgSearchEngine } from '@backstage/plugin-search-backend-module-pg'; @@ -27,7 +27,7 @@ import { LunrSearchEngine, SearchEngine, } from '@backstage/plugin-search-backend-node'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; import { Logger } from 'winston'; import { PluginEnvironment } from '../types'; @@ -70,7 +70,7 @@ export default async function createPlugin({ // particular collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -78,7 +78,7 @@ export default async function createPlugin({ indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, diff --git a/packages/catalog-client/api-report.md b/packages/catalog-client/api-report.md index c957859fec..9bbb24463b 100644 --- a/packages/catalog-client/api-report.md +++ b/packages/catalog-client/api-report.md @@ -3,8 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; // @public export type AddLocationRequest = { @@ -38,8 +38,13 @@ export interface CatalogApi { request: GetEntityAncestorsRequest, options?: CatalogRequestOptions, ): Promise; + // @deprecated getEntityByName( - name: EntityName, + name: CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( @@ -91,7 +96,11 @@ export class CatalogClient implements CatalogApi { options?: CatalogRequestOptions, ): Promise; getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + getEntityByRef( + entityRef: string | CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; getEntityFacets( @@ -130,18 +139,6 @@ export class CatalogClient implements CatalogApi { ): Promise; } -// @public @deprecated (undocumented) -export type CatalogEntitiesRequest = GetEntitiesRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; - -// @public @deprecated (undocumented) -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; - -// @public @deprecated (undocumented) -export type CatalogListResponse<_Entity> = GetEntitiesResponse; - // @public export interface CatalogRequestOptions { // (undocumented) diff --git a/packages/catalog-client/src/CatalogClient.test.ts b/packages/catalog-client/src/CatalogClient.test.ts index 0035691f2c..0b838eb4c2 100644 --- a/packages/catalog-client/src/CatalogClient.test.ts +++ b/packages/catalog-client/src/CatalogClient.test.ts @@ -195,6 +195,60 @@ describe('CatalogClient', () => { }); }); + describe('getEntityByRef', () => { + const existingEntity: Entity = { + apiVersion: 'v1', + kind: 'CustomKind', + metadata: { + namespace: 'default', + name: 'exists', + }, + }; + + beforeEach(() => { + server.use( + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/exists`, + (_, res, ctx) => { + return res(ctx.json(existingEntity)); + }, + ), + rest.get( + `${mockBaseUrl}/entities/by-name/customkind/default/missing`, + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), + ); + }); + + it('finds by string and compound', async () => { + await expect( + client.getEntityByRef('customkind:default/exists'), + ).resolves.toEqual(existingEntity); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'exists', + }), + ).resolves.toEqual(existingEntity); + }); + + it('returns undefined for 404s', async () => { + await expect( + client.getEntityByRef('customkind:default/missing'), + ).resolves.toBeUndefined(); + await expect( + client.getEntityByRef({ + kind: 'CustomKind', + namespace: 'default', + name: 'missing', + }), + ).resolves.toBeUndefined(); + }); + }); + describe('getLocationById', () => { const defaultResponse = { data: { diff --git a/packages/catalog-client/src/CatalogClient.ts b/packages/catalog-client/src/CatalogClient.ts index bb3c9604b3..f2bc3ed8f6 100644 --- a/packages/catalog-client/src/CatalogClient.ts +++ b/packages/catalog-client/src/CatalogClient.ts @@ -18,7 +18,7 @@ import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, Entity, - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, stringifyLocationRef, @@ -170,11 +170,32 @@ export class CatalogClient implements CatalogApi { return { items: entities.sort(refCompare) }; } + /** + * {@inheritdoc CatalogApi.getEntityByRef} + */ + async getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise { + const { kind, namespace, name } = parseEntityRef(entityRef); + return this.requestOptional( + 'GET', + `/entities/by-name/${encodeURIComponent(kind)}/${encodeURIComponent( + namespace, + )}/${encodeURIComponent(name)}`, + options, + ); + } + + // NOTE(freben): When we deprecate getEntityByName from the interface, we may + // still want to leave this implementation in place for quite some time + // longer, to minimize the risk for breakages. Suggested date for removal: + // August 2022 /** * {@inheritdoc CatalogApi.getEntityByName} */ async getEntityByName( - compoundName: EntityName, + compoundName: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise { const { kind, namespace = 'default', name } = compoundName; diff --git a/packages/catalog-client/src/types/api.ts b/packages/catalog-client/src/types/api.ts index 4a86ca41f3..26af3889e2 100644 --- a/packages/catalog-client/src/types/api.ts +++ b/packages/catalog-client/src/types/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef, Entity } from '@backstage/catalog-model'; /** * This symbol can be used in place of a value when passed to filters in e.g. @@ -306,11 +306,25 @@ export interface CatalogApi { * Gets a single entity from the catalog by its ref (kind, namespace, name) * triplet. * + * @param entityRef - A complete entity ref, either on string or compound form + * @param options - Additional options + * @returns The matching entity, or undefined if there was no entity with that ref + */ + getEntityByRef( + entityRef: string | CompoundEntityRef, + options?: CatalogRequestOptions, + ): Promise; + + /** + * Gets a single entity from the catalog by its ref (kind, namespace, name) + * triplet. + * + * @deprecated Use getEntityRef instead * @param name - A complete entity ref * @param options - Additional options */ getEntityByName( - name: EntityName, + name: CompoundEntityRef, options?: CatalogRequestOptions, ): Promise; diff --git a/packages/catalog-client/src/types/deprecated.ts b/packages/catalog-client/src/types/deprecated.ts deleted file mode 100644 index 0b145f3024..0000000000 --- a/packages/catalog-client/src/types/deprecated.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - GetEntitiesRequest, - GetEntitiesResponse, - GetEntityAncestorsRequest, - GetEntityAncestorsResponse, -} from './api'; - -/** - * @public - * @deprecated use GetEntitiesRequest instead - */ -export type CatalogEntitiesRequest = GetEntitiesRequest; -/** - * @public - * @deprecated use GetEntitiesResponse instead - */ -export type CatalogListResponse<_Entity> = GetEntitiesResponse; -/** - * @public - * @deprecated use GetEntityAncestorsRequest instead - */ -export type CatalogEntityAncestorsRequest = GetEntityAncestorsRequest; -/** - * @public - * @deprecated use GetEntityAncestorsResponse instead - */ -export type CatalogEntityAncestorsResponse = GetEntityAncestorsResponse; diff --git a/packages/catalog-client/src/types/index.ts b/packages/catalog-client/src/types/index.ts index 39c8962b68..8bf4e34da2 100644 --- a/packages/catalog-client/src/types/index.ts +++ b/packages/catalog-client/src/types/index.ts @@ -28,5 +28,4 @@ export type { GetEntityFacetsRequest, GetEntityFacetsResponse, } from './api'; -export * from './deprecated'; export { ENTITY_STATUS_CATALOG_PROCESSING_TYPE } from './status'; diff --git a/packages/catalog-model/api-report.md b/packages/catalog-model/api-report.md index 210d0187e2..99b4b39038 100644 --- a/packages/catalog-model/api-report.md +++ b/packages/catalog-model/api-report.md @@ -4,8 +4,6 @@ ```ts import { JsonObject } from '@backstage/types'; -import { JSONSchema7 } from 'json-schema'; -import { JsonValue } from '@backstage/types'; import { SerializedError } from '@backstage/errors'; // @alpha @@ -66,23 +64,6 @@ export class CommonValidatorFunctions { static isValidUrl(value: unknown): boolean; } -// @public @deprecated -export function compareEntityToRef( - entity: Entity, - ref: - | string - | { - kind?: string; - namespace?: string; - name: string; - } - | EntityName, - context?: { - defaultKind?: string; - defaultNamespace?: string; - }, -): boolean; - // @public interface ComponentEntityV1alpha1 extends Entity { // (undocumented) @@ -107,6 +88,13 @@ export { ComponentEntityV1alpha1 }; // @public export const componentEntityV1alpha1Validator: KindValidator; +// @public +export type CompoundEntityRef = { + kind: string; + namespace: string; + name: string; +}; + // @public export const DEFAULT_NAMESPACE = 'default'; @@ -134,9 +122,6 @@ export { DomainEntityV1alpha1 }; // @public export const domainEntityV1alpha1Validator: KindValidator; -// @public @deprecated -export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; - // @public export type Entity = { apiVersion: string; @@ -146,16 +131,6 @@ export type Entity = { relations?: EntityRelation[]; }; -// @public @deprecated -export const ENTITY_DEFAULT_NAMESPACE = 'default'; - -// @public @deprecated -export const ENTITY_META_GENERATED_FIELDS: readonly [ - 'uid', - 'etag', - 'generation', -]; - // @public export type EntityEnvelope = { apiVersion: string; @@ -171,9 +146,6 @@ export function entityEnvelopeSchemaValidator< T extends EntityEnvelope = EntityEnvelope, >(schema?: unknown): (data: unknown) => T; -// @public @deprecated -export function entityHasChanges(previous: Entity, next: Entity): boolean; - // @public export function entityKindSchemaValidator( schema: unknown, @@ -201,12 +173,8 @@ export type EntityMeta = JsonObject & { links?: EntityLink[]; }; -// @public -export type EntityName = { - kind: string; - namespace: string; - name: string; -}; +// @public @deprecated +export type EntityName = CompoundEntityRef; // @public export const EntityPolicies: { @@ -228,16 +196,10 @@ export type EntityRef = name: string; }; -// @public @deprecated -export type EntityRefContext = { - defaultKind?: string; - defaultNamespace?: string; -}; - // @public export type EntityRelation = { type: string; - target: EntityName; + target: CompoundEntityRef; targetRef: string; }; @@ -269,17 +231,11 @@ export class FieldFormatEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export function generateEntityEtag(): string; - -// @public @deprecated -export function generateEntityUid(): string; - -// @public @deprecated -export function generateUpdatedEntity(previous: Entity, next: Entity): Entity; - // @public -export function getEntityName(entity: Entity): EntityName; +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef; + +// @public @deprecated +export const getEntityName: typeof getCompoundEntityRef; // @public export function getEntitySourceLocation(entity: Entity): { @@ -312,11 +268,6 @@ export { GroupEntityV1alpha1 }; // @public export const groupEntityV1alpha1Validator: KindValidator; -// @public @deprecated -export type JSONSchema = JSONSchema7 & { - [key in string]?: JsonValue; -}; - // @public export type KindValidator = { check(entity: Entity): Promise; @@ -342,15 +293,6 @@ export class KubernetesValidatorFunctions { static isValidObjectName(value: unknown): boolean; } -// @public @deprecated -type Location_2 = { - id: string; -} & LocationSpec; -export { Location_2 as Location }; - -// @public @deprecated -export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; - // @public interface LocationEntityV1alpha1 extends Entity { // (undocumented) @@ -388,10 +330,6 @@ export class NoForeignRootFieldsEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export const ORIGIN_LOCATION_ANNOTATION = - 'backstage.io/managed-by-origin-location'; - // @public @deprecated export function parseEntityName( ref: @@ -405,7 +343,7 @@ export function parseEntityName( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseEntityRef( @@ -420,7 +358,7 @@ export function parseEntityRef( defaultKind?: string; defaultNamespace?: string; }, -): EntityName; +): CompoundEntityRef; // @public export function parseLocationRef(ref: string): { @@ -428,12 +366,6 @@ export function parseLocationRef(ref: string): { target: string; }; -// @public @deprecated -export function parseLocationReference(ref: string): { - type: string; - target: string; -}; - // @public export const RELATION_API_CONSUMED_BY = 'apiConsumedBy'; @@ -503,9 +435,6 @@ export class SchemaValidEntityPolicy implements EntityPolicy { enforce(entity: Entity): Promise; } -// @public @deprecated -export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; - // @public export function stringifyEntityRef( ref: @@ -523,12 +452,6 @@ export function stringifyLocationRef(ref: { target: string; }): string; -// @public @deprecated -export function stringifyLocationReference(ref: { - type: string; - target: string; -}): string; - // @public interface SystemEntityV1alpha1 extends Entity { // (undocumented) @@ -581,7 +504,4 @@ export type Validators = { isValidAnnotationValue(value: unknown): boolean; isValidTag(value: unknown): boolean; }; - -// @public @deprecated -export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; ``` diff --git a/packages/catalog-model/package.json b/packages/catalog-model/package.json index 70ed0fbe4d..f9781b14a2 100644 --- a/packages/catalog-model/package.json +++ b/packages/catalog-model/package.json @@ -37,7 +37,6 @@ "@backstage/config": "^0.1.15", "@backstage/errors": "^0.2.2", "@backstage/types": "^0.1.3", - "@types/json-schema": "^7.0.5", "ajv": "^7.0.3", "json-schema": "^0.4.0", "lodash": "^4.17.21", @@ -46,6 +45,7 @@ "devDependencies": { "@backstage/cli": "^0.14.1", "@types/jest": "^26.0.7", + "@types/json-schema": "^7.0.5", "@types/lodash": "^4.14.151", "yaml": "^1.9.2" }, diff --git a/packages/catalog-model/src/entity/Entity.ts b/packages/catalog-model/src/entity/Entity.ts index ef9b4352da..029d9e095b 100644 --- a/packages/catalog-model/src/entity/Entity.ts +++ b/packages/catalog-model/src/entity/Entity.ts @@ -15,7 +15,7 @@ */ import { JsonObject } from '@backstage/types'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { EntityStatus } from './EntityStatus'; /** @@ -201,7 +201,7 @@ export type EntityRelation = { * * @deprecated use targetRef instead */ - target: EntityName; + target: CompoundEntityRef; /** * The entity ref of the target of this relation. diff --git a/packages/catalog-model/src/entity/constants.ts b/packages/catalog-model/src/entity/constants.ts index 6cef19cac9..5dc21683bd 100644 --- a/packages/catalog-model/src/entity/constants.ts +++ b/packages/catalog-model/src/entity/constants.ts @@ -14,14 +14,6 @@ * limitations under the License. */ -/** - * The namespace that entities without an explicit namespace fall into. - * - * @public - * @deprecated use {@link DEFAULT_NAMESPACE} instead. - */ -export const ENTITY_DEFAULT_NAMESPACE = 'default'; - /** * The namespace that entities without an explicit namespace fall into. * @@ -29,34 +21,6 @@ export const ENTITY_DEFAULT_NAMESPACE = 'default'; */ export const DEFAULT_NAMESPACE = 'default'; -/** - * The keys of EntityMeta that are auto-generated. - * - * @public - * @deprecated will be removed in a future release. - */ -export const ENTITY_META_GENERATED_FIELDS = [ - 'uid', - 'etag', - 'generation', -] as const; - -/** - * Annotation for linking to entity page from catalog pages. - * - * @public - * @deprecated use {@link ANNOTATION_VIEW_URL} instead. - */ -export const VIEW_URL_ANNOTATION = 'backstage.io/view-url'; - -/** - * Annotation for linking to entity edit page from catalog pages. - * - * @public - * @deprecated use {@link ANNOTATION_EDIT_URL} instead. - */ -export const EDIT_URL_ANNOTATION = 'backstage.io/edit-url'; - /** * Annotation for linking to entity page from catalog pages. * diff --git a/packages/catalog-model/src/entity/index.ts b/packages/catalog-model/src/entity/index.ts index f619d0f39e..fb9eee0353 100644 --- a/packages/catalog-model/src/entity/index.ts +++ b/packages/catalog-model/src/entity/index.ts @@ -15,11 +15,7 @@ */ export { - EDIT_URL_ANNOTATION, - ENTITY_DEFAULT_NAMESPACE, DEFAULT_NAMESPACE, - ENTITY_META_GENERATED_FIELDS, - VIEW_URL_ANNOTATION, ANNOTATION_EDIT_URL, ANNOTATION_VIEW_URL, } from './constants'; @@ -38,16 +34,9 @@ export type { } from './EntityStatus'; export * from './policies'; export { - compareEntityToRef, + getCompoundEntityRef, getEntityName, parseEntityName, parseEntityRef, stringifyEntityRef, } from './ref'; -export type { EntityRefContext } from './ref'; -export { - entityHasChanges, - generateEntityEtag, - generateEntityUid, - generateUpdatedEntity, -} from './util'; diff --git a/packages/catalog-model/src/entity/ref.test.ts b/packages/catalog-model/src/entity/ref.test.ts index b0c7068fae..045c4710dc 100644 --- a/packages/catalog-model/src/entity/ref.test.ts +++ b/packages/catalog-model/src/entity/ref.test.ts @@ -15,8 +15,7 @@ */ import { DEFAULT_NAMESPACE } from './constants'; -import { Entity } from './Entity'; -import { compareEntityToRef, parseEntityName, parseEntityRef } from './ref'; +import { parseEntityName, parseEntityRef } from './ref'; describe('ref', () => { describe('parseEntityName', () => { @@ -313,320 +312,4 @@ describe('ref', () => { ).toThrow(/namespace/); }); }); - - describe('compareEntityToRef', () => { - const entityWithNamespace: Entity = { - apiVersion: 'a', - kind: 'K', - metadata: { - name: 'n', - namespace: 'ns', - }, - }; - const entityWithoutNamespace: Entity = { - apiVersion: 'a', - kind: 'K', - metadata: { - name: 'n', - }, - }; - - it('handles matching string refs', () => { - expect(compareEntityToRef(entityWithNamespace, 'K:ns/n')).toBe(true); - expect(compareEntityToRef(entityWithNamespace, 'k:nS/N')).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'K:n', { - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'K:n', { - defaultNamespace: 'Ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'ns/n', { defaultKind: 'K' }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'N', { - defaultKind: 'k', - defaultNamespace: 'nS', - }), - ).toBe(true); - - expect(compareEntityToRef(entityWithoutNamespace, 'K:default/n')).toBe( - true, - ); - expect(compareEntityToRef(entityWithoutNamespace, 'K:deFault/n')).toBe( - true, - ); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'default', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'deFault', - }), - ).toBe(true); - expect(compareEntityToRef(entityWithoutNamespace, 'K:default/n')).toBe( - true, - ); - expect(compareEntityToRef(entityWithoutNamespace, 'K:n')).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'default/n', { - defaultKind: 'K', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'default', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, 'n', { - defaultKind: 'K', - }), - ).toBe(true); - }); - - it('handles mismatching string refs', () => { - expect(compareEntityToRef(entityWithNamespace, 'X:ns/n')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'ns/n', { - defaultKind: 'X', - }), - ).toBe(false); - - expect(compareEntityToRef(entityWithNamespace, 'K:xx/n')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'K:n', { - defaultNamespace: 'xx', - }), - ).toBe(false); - - expect(compareEntityToRef(entityWithNamespace, 'K:ns/x')).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, 'x', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(false); - }); - - it('handles matching compound refs', () => { - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'ns', - name: 'n', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'k', - namespace: 'Ns', - name: 'N', - }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithNamespace, - { kind: 'K', name: 'n' }, - { - defaultNamespace: 'ns', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithNamespace, - { namespace: 'ns', name: 'n' }, - { defaultKind: 'K' }, - ), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'n', { - defaultKind: 'K', - defaultNamespace: 'ns', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithNamespace, 'N', { - defaultKind: 'k', - defaultNamespace: 'nS', - }), - ).toBe(true); - - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'default', - name: 'n', - }), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'k', - namespace: 'deFault', - name: 'N', - }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { kind: 'K', name: 'n' }, - { - defaultNamespace: 'default', - }, - ), - ).toBe(true); - expect( - compareEntityToRef(entityWithoutNamespace, { kind: 'K', name: 'n' }), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { namespace: 'default', name: 'n' }, - { - defaultKind: 'K', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'n' }, - { - defaultKind: 'K', - defaultNamespace: 'default', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'N' }, - { - defaultKind: 'k', - defaultNamespace: 'defAult', - }, - ), - ).toBe(true); - expect( - compareEntityToRef( - entityWithoutNamespace, - { name: 'n' }, - { - defaultKind: 'K', - }, - ), - ).toBe(true); - }); - - it('handles mismatching compound refs', () => { - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'X', - namespace: 'ns', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithNamespace, - { - namespace: 'ns', - name: 'n', - }, - { defaultKind: 'X' }, - ), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'X', - namespace: 'default', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - namespace: 'default', - name: 'n', - }, - { defaultKind: 'X' }, - ), - ).toBe(false); - - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'xx', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithNamespace, - { - kind: 'K', - name: 'n', - }, - { defaultNamespace: 'xx' }, - ), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'xx', - name: 'n', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - kind: 'K', - name: 'n', - }, - { defaultNamespace: 'xx' }, - ), - ).toBe(false); - - expect( - compareEntityToRef(entityWithNamespace, { - kind: 'K', - namespace: 'ns', - name: 'x', - }), - ).toBe(false); - expect( - compareEntityToRef(entityWithoutNamespace, { - kind: 'K', - namespace: 'default', - name: 'x', - }), - ).toBe(false); - expect( - compareEntityToRef( - entityWithoutNamespace, - { - kind: 'K', - name: 'x', - }, - { defaultNamespace: 'default' }, - ), - ).toBe(false); - }); - }); }); diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index 66974f5a01..4529b1174c 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -15,7 +15,7 @@ */ import { DEFAULT_NAMESPACE } from './constants'; -import { EntityName } from '../types'; +import { CompoundEntityRef } from '../types'; import { Entity } from './Entity'; function parseRefString(ref: string): { @@ -38,14 +38,25 @@ function parseRefString(ref: string): { } /** - * Extracts the kind, namespace and name that form the name triplet of the - * given entity. + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. + * + * @public + * @deprecated Use getCompoundEntityRef instead + * @param entity - An entity + * @returns The compound entity ref + */ +export const getEntityName = getCompoundEntityRef; + +/** + * Extracts the kind, namespace and name that form the compound entity ref + * triplet of the given entity. * * @public * @param entity - An entity - * @returns The complete entity name + * @returns The compound entity ref */ -export function getEntityName(entity: Entity): EntityName { +export function getCompoundEntityRef(entity: Entity): CompoundEntityRef { return { kind: entity.kind, namespace: entity.metadata.namespace || DEFAULT_NAMESPACE, @@ -53,19 +64,6 @@ export function getEntityName(entity: Entity): EntityName { }; } -/** - * The context of defaults that entity reference parsing happens within. - * - * @public - * @deprecated type inlined, will be removed in a future release. - */ -export type EntityRefContext = { - /** The default kind, if none is given in the reference */ - defaultKind?: string; - /** The default namespace, if none is given in the reference */ - defaultNamespace?: string; -}; - /** * Parses an entity reference, either on string or compound form, and always * returns a complete entity name including kind, namespace and name. @@ -90,7 +88,7 @@ export function parseEntityName( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; } = {}, -): EntityName { +): CompoundEntityRef { const { kind, namespace, name } = parseEntityRef(ref, { defaultNamespace: DEFAULT_NAMESPACE, ...context, @@ -127,7 +125,7 @@ export function parseEntityRef( /** The default namespace, if none is given in the reference */ defaultNamespace?: string; }, -): EntityName { +): CompoundEntityRef { if (!ref) { throw new Error(`Entity reference must not be empty`); } @@ -203,67 +201,3 @@ export function stringifyEntityRef( 'en-US', )}/${name.toLocaleLowerCase('en-US')}`; } - -/** - * Compares an entity to either a string reference or a compound reference. - * - * @remarks - * - * The comparison is case insensitive, and all of kind, namespace, and name - * must match (after applying the optional context to the ref). - * - * @public - * @param entity - The entity to match - * @param ref - A string or compound entity ref - * @param context - An optional context of default kind and namespace, that apply - * to the ref if given - * @returns True if matching, false otherwise - * @deprecated compare using stringifyEntityRef instead. - */ -export function compareEntityToRef( - entity: Entity, - ref: - | string - | { kind?: string; namespace?: string; name: string } - | EntityName, - context?: { - /** The default kind, if none is given in the reference */ - defaultKind?: string; - /** The default namespace, if none is given in the reference */ - defaultNamespace?: string; - }, -): boolean { - const entityKind = entity.kind; - const entityNamespace = entity.metadata.namespace || DEFAULT_NAMESPACE; - const entityName = entity.metadata.name; - - let refKind: string | undefined; - let refNamespace: string | undefined; - let refName: string; - if (typeof ref === 'string') { - const parsed = parseRefString(ref); - refKind = parsed.kind || context?.defaultKind; - refNamespace = - parsed.namespace || context?.defaultNamespace || DEFAULT_NAMESPACE; - refName = parsed.name; - } else { - refKind = ref.kind || context?.defaultKind; - refNamespace = - ref.namespace || context?.defaultNamespace || DEFAULT_NAMESPACE; - refName = ref.name; - } - - if (!refKind || !refNamespace) { - throw new Error( - `Entity reference or context did not contain kind and namespace`, - ); - } - - return ( - entityKind.toLocaleLowerCase('en-US') === - refKind.toLocaleLowerCase('en-US') && - entityNamespace.toLocaleLowerCase('en-US') === - refNamespace.toLocaleLowerCase('en-US') && - entityName.toLocaleLowerCase('en-US') === refName.toLocaleLowerCase('en-US') - ); -} diff --git a/packages/catalog-model/src/entity/util.test.ts b/packages/catalog-model/src/entity/util.test.ts deleted file mode 100644 index 1c39961c0f..0000000000 --- a/packages/catalog-model/src/entity/util.test.ts +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import lodash from 'lodash'; -import { - generateEntityEtag, - generateEntityUid, - entityHasChanges, - generateUpdatedEntity, -} from './util'; -import { Entity } from './Entity'; - -describe('util', () => { - describe('generateEntityUid', () => { - it('generates randomness', () => { - expect(generateEntityUid()).not.toEqual(''); - expect(generateEntityUid()).not.toEqual(generateEntityUid()); - }); - }); - - describe('generateEntityEtag', () => { - it('generates randomness', () => { - expect(generateEntityEtag()).not.toEqual(''); - expect(generateEntityEtag()).not.toEqual(generateEntityEtag()); - }); - }); - - describe('entityHasChanges', () => { - let a: Entity; - beforeEach(() => { - a = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'name', - custom: 'custom', - labels: { - labelKey: 'labelValue', - }, - annotations: { - annotationKey: 'annotationValue', - }, - }, - spec: { - a: 'a', - }, - }; - }); - - it('happy path: clone has no changes', () => { - const b = lodash.cloneDeep(a); - expect(entityHasChanges(a, b)).toBe(false); - }); - - it('detects root field changes', () => { - let b: any = lodash.cloneDeep(a); - b.apiVersion += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.apiVersion; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.kind += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.kind; - expect(entityHasChanges(a, b)).toBe(true); - }); - - it('detects metadata changes', () => { - let b: any = lodash.cloneDeep(a); - b.metadata.name += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.custom; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.custom; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.labels.n = 'n'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.labels.labelKey += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.metadata.annotations.annotationKey += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.metadata.annotations.annotationKey; - expect(entityHasChanges(a, b)).toBe(true); - }); - - it('detects spec changes', () => { - let b: any = lodash.cloneDeep(a); - b.spec.a += 'a'; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - delete b.spec.a; - expect(entityHasChanges(a, b)).toBe(true); - b = lodash.cloneDeep(a); - b.spec.n = 'n'; - expect(entityHasChanges(a, b)).toBe(true); - }); - }); - - describe('generateUpdatedEntity', () => { - let a: Entity; - let b: any; - beforeEach(() => { - a = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - uid: 'da921f56-f655-4e6e-9b8b-bb19a57818d8', - etag: 'NzY5NDA5NzQtYmEwNC00MDY0LWFiYmItNTYxYzQxM2JhZDcx', - generation: 2, - name: 'name', - custom: 'custom', - labels: { - labelKey: 'labelValue', - }, - annotations: { - annotationKey: 'annotationValue', - }, - }, - spec: { - a: 'a', - }, - }; - b = lodash.cloneDeep(a); - delete b.metadata.uid; - delete b.metadata.etag; - delete b.metadata.generation; - }); - - it('happy path: running on itself leaves it unchanged', () => { - const result = generateUpdatedEntity(a, b); - expect(result).toEqual(a); - }); - - it('bumps etag and generation when spec is changed', () => { - b.spec.a += 'a'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation! + 1); - expect(result.spec).toEqual({ a: 'aa' }); - }); - - it('bumps only etag when other things than spec are changed', () => { - b.metadata.n = 'n'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.n).toEqual('n'); - }); - - it('retains new annotations', () => { - b.metadata.annotations.annotationKey = 'changedValue'; - b.metadata.annotations.newKey = 'newValue'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.annotations).toEqual({ - annotationKey: 'changedValue', - newKey: 'newValue', - }); - }); - - it('retains old annotations', () => { - b.metadata.annotations.newKey = 'newValue'; - const result = generateUpdatedEntity(a, b); - expect(result.metadata.uid).toEqual(a.metadata.uid); - expect(result.metadata.etag).not.toEqual(a.metadata.etag); - expect(result.metadata.generation).toEqual(a.metadata.generation); - expect(result.metadata.annotations).toEqual({ - annotationKey: 'annotationValue', - newKey: 'newValue', - }); - }); - }); -}); diff --git a/packages/catalog-model/src/entity/util.ts b/packages/catalog-model/src/entity/util.ts deleted file mode 100644 index 78359b45f4..0000000000 --- a/packages/catalog-model/src/entity/util.ts +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import lodash from 'lodash'; -import { v4 as uuidv4 } from 'uuid'; -import { Entity, AlphaEntity } from './Entity'; - -/** - * Generates a new random UID for an entity. - * - * @public - * @returns A string with enough randomness to uniquely identify an entity - * @deprecated use `uuidv4()` instead. - */ -export function generateEntityUid(): string { - return uuidv4(); -} - -/** - * Generates a new random Etag for an entity. - * - * @public - * @returns A string with enough randomness to uniquely identify an entity - * revision - * @deprecated will be removed in a future release. - */ -export function generateEntityEtag(): string { - return Buffer.from(uuidv4(), 'utf8').toString('base64').replace(/[^\w]/g, ''); -} - -/** - * Checks whether there are any significant changes going from the previous to - * the next version of this entity. - * - * @remarks - * - * Significance, in this case, means that we do not compare generated fields - * such as uid, etag and generation. - * - * Note that this comparison does NOT take status, relations or similar into - * account. It only compares the actual input entity data, i.e. metadata and - * spec. - * - * @public - * @param previous - The old state of the entity - * @param next - The new state of the entity - * @deprecated will be removed in a future release. - */ -export function entityHasChanges(previous: Entity, next: Entity): boolean { - const e1 = lodash.cloneDeep(previous); - const e2 = lodash.cloneDeep(next); - - if (!e1.metadata.labels) { - e1.metadata.labels = {}; - } - if (!e2.metadata.labels) { - e2.metadata.labels = {}; - } - if (!e1.metadata.annotations) { - e1.metadata.annotations = {}; - } - if (!e2.metadata.annotations) { - e2.metadata.annotations = {}; - } - if (!e1.metadata.tags) { - e1.metadata.tags = []; - } - if (!e2.metadata.tags) { - e2.metadata.tags = []; - } - - // Remove generated fields - delete e1.metadata.uid; - delete e1.metadata.etag; - delete e1.metadata.generation; - delete e2.metadata.uid; - delete e2.metadata.etag; - delete e2.metadata.generation; - - // Remove things that we explicitly do not compare - delete e1.relations; - delete (e1 as AlphaEntity).status; - delete e2.relations; - delete (e2 as AlphaEntity).status; - - return !lodash.isEqual(e1, e2); -} - -/** - * Takes an old revision of an entity and a new desired state, and merges - * them into a complete new state. - * - * @remarks - * - * The previous revision is expected to be a complete model loaded from the - * catalog, including the uid, etag and generation fields. - * - * @public - * @param previous - The old state of the entity - * @param next - The new state of the entity - * @returns An entity with the merged state of both - * @deprecated will be removed in a future release. - */ -export function generateUpdatedEntity(previous: Entity, next: Entity): Entity { - const { uid, etag, generation } = previous.metadata; - if (!uid || !etag || !generation) { - throw new Error('Previous entity must have uid, etag and generation'); - } - - const result = lodash.cloneDeep(next); - - // Generated fields are copied and updated - const bumpEtag = entityHasChanges(previous, result); - const bumpGeneration = !lodash.isEqual(previous.spec, result.spec); - result.metadata.uid = uid; - result.metadata.etag = bumpEtag ? generateEntityEtag() : etag; - result.metadata.generation = bumpGeneration ? generation + 1 : generation; - - return result; -} diff --git a/packages/catalog-model/src/index.ts b/packages/catalog-model/src/index.ts index ef55337ae2..19eacd4925 100644 --- a/packages/catalog-model/src/index.ts +++ b/packages/catalog-model/src/index.ts @@ -24,5 +24,5 @@ export * from './entity'; export { EntityPolicies } from './EntityPolicies'; export * from './kinds'; export * from './location'; -export type { EntityName, EntityRef, JSONSchema } from './types'; +export type { EntityName, EntityRef, CompoundEntityRef } from './types'; export * from './validation'; diff --git a/packages/catalog-model/src/location/annotation.ts b/packages/catalog-model/src/location/annotation.ts index 3ff624360e..61dbc0b305 100644 --- a/packages/catalog-model/src/location/annotation.ts +++ b/packages/catalog-model/src/location/annotation.ts @@ -14,30 +14,6 @@ * limitations under the License. */ -/** - * Constant storing location annotation. - * - * @public - * @deprecated use {@link ANNOTATION_LOCATION} instead. - * */ -export const LOCATION_ANNOTATION = 'backstage.io/managed-by-location'; -/** - * Constant storing origin location annotation - * - * @public - * @deprecated use {@link ANNOTATION_ORIGIN_LOCATION} instead. - */ -export const ORIGIN_LOCATION_ANNOTATION = - 'backstage.io/managed-by-origin-location'; - -/** - * Contant storing source location annotation - * - * @public - * @deprecated use {@link ANNOTATION_SOURCE_LOCATION} instead. - * */ -export const SOURCE_LOCATION_ANNOTATION = 'backstage.io/source-location'; - /** * Constant storing location annotation. * diff --git a/packages/catalog-model/src/location/helpers.ts b/packages/catalog-model/src/location/helpers.ts index 7c73908157..8f8acc3b1d 100644 --- a/packages/catalog-model/src/location/helpers.ts +++ b/packages/catalog-model/src/location/helpers.ts @@ -18,18 +18,6 @@ import { ANNOTATION_SOURCE_LOCATION } from '.'; import { Entity, stringifyEntityRef } from '../entity'; import { ANNOTATION_LOCATION } from './annotation'; -/** - * Parses a string form location reference. - * - * @public - * @param ref - A string-form location reference, e.g. `'url:https://host'` - * @returns A location reference, e.g. `{ type: 'url', target: 'https://host' }` - * @deprecated use {@link parseLocationRef} instead - */ -export function parseLocationReference(ref: string) { - return parseLocationRef(ref); -} - /** * Parses a string form location reference. * @@ -72,21 +60,6 @@ export function parseLocationRef(ref: string): { return { type, target }; } -/** - * Turns a location reference into its string form. - * - * @public - * @param ref - A location reference, e.g. `{ type: 'url', target: 'https://host' }` - * @returns A string-form location reference, e.g. `'url:https://host'` - * @deprecated use {@link stringifyLocationRef} instead - */ -export function stringifyLocationReference(ref: { - type: string; - target: string; -}): string { - return stringifyLocationRef(ref); -} - /** * Turns a location ref into its string form. * diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts index 8302ec202f..dc709144b2 100644 --- a/packages/catalog-model/src/location/index.ts +++ b/packages/catalog-model/src/location/index.ts @@ -18,15 +18,10 @@ export { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, ANNOTATION_SOURCE_LOCATION, - LOCATION_ANNOTATION, - ORIGIN_LOCATION_ANNOTATION, - SOURCE_LOCATION_ANNOTATION, } from './annotation'; export { getEntitySourceLocation, parseLocationRef, - parseLocationReference, stringifyLocationRef, - stringifyLocationReference, } from './helpers'; -export type { Location, LocationSpec } from './types'; +export type { LocationSpec } from './types'; diff --git a/packages/catalog-model/src/location/types.ts b/packages/catalog-model/src/location/types.ts index 9b87e4919d..9e2c9339e6 100644 --- a/packages/catalog-model/src/location/types.ts +++ b/packages/catalog-model/src/location/types.ts @@ -31,13 +31,3 @@ export type LocationSpec = { target: string; presence?: 'optional' | 'required'; }; - -/** - * Entity location for a specific entity. - * - * @public - * @deprecated import from {@link @backstage/catalog-client#Location} instead. - */ -export type Location = { - id: string; -} & LocationSpec; diff --git a/packages/catalog-model/src/types.ts b/packages/catalog-model/src/types.ts index 13550968be..e7d819ace9 100644 --- a/packages/catalog-model/src/types.ts +++ b/packages/catalog-model/src/types.ts @@ -14,28 +14,26 @@ * limitations under the License. */ -import { JsonValue } from '@backstage/types'; -import { JSONSchema7 } from 'json-schema'; - /** - * JSONSchema extendable by arbitrary JSON attributes - * - * @public - * @deprecated use JSONSchema7 from the json-schema package instead. - */ -export type JSONSchema = JSONSchema7 & { [key in string]?: JsonValue }; - -/** - * A complete entity name, with the full kind-namespace-name triplet. + * All parts of a complete entity ref, forming a full kind-namespace-name + * triplet. * * @public */ -export type EntityName = { +export type CompoundEntityRef = { kind: string; namespace: string; name: string; }; +/** + * A complete entity name, with the full kind-namespace-name triplet. + * + * @deprecated Use CompoundEntityRef instead + * @public + */ +export type EntityName = CompoundEntityRef; + /** * A reference by name to an entity, either as a compact string representation, * or as a compound reference structure. diff --git a/packages/cli/package.json b/packages/cli/package.json index b17d517ba1..e5551ce29e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -37,7 +37,8 @@ "@backstage/errors": "^0.2.2", "@backstage/release-manifests": "^0.0.2", "@backstage/types": "^0.1.3", - "@hot-loader/react-dom": "^17.0.2", + "@hot-loader/react-dom-v16": "npm:@hot-loader/react-dom@^16.0.2", + "@hot-loader/react-dom-v17": "npm:@hot-loader/react-dom@^17.0.2", "@manypkg/get-packages": "^1.1.3", "@octokit/request": "^5.4.12", "@rollup/plugin-commonjs": "^21.0.1", diff --git a/packages/cli/src/lib/bundler/config.ts b/packages/cli/src/lib/bundler/config.ts index 3cfcafc005..bfd05807ad 100644 --- a/packages/cli/src/lib/bundler/config.ts +++ b/packages/cli/src/lib/bundler/config.ts @@ -15,7 +15,6 @@ */ import fs from 'fs-extra'; -import chalk from 'chalk'; import { resolve as resolvePath } from 'path'; import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'; import HtmlWebpackPlugin from 'html-webpack-plugin'; @@ -118,43 +117,11 @@ export async function createConfig( }), ); - const appParamDeprecationMsg = chalk.red( - 'DEPRECATION WARNING: using `app.` in the index.html template is deprecated, use `config.getString("app.")` instead.', - ); plugins.push( new HtmlWebpackPlugin({ template: paths.targetHtml, templateParameters: { publicPath: validBaseUrl.pathname.replace(/\/$/, ''), - app: { - get title() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getString('app.title'); - }, - get baseUrl() { - console.warn(appParamDeprecationMsg); - return validBaseUrl.href; - }, - get googleAnalyticsTrackingId() { - console.warn(appParamDeprecationMsg); - return frontendConfig.getOptionalString( - 'app.googleAnalyticsTrackingId', - ); - }, - get datadogRum() { - console.warn(appParamDeprecationMsg); - return { - env: frontendConfig.getOptionalString('app.datadogRum.env'), - clientToken: frontendConfig.getOptionalString( - 'app.datadogRum.clientToken', - ), - applicationId: frontendConfig.getOptionalString( - 'app.datadogRum.applicationId', - ), - site: frontendConfig.getOptionalString('app.datadogRum.site'), - }; - }, - }, config: frontendConfig, }, }), @@ -167,13 +134,16 @@ export async function createConfig( }), ); + // Detect and use the appropriate react-dom hot-loader patch based on what + // version of React is used within the target repo. const resolveAliases: Record = {}; try { // eslint-disable-next-line import/no-extraneous-dependencies const { version: reactDomVersion } = require('react-dom/package.json'); - // Only apply the alias for hook support if we're running with React 16 if (reactDomVersion.startsWith('16.')) { - resolveAliases['react-dom'] = '@hot-loader/react-dom'; + resolveAliases['react-dom'] = '@hot-loader/react-dom-v16'; + } else { + resolveAliases['react-dom'] = '@hot-loader/react-dom-v17'; } } catch (error) { console.warn(`WARNING: Failed to read react-dom version, ${error}`); diff --git a/packages/config-loader/src/lib/schema/collect.ts b/packages/config-loader/src/lib/schema/collect.ts index a05796139b..3ff161cec4 100644 --- a/packages/config-loader/src/lib/schema/collect.ts +++ b/packages/config-loader/src/lib/schema/collect.ts @@ -182,7 +182,7 @@ function compileTsSchemas(paths: string[]) { program, // All schemas should export a `Config` symbol 'Config', - // This enables usage of @visibility and @deprecated in doc comments + // This enables the use of these tags in TSDoc comments { required: true, validationKeywords: ['visibility', 'deprecated'], diff --git a/packages/core-app-api/api-report.md b/packages/core-app-api/api-report.md index 3b53813615..116b33c598 100644 --- a/packages/core-app-api/api-report.md +++ b/packages/core-app-api/api-report.md @@ -16,7 +16,6 @@ import { AppConfig } from '@backstage/config'; import { AppTheme } from '@backstage/core-plugin-api'; import { AppThemeApi } from '@backstage/core-plugin-api'; import { atlassianAuthApiRef } from '@backstage/core-plugin-api'; -import { auth0AuthApiRef } from '@backstage/core-plugin-api'; import { AuthProviderInfo } from '@backstage/core-plugin-api'; import { AuthRequestOptions } from '@backstage/core-plugin-api'; import { BackstageIdentityApi } from '@backstage/core-plugin-api'; @@ -246,12 +245,6 @@ export class AtlassianAuth { static create(options: OAuthApiCreateOptions): typeof atlassianAuthApiRef.T; } -// @public @deprecated -export class Auth0Auth { - // (undocumented) - static create(options: OAuthApiCreateOptions): typeof auth0AuthApiRef.T; -} - // @public export type AuthApiCreateOptions = { discoveryApi: DiscoveryApi; @@ -383,21 +376,8 @@ export type FlatRoutesProps = { export class GithubAuth { // (undocumented) static create(options: OAuthApiCreateOptions): typeof githubAuthApiRef.T; - // @deprecated (undocumented) - static normalizeScope(scope?: string): Set; } -// @public @deprecated -export type GithubSession = { - providerInfo: { - accessToken: string; - scopes: Set; - expiresAt?: Date; - }; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - // @public export class GitlabAuth { // (undocumented) @@ -539,13 +519,6 @@ export class SamlAuth signOut(): Promise; } -// @public @deprecated -export type SamlSession = { - userId: string; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - // @public export type SignInPageProps = { onSignInSuccess(identityApi: IdentityApi): void; diff --git a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts index d81cba48d2..2f82ed5e88 100644 --- a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts +++ b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.test.ts @@ -33,12 +33,11 @@ describe('WebStorage Storage API', () => { it('should return undefined for values which are unset', async () => { const storage = createWebStorage(); - expect(storage.get('myfakekey')).toBeUndefined(); + expect(storage.snapshot('myfakekey').value).toBeUndefined(); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'absent', value: undefined, - newValue: undefined, }); }); @@ -48,26 +47,23 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', 'helloimastring'); await storage.set('mysecondfakekey', 1234); await storage.set('mythirdfakekey', true); - expect(storage.get('myfakekey')).toBe('helloimastring'); - expect(storage.get('mysecondfakekey')).toBe(1234); - expect(storage.get('mythirdfakekey')).toBe(true); + expect(storage.snapshot('myfakekey').value).toBe('helloimastring'); + expect(storage.snapshot('mysecondfakekey').value).toBe(1234); + expect(storage.snapshot('mythirdfakekey').value).toBe(true); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: 'helloimastring', - newValue: 'helloimastring', }); expect(storage.snapshot('mysecondfakekey')).toEqual({ key: 'mysecondfakekey', presence: 'present', value: 1234, - newValue: 1234, }); expect(storage.snapshot('mythirdfakekey')).toEqual({ key: 'mythirdfakekey', presence: 'present', value: true, - newValue: true, }); }); @@ -81,12 +77,11 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', mockData); - expect(storage.get('myfakekey')).toEqual(mockData); + expect(storage.snapshot('myfakekey').value).toEqual(mockData); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -118,7 +113,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -152,7 +146,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'absent', value: undefined, - newValue: undefined, }); }); @@ -166,9 +159,11 @@ describe('WebStorage Storage API', () => { await firstStorage.set(keyName, 'boop'); await secondStorage.set(keyName, 'deerp'); - expect(firstStorage.get(keyName)).not.toBe(secondStorage.get(keyName)); - expect(firstStorage.get(keyName)).toBe('boop'); - expect(secondStorage.get(keyName)).toBe('deerp'); + expect(firstStorage.snapshot(keyName)).not.toBe( + secondStorage.snapshot(keyName), + ); + expect(firstStorage.snapshot(keyName).value).toBe('boop'); + expect(secondStorage.snapshot(keyName).value).toBe('deerp'); expect(firstStorage.snapshot(keyName)).not.toEqual( secondStorage.snapshot(keyName), ); @@ -176,13 +171,11 @@ describe('WebStorage Storage API', () => { key: keyName, presence: 'present', value: 'boop', - newValue: 'boop', }); expect(secondStorage.snapshot(keyName)).toEqual({ key: keyName, presence: 'present', value: 'deerp', - newValue: 'deerp', }); }); @@ -217,7 +210,6 @@ describe('WebStorage Storage API', () => { key: 'key', presence: 'absent', value: undefined, - newValue: undefined, }); expect(mockErrorApi.post).toHaveBeenCalledWith(expect.any(Error)); expect(mockErrorApi.post).toHaveBeenCalledWith( diff --git a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts index c4cffaf184..ac3d3f20f2 100644 --- a/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts +++ b/packages/core-app-api/src/apis/implementations/StorageApi/WebStorage.ts @@ -65,7 +65,7 @@ export class WebStorage implements StorageApi { new Error(`Error when parsing JSON config from storage for: ${key}`), ); } - return { key, value, newValue: value, presence }; + return { key, value, presence }; } forBucket(name: string): WebStorage { diff --git a/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts b/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts deleted file mode 100644 index d8f942b94b..0000000000 --- a/packages/core-app-api/src/apis/implementations/auth/auth0/Auth0Auth.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { auth0AuthApiRef } from '@backstage/core-plugin-api'; -import { OAuth2 } from '../oauth2'; -import { OAuthApiCreateOptions } from '../types'; - -const DEFAULT_PROVIDER = { - id: 'auth0', - title: 'Auth0', - icon: () => null, -}; - -/** - * Implements the OAuth flow to Auth0 products. - * - * @public - * @deprecated Use {@link OAuth2} instead - * - * @example - * - * ```ts - * OAuth2.create({ - * discoveryApi, - * oauthRequestApi, - * provider: { - * id: 'auth0', - * title: 'Auth0', - * icon: () => null, - * }, - * defaultScopes: ['openid', 'email', 'profile'], - * environment: configApi.getOptionalString('auth.environment'), - * }) - * ``` - */ -export default class Auth0Auth { - static create(options: OAuthApiCreateOptions): typeof auth0AuthApiRef.T { - const { - discoveryApi, - environment = 'development', - provider = DEFAULT_PROVIDER, - oauthRequestApi, - defaultScopes = ['openid', `email`, `profile`], - } = options; - - return OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider, - environment, - defaultScopes, - }); - } -} diff --git a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts index b0af4c7ade..7efe4e95c6 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/GithubAuth.ts @@ -47,19 +47,4 @@ export default class GithubAuth { defaultScopes, }); } - - /** - * @deprecated This method is deprecated and will be removed in a future release. - */ - static normalizeScope(scope?: string): Set { - if (!scope) { - return new Set(); - } - - const scopeList = Array.isArray(scope) - ? scope - : scope.split(/[\s|,]/).filter(Boolean); - - return new Set(scopeList); - } } diff --git a/packages/core-app-api/src/apis/implementations/auth/github/index.ts b/packages/core-app-api/src/apis/implementations/auth/github/index.ts index b5aa1a0a25..5e53bacfc1 100644 --- a/packages/core-app-api/src/apis/implementations/auth/github/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/github/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ -export type { GithubSession } from './types'; export { default as GithubAuth } from './GithubAuth'; diff --git a/packages/core-app-api/src/apis/implementations/auth/github/types.ts b/packages/core-app-api/src/apis/implementations/auth/github/types.ts deleted file mode 100644 index 0ef662905f..0000000000 --- a/packages/core-app-api/src/apis/implementations/auth/github/types.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - ProfileInfo, - BackstageIdentityResponse, -} from '@backstage/core-plugin-api'; -import { z } from 'zod'; - -// TODO(Rugvip): Make GithubSession internal - -/** - * Session information for GitHub auth. - * - * @public - * @deprecated This type is internal and will be removed - */ -export type GithubSession = { - providerInfo: { - accessToken: string; - scopes: Set; - expiresAt?: Date; - }; - profile: ProfileInfo; - // TODO(Rugvip): This should be made optional once the type is no longer public - backstageIdentity: BackstageIdentityResponse; -}; - -export const githubSessionSchema: z.ZodSchema = z.object({ - providerInfo: z.object({ - accessToken: z.string(), - scopes: z.set(z.string()), - expiresAt: z.date().optional(), - }), - profile: z.object({ - email: z.string().optional(), - displayName: z.string().optional(), - picture: z.string().optional(), - }), - backstageIdentity: z.object({ - id: z.string(), - token: z.string(), - identity: z.object({ - type: z.literal('user'), - userEntityRef: z.string(), - ownershipEntityRefs: z.array(z.string()), - }), - }), -}); diff --git a/packages/core-app-api/src/apis/implementations/auth/index.ts b/packages/core-app-api/src/apis/implementations/auth/index.ts index 50333f07a0..c4cf520db5 100644 --- a/packages/core-app-api/src/apis/implementations/auth/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/index.ts @@ -20,7 +20,6 @@ export * from './google'; export * from './oauth2'; export * from './okta'; export * from './saml'; -export * from './auth0'; export * from './microsoft'; export * from './onelogin'; export * from './bitbucket'; diff --git a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts index 2e749a0648..930e6cb115 100644 --- a/packages/core-app-api/src/apis/implementations/auth/saml/index.ts +++ b/packages/core-app-api/src/apis/implementations/auth/saml/index.ts @@ -14,4 +14,3 @@ * limitations under the License. */ export { default as SamlAuth } from './SamlAuth'; -export type { ExportedSamlSession as SamlSession } from './types'; diff --git a/packages/core-app-api/src/apis/implementations/auth/saml/types.ts b/packages/core-app-api/src/apis/implementations/auth/saml/types.ts index f1345fd154..55b278b429 100644 --- a/packages/core-app-api/src/apis/implementations/auth/saml/types.ts +++ b/packages/core-app-api/src/apis/implementations/auth/saml/types.ts @@ -20,18 +20,6 @@ import { } from '@backstage/core-plugin-api'; import { z } from 'zod'; -/** - * Session information for SAML auth. - * - * @public - * @deprecated This type is internal and will be removed - */ -export type ExportedSamlSession = { - userId: string; - profile: ProfileInfo; - backstageIdentity: BackstageIdentityResponse; -}; - /** @internal */ export type SamlSession = { profile: ProfileInfo; diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx index e62b3fa82d..3348c278ee 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.test.tsx @@ -75,7 +75,7 @@ describe('', () => { ); fireEvent.click(button); const dismissedBanners = - notifications?.get('dismissedBanners') ?? []; + notifications?.snapshot('dismissedBanners').value ?? []; expect( dismissedBanners.includes('catalog_page_welcome_banner'), ).toBeTruthy(); diff --git a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx index 566b326cb5..9ff7616c45 100644 --- a/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx +++ b/packages/core-components/src/components/DismissableBanner/DismissableBanner.tsx @@ -101,7 +101,7 @@ export const DismissableBanner = (props: Props) => { const storageApi = useApi(storageApiRef); const notificationsStore = storageApi.forBucket('notifications'); const rawDismissedBanners = - notificationsStore.get('dismissedBanners') ?? []; + notificationsStore.snapshot('dismissedBanners').value ?? []; const [dismissedBanners, setDismissedBanners] = useState( new Set(rawDismissedBanners), @@ -112,11 +112,11 @@ export const DismissableBanner = (props: Props) => { ); useEffect(() => { - if (observedItems?.newValue) { - const currentValue = observedItems?.newValue ?? []; + if (observedItems?.value) { + const currentValue = observedItems?.value ?? []; setDismissedBanners(new Set(currentValue)); } - }, [observedItems?.newValue]); + }, [observedItems?.value]); const handleClick = () => { notificationsStore.set('dismissedBanners', [...dismissedBanners, id]); diff --git a/packages/core-components/src/layout/SignInPage/auth0Provider.tsx b/packages/core-components/src/layout/SignInPage/auth0Provider.tsx deleted file mode 100644 index 739c3709a9..0000000000 --- a/packages/core-components/src/layout/SignInPage/auth0Provider.tsx +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import React from 'react'; -import Grid from '@material-ui/core/Grid'; -import Typography from '@material-ui/core/Typography'; -import Button from '@material-ui/core/Button'; -import { InfoCard } from '../InfoCard/InfoCard'; -import { ProviderComponent, ProviderLoader, SignInProvider } from './types'; -import { - useApi, - auth0AuthApiRef, - errorApiRef, -} from '@backstage/core-plugin-api'; -import { ForwardedError } from '@backstage/errors'; -import { UserIdentity } from './UserIdentity'; - -const Component: ProviderComponent = ({ onSignInSuccess }) => { - const auth0AuthApi = useApi(auth0AuthApiRef); - const errorApi = useApi(errorApiRef); - - const handleLogin = async () => { - try { - const identityResponse = await auth0AuthApi.getBackstageIdentity({ - instantPopup: true, - }); - if (!identityResponse) { - throw new Error( - 'The Auth0 provider is not configured to support sign-in', - ); - } - - const profile = await auth0AuthApi.getProfile(); - - onSignInSuccess( - UserIdentity.create({ - identity: identityResponse.identity, - authApi: auth0AuthApi, - profile, - }), - ); - } catch (error) { - errorApi.post(new ForwardedError('Auth0 login failed', error)); - } - }; - - return ( - - - Sign In - - } - > - Sign In using Auth0 - - - ); -}; - -const loader: ProviderLoader = async apis => { - const auth0AuthApi = apis.get(auth0AuthApiRef)!; - - const identityResponse = await auth0AuthApi.getBackstageIdentity({ - optional: true, - }); - - if (!identityResponse) { - return undefined; - } - - const profile = await auth0AuthApi.getProfile(); - return UserIdentity.create({ - identity: identityResponse.identity, - authApi: auth0AuthApi, - profile, - }); -}; - -export const auth0Provider: SignInProvider = { Component, loader }; diff --git a/packages/core-plugin-api/api-report.md b/packages/core-plugin-api/api-report.md index 8c49deec2b..89361419ba 100644 --- a/packages/core-plugin-api/api-report.md +++ b/packages/core-plugin-api/api-report.md @@ -184,11 +184,6 @@ export function attachComponentData

( data: unknown, ): void; -// @public @deprecated -export const auth0AuthApiRef: ApiRef< - OpenIdConnectApi & ProfileInfoApi & BackstageIdentityApi & SessionApi ->; - // @public export type AuthProviderInfo = { id: string; @@ -531,15 +526,6 @@ export const microsoftAuthApiRef: ApiRef< SessionApi >; -// @public @deprecated -export const oauth2ApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi ->; - // @public export type OAuthApi = { getAccessToken( @@ -573,15 +559,6 @@ export type OAuthRequesterOptions = { // @public export type OAuthScope = string | string[]; -// @public @deprecated -export const oidcAuthApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi ->; - // @alpha export const oktaAuthApiRef: ApiRef< OAuthApi & @@ -680,11 +657,6 @@ export type RouteRef = { params: ParamKeys; }; -// @public @deprecated -export const samlAuthApiRef: ApiRef< - ProfileInfoApi & BackstageIdentityApi & SessionApi ->; - // @public export type SessionApi = { signIn(): Promise; @@ -706,8 +678,6 @@ export type SignInPageProps = { // @public export interface StorageApi { forBucket(name: string): StorageApi; - // @deprecated - get(key: string): T | undefined; observe$( key: string, ): Observable>; @@ -719,23 +689,17 @@ export interface StorageApi { // @public export const storageApiRef: ApiRef; -// @public @deprecated (undocumented) -export type StorageValueChange = - StorageValueSnapshot; - // @public export type StorageValueSnapshot = | { key: string; presence: 'unknown' | 'absent'; value?: undefined; - newValue?: undefined; } | { key: string; presence: 'present'; value: TValue; - newValue?: TValue; }; // @public diff --git a/packages/core-plugin-api/src/apis/definitions/StorageApi.ts b/packages/core-plugin-api/src/apis/definitions/StorageApi.ts index 4bd49f9b83..1506e5f143 100644 --- a/packages/core-plugin-api/src/apis/definitions/StorageApi.ts +++ b/packages/core-plugin-api/src/apis/definitions/StorageApi.ts @@ -27,24 +27,13 @@ export type StorageValueSnapshot = key: string; presence: 'unknown' | 'absent'; value?: undefined; - /** @deprecated Use `value` instead */ - newValue?: undefined; } | { key: string; presence: 'present'; value: TValue; - /** @deprecated Use `value` instead */ - newValue?: TValue; }; -/** - * @public - * @deprecated Use StorageValueSnapshot instead - */ -export type StorageValueChange = - StorageValueSnapshot; - /** * Provides a key-value persistence API. * @@ -59,14 +48,6 @@ export interface StorageApi { */ forBucket(name: string): StorageApi; - /** - * Get the current value for persistent data, use observe$ to be notified of updates. - * - * @deprecated Use `snapshot` instead. - * @param key - Unique key associated with the data. - */ - get(key: string): T | undefined; - /** * Remove persistent data. * diff --git a/packages/core-plugin-api/src/apis/definitions/auth.ts b/packages/core-plugin-api/src/apis/definitions/auth.ts index ed5a7a3264..497ce8233b 100644 --- a/packages/core-plugin-api/src/apis/definitions/auth.ts +++ b/packages/core-plugin-api/src/apis/definitions/auth.ts @@ -366,23 +366,6 @@ export const gitlabAuthApiRef: ApiRef< id: 'core.auth.gitlab', }); -/** - * Provides authentication towards Auth0 APIs. - * - * @remarks - * - * See {@link https://auth0.com/docs/scopes/current/oidc-scopes} - * for a full list of supported scopes. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const auth0AuthApiRef: ApiRef< - OpenIdConnectApi & ProfileInfoApi & BackstageIdentityApi & SessionApi -> = createApiRef({ - id: 'core.auth.auth0', -}); - /** * Provides authentication towards Microsoft APIs and identities. * @@ -404,50 +387,6 @@ export const microsoftAuthApiRef: ApiRef< id: 'core.auth.microsoft', }); -/** - * Provides authentication for custom identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const oauth2ApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi -> = createApiRef({ - id: 'core.auth.oauth2', -}); - -/** - * Provides authentication for custom OpenID Connect identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const oidcAuthApiRef: ApiRef< - OAuthApi & - OpenIdConnectApi & - ProfileInfoApi & - BackstageIdentityApi & - SessionApi -> = createApiRef({ - id: 'core.auth.oidc', -}); - -/** - * Provides authentication for SAML-based identity providers. - * - * @public - * @deprecated See https://backstage.io/docs/api/deprecations#generic-auth-api-refs - */ -export const samlAuthApiRef: ApiRef< - ProfileInfoApi & BackstageIdentityApi & SessionApi -> = createApiRef({ - id: 'core.auth.saml', -}); - /** * Provides authentication towards OneLogin APIs. * diff --git a/packages/create-app/templates/default-app/packages/app/src/App.tsx b/packages/create-app/templates/default-app/packages/app/src/App.tsx index 6f00993273..f4ff424926 100644 --- a/packages/create-app/templates/default-app/packages/app/src/App.tsx +++ b/packages/create-app/templates/default-app/packages/app/src/App.tsx @@ -30,7 +30,7 @@ import { createApp } from '@backstage/app-defaults'; import { FlatRoutes } from '@backstage/core-app-api'; import { CatalogGraphPage } from '@backstage/plugin-catalog-graph'; import { PermissionedRoute } from '@backstage/plugin-permission-react'; -import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common'; +import { catalogEntityCreatePermission } from '@backstage/plugin-catalog-common/alpha'; const app = createApp({ apis, diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts index a0a1cc3701..c359cb4986 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts @@ -5,8 +5,8 @@ import { LunrSearchEngine, } from '@backstage/plugin-search-backend-node'; import { PluginEnvironment } from '../types'; -import { DefaultCatalogCollator } from '@backstage/plugin-catalog-backend'; -import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend'; +import { DefaultCatalogCollatorFactory } from '@backstage/plugin-catalog-backend'; +import { DefaultTechDocsCollatorFactory } from '@backstage/plugin-techdocs-backend'; export default async function createPlugin({ logger, @@ -23,7 +23,7 @@ export default async function createPlugin({ // collator gathers entities from the software catalog. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultCatalogCollator.fromConfig(config, { + factory: DefaultCatalogCollatorFactory.fromConfig(config, { discovery, tokenManager, }), @@ -32,7 +32,7 @@ export default async function createPlugin({ // collator gathers entities from techdocs. indexBuilder.addCollator({ defaultRefreshIntervalSeconds: 600, - collator: DefaultTechDocsCollator.fromConfig(config, { + factory: DefaultTechDocsCollatorFactory.fromConfig(config, { discovery, logger, tokenManager, diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 10da05784c..9a6ba03f8e 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -88,7 +88,7 @@ export class BitbucketIntegration implements ScmIntegration { // @public export type BitbucketIntegrationConfig = { host: string; - apiBaseUrl?: string; + apiBaseUrl: string; token?: string; username?: string; appPassword?: string; diff --git a/packages/integration/src/bitbucket/config.ts b/packages/integration/src/bitbucket/config.ts index 1cd911aad3..44a2f0cc1f 100644 --- a/packages/integration/src/bitbucket/config.ts +++ b/packages/integration/src/bitbucket/config.ts @@ -36,12 +36,10 @@ export type BitbucketIntegrationConfig = { * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", * with no trailing slash. * - * May be omitted specifically for Bitbucket Cloud; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. + * Values omitted at the optional property at the app-config will be deduced + * from the "host" value. */ - apiBaseUrl?: string; + apiBaseUrl: string; /** * The authorization token to use for requests to a Bitbucket Server provider. @@ -90,6 +88,8 @@ export function readBitbucketIntegrationConfig( apiBaseUrl = trimEnd(apiBaseUrl, '/'); } else if (host === BITBUCKET_HOST) { apiBaseUrl = BITBUCKET_API_BASE_URL; + } else { + apiBaseUrl = `https://${host}/rest/api/1.0`; } return { diff --git a/packages/integration/src/helpers.test.ts b/packages/integration/src/helpers.test.ts index a69c7faec9..60a2789f05 100644 --- a/packages/integration/src/helpers.test.ts +++ b/packages/integration/src/helpers.test.ts @@ -24,7 +24,10 @@ import { describe('basicIntegrations', () => { describe('byUrl', () => { it('handles hosts without a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com' }); + const integration = new BitbucketIntegration({ + host: 'host.com', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, @@ -33,7 +36,10 @@ describe('basicIntegrations', () => { expect(integrations.byUrl('https://host.com:8080/a')).toBeUndefined(); }); it('handles hosts with a port', () => { - const integration = new BitbucketIntegration({ host: 'host.com:8080' }); + const integration = new BitbucketIntegration({ + host: 'host.com:8080', + apiBaseUrl: 'a', + }); const integrations = basicIntegrations( [integration], i => i.config.host, diff --git a/packages/search-common/api-report.md b/packages/search-common/api-report.md index 65f9f25180..475d85e11c 100644 --- a/packages/search-common/api-report.md +++ b/packages/search-common/api-report.md @@ -3,38 +3,33 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { JsonObject } from '@backstage/types'; import { Permission } from '@backstage/plugin-permission-common'; +import { Readable } from 'stream'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export interface DocumentCollator { - // (undocumented) - execute(): Promise; +// @beta +export interface DocumentCollatorFactory { + getCollator(): Promise; readonly type: string; readonly visibilityPermission?: Permission; } -// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export interface DocumentDecorator { - // (undocumented) - execute(documents: IndexableDocument[]): Promise; +// @beta +export interface DocumentDecoratorFactory { + getDecorator(): Promise; readonly types?: string[]; } -// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type DocumentTypeInfo = { visibilityPermission?: Permission; }; -// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface IndexableDocument { authorization?: { resourceRef: string; @@ -44,23 +39,17 @@ export interface IndexableDocument { title: string; } -// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta export type QueryRequestOptions = { token?: string; }; -// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export type QueryTranslator = (query: SearchQuery) => unknown; -// Warning: (ae-missing-release-tag) "SearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta export interface SearchEngine { - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; query( query: SearchQuery, options?: QueryRequestOptions, @@ -68,9 +57,7 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; } -// Warning: (ae-missing-release-tag) "SearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchQuery { // (undocumented) filters?: JsonObject; @@ -82,9 +69,7 @@ export interface SearchQuery { types?: string[]; } -// Warning: (ae-missing-release-tag) "SearchResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResult { // (undocumented) document: IndexableDocument; @@ -92,9 +77,7 @@ export interface SearchResult { type: string; } -// Warning: (ae-missing-release-tag) "SearchResultSet" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) export interface SearchResultSet { // (undocumented) nextPageCursor?: string; diff --git a/packages/search-common/src/types.ts b/packages/search-common/src/types.ts index 4e61767ea0..51ce45617b 100644 --- a/packages/search-common/src/types.ts +++ b/packages/search-common/src/types.ts @@ -16,7 +16,11 @@ import { Permission } from '@backstage/plugin-permission-common'; import { JsonObject } from '@backstage/types'; +import { Readable, Transform, Writable } from 'stream'; +/** + * @beta + */ export interface SearchQuery { term: string; filters?: JsonObject; @@ -24,11 +28,17 @@ export interface SearchQuery { pageCursor?: string; } +/** + * @beta + */ export interface SearchResult { type: string; document: IndexableDocument; } +/** + * @beta + */ export interface SearchResultSet { results: SearchResult[]; nextPageCursor?: string; @@ -38,6 +48,7 @@ export interface SearchResultSet { /** * Base properties that all indexed documents must include, as well as some * common properties that documents are encouraged to use where appropriate. + * @beta */ export interface IndexableDocument { /** @@ -72,6 +83,7 @@ export interface IndexableDocument { * Information about a specific document type. Intended to be used in the * {@link @backstage/search-backend-node#IndexBuilder} to collect information * about the types stored in the index. + * @beta */ export type DocumentTypeInfo = { /** @@ -82,10 +94,10 @@ export type DocumentTypeInfo = { }; /** - * Interface that must be implemented in order to expose new documents to - * search. + * Factory class for instantiating collators. + * @beta */ -export interface DocumentCollator { +export interface DocumentCollatorFactory { /** * The type or name of the document set returned by this collator. Used as an * index name by Search Engines. @@ -98,29 +110,41 @@ export interface DocumentCollator { */ readonly visibilityPermission?: Permission; - execute(): Promise; + /** + * Instantiates and resolves a document collator. + */ + getCollator(): Promise; } /** - * Interface that must be implemented in order to decorate existing documents with - * additional metadata. + * Factory class for instantiating decorators. + * @beta */ -export interface DocumentDecorator { +export interface DocumentDecoratorFactory { /** * An optional array of document/index types on which this decorator should * be applied. If no types are provided, this decorator will be applied to * all document/index types. */ readonly types?: string[]; - execute(documents: IndexableDocument[]): Promise; + + /** + * Instantiates and resolves a document decorator. + */ + getDecorator(): Promise; } /** * A type of function responsible for translating an abstract search query into * a concrete query relevant to a particular search engine. + * @beta */ export type QueryTranslator = (query: SearchQuery) => unknown; +/** + * Options when querying a search engine. + * @beta + */ export type QueryRequestOptions = { token?: string; }; @@ -129,6 +153,7 @@ export type QueryRequestOptions = { * Interface that must be implemented by specific search engines, responsible * for performing indexing and querying and translating abstract queries into * concrete, search engine-specific queries. + * @beta */ export interface SearchEngine { /** @@ -137,9 +162,15 @@ export interface SearchEngine { setTranslator(translator: QueryTranslator): void; /** - * Add the given documents to the SearchEngine index of the given type. + * Factory method for getting a search engine indexer for a given document + * type. + * + * @param type - The type or name of the document set for which an indexer + * should be retrieved. This corresponds to the `type` property on the + * document collator/decorator factories and will most often be used to + * identify an index or group to which documents should be written. */ - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; /** * Perform a search query against the SearchEngine. diff --git a/packages/techdocs-cli-embedded-app/src/apis.ts b/packages/techdocs-cli-embedded-app/src/apis.ts index 6234e3b719..1fe6ba8086 100644 --- a/packages/techdocs-cli-embedded-app/src/apis.ts +++ b/packages/techdocs-cli-embedded-app/src/apis.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { scmIntegrationsApiRef, @@ -81,7 +81,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return this.configApi.getString('techdocs.builder'); } - async getEntityDocs(_entityId: EntityName, path: string) { + async getEntityDocs(_entityId: CompoundEntityRef, path: string) { const apiOrigin = await this.getApiOrigin(); // Irrespective of the entity, use mkdocs server to find the file for the path. const url = `${apiOrigin}/${path}`; @@ -97,7 +97,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { return request.text(); } - async syncEntityDocs(_: EntityName): Promise { + async syncEntityDocs(_: CompoundEntityRef): Promise { // this is just stub of this function as we don't need to check if docs are up to date, // we always want to retrigger a new build return 'cached'; @@ -106,7 +106,7 @@ class TechDocsDevStorageApi implements TechDocsStorageApi { // Used by transformer to modify the request to assets (CSS, Image) from inside the HTML. async getBaseUrl( oldBaseUrl: string, - _entityId: EntityName, + _entityId: CompoundEntityRef, path: string, ): Promise { const apiOrigin = await this.getApiOrigin(); @@ -154,7 +154,7 @@ class TechDocsDevApi implements TechDocsApi { }; } - async getTechDocsMetadata(_entityId: EntityName) { + async getTechDocsMetadata(_entityId: CompoundEntityRef) { return { site_name: 'Live preview environment', site_description: '', diff --git a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx index d9a518c165..f2dcefc00e 100644 --- a/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx +++ b/packages/techdocs-cli-embedded-app/src/components/TechDocsPage/TechDocsPage.tsx @@ -29,7 +29,7 @@ import LightIcon from '@material-ui/icons/Brightness7'; import DarkIcon from '@material-ui/icons/Brightness4'; import { lightTheme, darkTheme } from '@backstage/theme'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Content } from '@backstage/core-components'; @@ -127,7 +127,7 @@ const TechDocsPageContent = ({ onReady, entityRef, }: { - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => { const classes = useStyles(); diff --git a/packages/techdocs-cli/src/commands/generate/generate.ts b/packages/techdocs-cli/src/commands/generate/generate.ts index a255805139..0f5213df34 100644 --- a/packages/techdocs-cli/src/commands/generate/generate.ts +++ b/packages/techdocs-cli/src/commands/generate/generate.ts @@ -39,6 +39,7 @@ export default async function generate(cmd: Command) { const sourceDir = resolve(cmd.sourceDir); const outputDir = resolve(cmd.outputDir); + const omitTechdocsCorePlugin = cmd.omitTechdocsCoreMkdocsPlugin; const dockerImage = cmd.dockerImage; const pullImage = cmd.pull; @@ -55,6 +56,9 @@ export default async function generate(cmd: Command) { runIn: cmd.docker ? 'docker' : 'local', dockerImage, pullImage, + mkdocs: { + omitTechdocsCorePlugin, + }, }, }, }); diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index 115fe8e364..6f1d9847e1 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -54,6 +54,11 @@ export function registerCommands(program: CommanderStatic) { 'A unique identifier for the prepared tree e.g. commit SHA. If provided it will be stored in techdocs_metadata.json.', ) .option('-v --verbose', 'Enable verbose output.', false) + .option( + '--omitTechdocsCoreMkdocsPlugin', + "Don't patch MkDocs file automatically with techdocs-core plugin.", + false, + ) .alias('build') .action(lazy(() => import('./generate/generate').then(m => m.default))); diff --git a/packages/techdocs-common/api-report.md b/packages/techdocs-common/api-report.md index 7c4fc95abb..13d610f66b 100644 --- a/packages/techdocs-common/api-report.md +++ b/packages/techdocs-common/api-report.md @@ -5,10 +5,10 @@ ```ts /// +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { ContainerRunner } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; @@ -157,7 +157,9 @@ export class Publisher { // @public export interface PublisherBase { docsRouter(): express.Handler; - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; getReadiness(): Promise; hasDocsBeenGenerated(entityName: Entity): Promise; migrateDocsCase?(migrateRequest: MigrateRequest): Promise; diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml new file mode 100644 index 0000000000..09e8fd7ac7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_additional_plugins.yml @@ -0,0 +1,6 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ +plugins: + - not-techdocs-core + - also-not-techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml new file mode 100644 index 0000000000..eea9a8a3d9 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_techdocs_plugin.yml @@ -0,0 +1,5 @@ +site_name: Test site name +site_description: Test site description +# This is a comment that is removed after editing +plugins: + - techdocs-core diff --git a/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml new file mode 100644 index 0000000000..e75b06ada7 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_without_plugins.yml @@ -0,0 +1,3 @@ +site_name: Test site name +site_description: Test site description +docs_dir: docs/ diff --git a/packages/techdocs-common/src/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts index 9323ab4f18..7590df94fb 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -28,10 +28,14 @@ import { getMkdocsYml, getRepoUrlFromLocationAnnotation, patchIndexPreBuild, - patchMkdocsYmlPreBuild, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; +import yaml from 'js-yaml'; const mockEntity = { apiVersion: 'version', @@ -65,6 +69,15 @@ const mkdocsYmlWithInvalidDocDir2 = fs.readFileSync( const mkdocsYmlWithComments = fs.readFileSync( resolvePath(__filename, '../__fixtures__/mkdocs_with_comments.yml'), ); +const mkdocsYmlWithTechdocsPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_techdocs_plugin.yml'), +); +const mkdocsYmlWithoutPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_without_plugins.yml'), +); +const mkdocsYmlWithAdditionalPlugins = fs.readFileSync( + resolvePath(__filename, '../__fixtures__/mkdocs_with_additional_plugins.yml'), +); const mockLogger = getVoidLogger(); const warn = jest.spyOn(mockLogger, 'warn'); @@ -289,6 +302,60 @@ describe('helpers', () => { }); }); + describe('pathMkdocsYmlWithTechdocsPlugin', () => { + beforeEach(() => { + mockFs({ + '/mkdocs_with_techdocs_plugin.yml': mkdocsYmlWithTechdocsPlugins, + '/mkdocs_without_plugins.yml': mkdocsYmlWithoutPlugins, + '/mkdocs_with_additional_plugins.yml': mkdocsYmlWithAdditionalPlugins, + }); + }); + it('should not add additional plugins if techdocs exists already in mkdocs file', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_techdocs_plugin.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_techdocs_plugin.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it("should add the needed plugin if it doesn't exist in mkdocs file", async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_without_plugins.yml', + mockLogger, + ); + + const updatedMkdocsYml = await fs.readFile('/mkdocs_without_plugins.yml'); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(1); + expect(parsedYml.plugins).toContain('techdocs-core'); + }); + it('should not override existing plugins', async () => { + await pathMkdocsYmlWithTechdocsPlugin( + '/mkdocs_with_additional_plugins.yml', + mockLogger, + ); + const updatedMkdocsYml = await fs.readFile( + '/mkdocs_with_additional_plugins.yml', + ); + const parsedYml = yaml.load(updatedMkdocsYml.toString()) as { + plugins: string[]; + }; + expect(parsedYml.plugins).toHaveLength(3); + expect(parsedYml.plugins).toContain('techdocs-core'); + expect(parsedYml.plugins).toContain('not-techdocs-core'); + expect(parsedYml.plugins).toContain('also-not-techdocs-core'); + }); + }); + describe('patchIndexPreBuild', () => { afterEach(() => { warn.mockClear(); diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index 2debd5822c..349783d2bd 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -125,7 +125,7 @@ class UnknownTag { constructor(public readonly data: any, public readonly type?: string) {} } -const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ +export const MKDOCS_SCHEMA = DEFAULT_SCHEMA.extend([ new Type('', { kind: 'scalar', multi: true, @@ -203,101 +203,6 @@ export const validateMkdocsYaml = async ( return parsedMkdocsYml.docs_dir; }; -/** - * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. - * - * List of tasks: - * - Add repo_url or edit_uri if it does not exists - * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. - * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get - * the repository URL. - * - * This function will not throw an error since this is not critical to the whole TechDocs pipeline. - * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. - * - * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site - * @param logger - A logger instance - * @param parsedLocationAnnotation - Object with location url and type - * @param scmIntegrations - the scmIntegration to do url transformations - */ -export const patchMkdocsYmlPreBuild = async ( - mkdocsYmlPath: string, - logger: Logger, - parsedLocationAnnotation: ParsedLocationAnnotation, - scmIntegrations: ScmIntegrationRegistry, -) => { - // We only want to override the mkdocs.yml if it has actually changed. This is relevant if - // used with a 'dir' location on the file system as this would permanently update the file. - let didEdit = false; - - let mkdocsYmlFileString; - try { - mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); - } catch (error) { - assertError(error); - logger.warn( - `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, - ); - return; - } - - let mkdocsYml: any; - try { - mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); - - // mkdocsYml should be an object type after successful parsing. - // But based on its type definition, it can also be a string or undefined, which we don't want. - if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { - throw new Error('Bad YAML format.'); - } - } catch (error) { - assertError(error); - logger.warn( - `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, - ); - return; - } - - // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. - // This will enable the Page edit button generated by MkDocs. - // If the either has been set, keep the original value - if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { - const result = getRepoUrlFromLocationAnnotation( - parsedLocationAnnotation, - scmIntegrations, - mkdocsYml.docs_dir, - ); - - if (result.repo_url || result.edit_uri) { - mkdocsYml.repo_url = result.repo_url; - mkdocsYml.edit_uri = result.edit_uri; - didEdit = true; - - logger.info( - `Set ${JSON.stringify( - result, - )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, - ); - } - } - - try { - if (didEdit) { - await fs.writeFile( - mkdocsYmlPath, - yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), - 'utf8', - ); - } - } catch (error) { - assertError(error); - logger.warn( - `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, - ); - return; - } -}; - /** * Update docs/index.md file before TechDocs generator uses it to generate docs site, * falling back to docs/README.md or README.md in case a default docs/index.md diff --git a/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts new file mode 100644 index 0000000000..d03b5d83c2 --- /dev/null +++ b/packages/techdocs-common/src/stages/generate/mkDocsPatchers.ts @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import fs from 'fs-extra'; +import yaml from 'js-yaml'; +import { ParsedLocationAnnotation } from '../../helpers'; +import { getRepoUrlFromLocationAnnotation, MKDOCS_SCHEMA } from './helpers'; +import { assertError } from '@backstage/errors'; +import { ScmIntegrationRegistry } from '@backstage/integration'; + +type MkDocsObject = { + plugins?: string[]; + docs_dir: string; + repo_url?: string; + edit_uri?: string; +}; + +const patchMkdocsFile = async ( + mkdocsYmlPath: string, + logger: Logger, + updateAction: (mkdocsYml: MkDocsObject) => boolean, +) => { + // We only want to override the mkdocs.yml if it has actually changed. This is relevant if + // used with a 'dir' location on the file system as this would permanently update the file. + let didEdit = false; + + let mkdocsYmlFileString; + try { + mkdocsYmlFileString = await fs.readFile(mkdocsYmlPath, 'utf8'); + } catch (error) { + assertError(error); + logger.warn( + `Could not read MkDocs YAML config file ${mkdocsYmlPath} before running the generator: ${error.message}`, + ); + return; + } + + let mkdocsYml: any; + try { + mkdocsYml = yaml.load(mkdocsYmlFileString, { schema: MKDOCS_SCHEMA }); + + // mkdocsYml should be an object type after successful parsing. + // But based on its type definition, it can also be a string or undefined, which we don't want. + if (typeof mkdocsYml === 'string' || typeof mkdocsYml === 'undefined') { + throw new Error('Bad YAML format.'); + } + } catch (error) { + assertError(error); + logger.warn( + `Error in parsing YAML at ${mkdocsYmlPath} before running the generator. ${error.message}`, + ); + return; + } + + didEdit = updateAction(mkdocsYml); + + try { + if (didEdit) { + await fs.writeFile( + mkdocsYmlPath, + yaml.dump(mkdocsYml, { schema: MKDOCS_SCHEMA }), + 'utf8', + ); + } + } catch (error) { + assertError(error); + logger.warn( + `Could not write to ${mkdocsYmlPath} after updating it before running the generator. ${error.message}`, + ); + return; + } +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add repo_url or edit_uri if it does not exists + * If mkdocs.yml has a repo_url, the generated docs site gets an Edit button on the pages by default. + * If repo_url is missing in mkdocs.yml, we will use techdocs annotation of the entity to possibly get + * the repository URL. + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + * @param parsedLocationAnnotation - Object with location url and type + * @param scmIntegrations - the scmIntegration to do url transformations + */ +export const patchMkdocsYmlPreBuild = async ( + mkdocsYmlPath: string, + logger: Logger, + parsedLocationAnnotation: ParsedLocationAnnotation, + scmIntegrations: ScmIntegrationRegistry, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + if (!('repo_url' in mkdocsYml) && !('edit_uri' in mkdocsYml)) { + // Add edit_uri and/or repo_url to mkdocs.yml if it is missing. + // This will enable the Page edit button generated by MkDocs. + // If the either has been set, keep the original value + const result = getRepoUrlFromLocationAnnotation( + parsedLocationAnnotation, + scmIntegrations, + mkdocsYml.docs_dir, + ); + + if (result.repo_url || result.edit_uri) { + mkdocsYml.repo_url = result.repo_url; + mkdocsYml.edit_uri = result.edit_uri; + + logger.info( + `Set ${JSON.stringify( + result, + )}. You can disable this feature by manually setting 'repo_url' or 'edit_uri' according to the MkDocs documentation at https://www.mkdocs.org/user-guide/configuration/#repo_url`, + ); + return true; + } + } + return false; + }); +}; + +/** + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. + * + * List of tasks: + * - Add techdocs-core plugin to mkdocs file if it doesn't exist + * + * This function will not throw an error since this is not critical to the whole TechDocs pipeline. + * Instead it will log warnings if there are any errors in reading, parsing or writing YAML. + * + * @param mkdocsYmlPath - Absolute path to mkdocs.yml or equivalent of a docs site + * @param logger - A logger instance + */ +export const pathMkdocsYmlWithTechdocsPlugin = async ( + mkdocsYmlPath: string, + logger: Logger, +) => { + await patchMkdocsFile(mkdocsYmlPath, logger, mkdocsYml => { + // Modify mkdocs.yaml to contain the needed techdocs-core plugin if it is not there + if (!('plugins' in mkdocsYml)) { + mkdocsYml.plugins = ['techdocs-core']; + return true; + } + + if (mkdocsYml.plugins && !mkdocsYml.plugins.includes('techdocs-core')) { + mkdocsYml.plugins.push('techdocs-core'); + return true; + } + return false; + }); +}; diff --git a/packages/techdocs-common/src/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts index baa2e008a1..b46f52f99a 100644 --- a/packages/techdocs-common/src/stages/generate/techdocs.ts +++ b/packages/techdocs-common/src/stages/generate/techdocs.ts @@ -26,11 +26,15 @@ import { createOrUpdateMetadata, getMkdocsYml, patchIndexPreBuild, - patchMkdocsYmlPreBuild, runCommand, storeEtagMetadata, validateMkdocsYaml, } from './helpers'; + +import { + patchMkdocsYmlPreBuild, + pathMkdocsYmlWithTechdocsPlugin, +} from './mkDocsPatchers'; import { GeneratorBase, GeneratorConfig, @@ -110,6 +114,10 @@ export class TechdocsGenerator implements GeneratorBase { await patchIndexPreBuild({ inputDir, logger: childLogger, docsDir }); } + if (!this.options.omitTechdocsCoreMkdocsPlugin) { + await pathMkdocsYmlWithTechdocsPlugin(mkdocsYmlPath, childLogger); + } + // Directories to bind on container const mountDirs = { [inputDir]: '/input', @@ -207,5 +215,8 @@ export function readGeneratorConfig( 'docker', dockerImage: config.getOptionalString('techdocs.generator.dockerImage'), pullImage: config.getOptionalBoolean('techdocs.generator.pullImage'), + omitTechdocsCoreMkdocsPlugin: config.getOptionalBoolean( + 'techdocs.generator.mkdocs.omitTechdocsCorePlugin', + ), }; } diff --git a/packages/techdocs-common/src/stages/generate/types.ts b/packages/techdocs-common/src/stages/generate/types.ts index 2ff3064991..f46dbdc56a 100644 --- a/packages/techdocs-common/src/stages/generate/types.ts +++ b/packages/techdocs-common/src/stages/generate/types.ts @@ -39,6 +39,7 @@ export type GeneratorConfig = { runIn: GeneratorRunInType; dockerImage?: string; pullImage?: boolean; + omitTechdocsCoreMkdocsPlugin?: boolean; }; /** diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index ed76edc0cb..e5914d2fdf 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import aws, { Credentials } from 'aws-sdk'; @@ -321,7 +321,7 @@ export class AwsS3Publish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { try { return await new Promise(async (resolve, reject) => { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index b082079be2..bcbc10a94d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -19,7 +19,7 @@ import { ContainerClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, ForwardedError } from '@backstage/errors'; import express from 'express'; @@ -300,7 +300,7 @@ export class AzureBlobStoragePublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityRootDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index c3d9d3330b..987dfc6f09 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError } from '@backstage/errors'; import { File, FileExistsResponse, Storage } from '@google-cloud/storage'; @@ -238,7 +238,9 @@ export class GoogleGCSPublish implements PublisherBase { return { objects }; } - fetchTechDocsMetadata(entityName: EntityName): Promise { + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise { return new Promise((resolve, reject) => { const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; const entityDir = this.legacyPathCasing diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 9c146d2935..af9bb30d46 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -17,7 +17,7 @@ import { PluginEndpointDiscovery, resolvePackagePath, } from '@backstage/backend-common'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -142,7 +142,7 @@ export class LocalPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const metadataPath = this.staticEntityPathJoin( entityName.namespace, diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index b240ce73da..da0a60343f 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; @@ -45,7 +45,7 @@ const createMockEntity = (annotations = {}): Entity => { }; }; -const createMockEntityName = (): EntityName => ({ +const createMockEntityName = (): CompoundEntityRef => ({ kind: 'TestKind', name: 'test-component-name', namespace: 'test-namespace', diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 62b40f9a76..734a7e4451 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import fs from 'fs-extra'; @@ -194,7 +194,7 @@ export class OpenStackSwiftPublish implements PublisherBase { } async fetchTechDocsMetadata( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { return await new Promise(async (resolve, reject) => { const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index c6ad032484..86772c25f4 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Logger } from 'winston'; import express from 'express'; @@ -133,7 +133,9 @@ export interface PublisherBase { * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. */ - fetchTechDocsMetadata(entityName: EntityName): Promise; + fetchTechDocsMetadata( + entityName: CompoundEntityRef, + ): Promise; /** * Route middleware to serve static documentation files for an entity. diff --git a/packages/test-utils/api-report.md b/packages/test-utils/api-report.md index 4e5361ff80..2f2671885e 100644 --- a/packages/test-utils/api-report.md +++ b/packages/test-utils/api-report.md @@ -164,8 +164,6 @@ export class MockStorageApi implements StorageApi { // (undocumented) forBucket(name: string): StorageApi; // (undocumented) - get(key: string): T | undefined; - // (undocumented) observe$(key: string): Observable>; // (undocumented) remove(key: string): Promise; diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts index c974198be1..63aa4235e1 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.test.ts @@ -24,7 +24,7 @@ describe('WebStorage Storage API', () => { it('should return undefined for values which are unset', async () => { const storage = createMockStorage(); - expect(storage.get('myfakekey')).toBeUndefined(); + expect(storage.snapshot('myfakekey').value).toBeUndefined(); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'absent', @@ -33,32 +33,29 @@ describe('WebStorage Storage API', () => { }); }); - it('should allow the setting and getting of the simple data structures', async () => { + it('should allow the setting and snapshotting of the simple data structures', async () => { const storage = createMockStorage(); await storage.set('myfakekey', 'helloimastring'); await storage.set('mysecondfakekey', 1234); await storage.set('mythirdfakekey', true); - expect(storage.get('myfakekey')).toBe('helloimastring'); - expect(storage.get('mysecondfakekey')).toBe(1234); - expect(storage.get('mythirdfakekey')).toBe(true); + expect(storage.snapshot('myfakekey').value).toBe('helloimastring'); + expect(storage.snapshot('mysecondfakekey').value).toBe(1234); + expect(storage.snapshot('mythirdfakekey').value).toBe(true); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: 'helloimastring', - newValue: 'helloimastring', }); expect(storage.snapshot('mysecondfakekey')).toEqual({ key: 'mysecondfakekey', presence: 'present', value: 1234, - newValue: 1234, }); expect(storage.snapshot('mythirdfakekey')).toEqual({ key: 'mythirdfakekey', presence: 'present', value: true, - newValue: true, }); }); @@ -72,12 +69,11 @@ describe('WebStorage Storage API', () => { await storage.set('myfakekey', mockData); - expect(storage.get('myfakekey')).toEqual(mockData); + expect(storage.snapshot('myfakekey').value).toEqual(mockData); expect(storage.snapshot('myfakekey')).toEqual({ key: 'myfakekey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -107,7 +103,6 @@ describe('WebStorage Storage API', () => { key: 'correctKey', presence: 'present', value: mockData, - newValue: mockData, }); }); @@ -153,9 +148,11 @@ describe('WebStorage Storage API', () => { await firstStorage.set(keyName, 'boop'); await secondStorage.set(keyName, 'deerp'); - expect(firstStorage.get(keyName)).not.toBe(secondStorage.get(keyName)); - expect(firstStorage.get(keyName)).toBe('boop'); - expect(secondStorage.get(keyName)).toBe('deerp'); + expect(firstStorage.snapshot(keyName)).not.toBe( + secondStorage.snapshot(keyName), + ); + expect(firstStorage.snapshot(keyName).value).toBe('boop'); + expect(secondStorage.snapshot(keyName).value).toBe('deerp'); expect(firstStorage.snapshot(keyName)).not.toEqual( secondStorage.snapshot(keyName), ); @@ -163,13 +160,11 @@ describe('WebStorage Storage API', () => { key: keyName, presence: 'present', value: 'boop', - newValue: 'boop', }); expect(secondStorage.snapshot(keyName)).toEqual({ key: keyName, presence: 'present', value: 'deerp', - newValue: 'deerp', }); }); @@ -186,7 +181,7 @@ describe('WebStorage Storage API', () => { await firstStorage.set('test2', { error: true }); - expect(secondStorage.get('deep/test2')).toBe(undefined); + expect(secondStorage.snapshot('deep/test2').value).toBe(undefined); expect(secondStorage.snapshot('deep/test2')).toMatchObject({ presence: 'absent', }); @@ -201,19 +196,17 @@ describe('WebStorage Storage API', () => { await firstStorage.set('test2', true); - expect(firstStorage.get('test2')).toBe(true); - expect(secondStorage.get('test2')).toBe(undefined); + expect(firstStorage.snapshot('test2').value).toBe(true); + expect(secondStorage.snapshot('test2').value).toBe(undefined); expect(firstStorage.snapshot('test2')).toEqual({ key: 'test2', presence: 'present', value: true, - newValue: true, }); expect(secondStorage.snapshot('test2')).toEqual({ key: 'test2', presence: 'absent', value: undefined, - newValue: undefined, }); }); diff --git a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts index 25024f2211..6ce91a4a31 100644 --- a/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts +++ b/packages/test-utils/src/testUtils/apis/StorageApi/MockStorageApi.ts @@ -61,10 +61,6 @@ export class MockStorageApi implements StorageApi { return this.bucketStorageApis.get(name)!; } - get(key: string): T | undefined { - return this.snapshot(key).value as T | undefined; - } - snapshot(key: string): StorageValueSnapshot { if (this.data.hasOwnProperty(this.getKeyName(key))) { const data = this.data[this.getKeyName(key)]; @@ -72,14 +68,12 @@ export class MockStorageApi implements StorageApi { key, presence: 'present', value: data, - newValue: data, }; } return { key, presence: 'absent', value: undefined, - newValue: undefined, }; } @@ -95,7 +89,6 @@ export class MockStorageApi implements StorageApi { key, presence: 'present', value: serialized, - newValue: serialized, }); } @@ -105,7 +98,6 @@ export class MockStorageApi implements StorageApi { key, presence: 'absent', value: undefined, - newValue: undefined, }); } diff --git a/packages/test-utils/src/testUtils/defaultApis.ts b/packages/test-utils/src/testUtils/defaultApis.ts index f06e1ba6b7..1ff10bfe6f 100644 --- a/packages/test-utils/src/testUtils/defaultApis.ts +++ b/packages/test-utils/src/testUtils/defaultApis.ts @@ -21,16 +21,13 @@ import { ErrorAlerter, GoogleAuth, GithubAuth, - OAuth2, OktaAuth, GitlabAuth, - Auth0Auth, MicrosoftAuth, BitbucketAuth, OAuthRequestManager, WebStorage, UrlPatternDiscovery, - SamlAuth, OneLoginAuth, UnhandledErrorForwarder, AtlassianAuth, @@ -45,16 +42,12 @@ import { oauthRequestApiRef, googleAuthApiRef, githubAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, gitlabAuthApiRef, - auth0AuthApiRef, microsoftAuthApiRef, storageApiRef, configApiRef, - samlAuthApiRef, oneloginAuthApiRef, - oidcAuthApiRef, bitbucketAuthApiRef, atlassianAuthApiRef, } from '@backstage/core-plugin-api'; @@ -158,46 +151,6 @@ export const defaultApis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: auth0AuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - Auth0Auth.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: oauth2ApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), - createApiFactory({ - api: samlAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, configApi }) => - SamlAuth.create({ - discoveryApi, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: oneloginAuthApiRef, deps: { @@ -212,25 +165,6 @@ export const defaultApis = [ environment: configApi.getOptionalString('auth.environment'), }), }), - createApiFactory({ - api: oidcAuthApiRef, - deps: { - discoveryApi: discoveryApiRef, - oauthRequestApi: oauthRequestApiRef, - configApi: configApiRef, - }, - factory: ({ discoveryApi, oauthRequestApi, configApi }) => - OAuth2.create({ - discoveryApi, - oauthRequestApi, - provider: { - id: 'oidc', - title: 'Your Identity Provider', - icon: () => null, - }, - environment: configApi.getOptionalString('auth.environment'), - }), - }), createApiFactory({ api: bitbucketAuthApiRef, deps: { diff --git a/plugins/airbrake-backend/README.md b/plugins/airbrake-backend/README.md index 9c9b56ade4..21056229de 100644 --- a/plugins/airbrake-backend/README.md +++ b/plugins/airbrake-backend/README.md @@ -24,7 +24,7 @@ This method of serving the plugin provides quicker iteration speed and a faster 3. Go into the plugin's directory and run it in standalone mode by running `yarn start`. -Access it from http://localhost:7007/api/airbrake. Or use the Airbrake plugin which will talk to it automatically. +Access it from http://localhost:7007/api/airbrake. Or use the [Airbrake plugin in standalone mode](../airbrake/README.md#local-development) which will talk to it automatically. Here are some example endpoints: diff --git a/plugins/airbrake/README.md b/plugins/airbrake/README.md index d5ea9bb006..bc830c326d 100644 --- a/plugins/airbrake/README.md +++ b/plugins/airbrake/README.md @@ -20,7 +20,7 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: yarn add @backstage/plugin-airbrake-backend ``` -3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx`: +3. Add the `EntityAirbrakeContent` to `packages/app/src/components/catalog/EntityPage.tsx` for all the entity pages you want Airbrake to be in: ```typescript jsx import { EntityAirbrakeContent } from '@backstage/plugin-airbrake'; @@ -32,40 +32,78 @@ The Airbrake plugin provides connectivity between Backstage and Airbrake (https: ); + + const websiteEntityPage = ( + + + + + + ); + + const defaultEntityPage = ( + + + + + + ); ``` -4. Setup the Backend code in `packages/backend/src/index.ts`: +4. Create `packages/backend/src/plugins/airbrake.ts` with these contents: ```typescript + import { Router } from 'express'; + import { PluginEnvironment } from '../types'; import { - createRouter as createAirbrakeRouter, + createRouter, extractAirbrakeConfig, } from '@backstage/plugin-airbrake-backend'; - async function main() { - //... After const config = await loadBackendConfig({ ... - - const airbrakeRouter = await createAirbrakeRouter({ + export default async function createPlugin({ + logger, + config, + }: PluginEnvironment): Promise { + return createRouter({ logger, airbrakeConfig: extractAirbrakeConfig(config), }); - - const service = createServiceBuilder(module) - // ... Add the airbrakeRouter here - .addRouter('/api/airbrake', airbrakeRouter); } ``` -5. Add this config as a top level section in your `app-config.yaml`: +5. Setup the Backend code in `packages/backend/src/index.ts`: + + ```typescript + import airbrake from './plugins/airbrake'; + + async function main() { + //... After const createEnv = makeCreateEnv(config) ... + + const airbrakeEnv = useHotMemoize(module, () => createEnv('airbrake')); + + //... After const apiRouter = Router() ... + apiRouter.use('/airbrake', await airbrake(airbrakeEnv)); + } + ``` + +6. Add this config as a top level section in your `app-config.yaml`: ```yaml airbrake: apiKey: ${AIRBRAKE_API_KEY} ``` -6. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) +7. Set an environment variable `AIRBRAKE_API_KEY` with your [API key](https://airbrake.io/docs/api/#authentication) before starting Backstage backend. +8. Add the following annotation to the `catalog-info.yaml` for a repo you want to link to an Airbrake project: + + ```yaml + metadata: + annotations: + airbrake.io/project-id: '123456' + ``` + ## Local Development Start this plugin in standalone mode by running `yarn start` inside the plugin directory. This method of serving the plugin provides quicker diff --git a/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx b/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx index 34d12aa032..e6355d861f 100644 --- a/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx +++ b/plugins/allure/src/components/AllureReportComponent/AllureReportComponent.test.tsx @@ -23,6 +23,7 @@ import { setupRequestMockHandlers, renderInTestApp, } from '@backstage/test-utils'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; describe('ExampleComponent', () => { const server = setupServer(); @@ -39,7 +40,15 @@ describe('ExampleComponent', () => { it('should render', async () => { const rendered = await renderInTestApp( - + + + , ); expect(rendered.getByText('Missing Annotation')).toBeInTheDocument(); diff --git a/plugins/api-docs/dev/index.tsx b/plugins/api-docs/dev/index.tsx index 633a285b98..9975e91e98 100644 --- a/plugins/api-docs/dev/index.tsx +++ b/plugins/api-docs/dev/index.tsx @@ -53,8 +53,8 @@ createDevApp() items: mockEntities.slice(), }; }, - async getEntityByName(name: string) { - return mockEntities.find(e => e.metadata.name === name); + async getEntityByRef(ref: string) { + return mockEntities.find(e => e.metadata.name === ref); }, } as unknown as typeof catalogApiRef.T), }) diff --git a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx index 16b335b0a7..bc413661a8 100644 --- a/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx +++ b/plugins/api-docs/src/components/ApiExplorerPage/DefaultApiExplorerPage.test.tsx @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, RELATION_MEMBER_OF } from '@backstage/catalog-model'; +import { + Entity, + parseEntityRef, + RELATION_MEMBER_OF, +} from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/core-app-api'; import { TableColumn, TableProps } from '@backstage/core-components'; import { @@ -22,11 +26,13 @@ import { configApiRef, storageApiRef, } from '@backstage/core-plugin-api'; -import { CatalogTableRow } from '@backstage/plugin-catalog'; +import { + CatalogTableRow, + DefaultStarredEntitiesApi, +} from '@backstage/plugin-catalog'; import { CatalogApi, catalogApiRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; @@ -58,11 +64,11 @@ describe('DefaultApiExplorerPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, diff --git a/plugins/auth-backend/src/identity/TokenFactory.test.ts b/plugins/auth-backend/src/identity/TokenFactory.test.ts index f5f06f1209..4b1c1804df 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.test.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.test.ts @@ -18,6 +18,7 @@ import { MemoryKeyStore } from './MemoryKeyStore'; import { TokenFactory } from './TokenFactory'; import { getVoidLogger } from '@backstage/backend-common'; import { JWKS, JSONWebKey, JWT } from 'jose'; +import { stringifyEntityRef } from '@backstage/catalog-model'; const logger = getVoidLogger(); @@ -28,6 +29,12 @@ function jwtKid(jwt: string): string { return header.kid; } +const entityRef = stringifyEntityRef({ + kind: 'User', + namespace: 'default', + name: 'JackFrost', +}); + describe('TokenFactory', () => { it('should issue valid tokens signed by a listed key', async () => { const keyDurationSeconds = 5; @@ -39,7 +46,7 @@ describe('TokenFactory', () => { }); await expect(factory.listPublicKeys()).resolves.toEqual({ keys: [] }); - const token = await factory.issueToken({ claims: { sub: 'foo' } }); + const token = await factory.issueToken({ claims: { sub: entityRef } }); const { keys } = await factory.listPublicKeys(); const keyStore = JWKS.asKeyStore({ @@ -53,7 +60,7 @@ describe('TokenFactory', () => { expect(payload).toEqual({ iss: 'my-issuer', aud: 'backstage', - sub: 'foo', + sub: entityRef, iat: expect.any(Number), exp: expect.any(Number), }); @@ -71,8 +78,12 @@ describe('TokenFactory', () => { logger, }); - const token1 = await factory.issueToken({ claims: { sub: 'foo' } }); - const token2 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token1 = await factory.issueToken({ + claims: { sub: entityRef }, + }); + const token2 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token1)).toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -89,7 +100,9 @@ describe('TokenFactory', () => { keys: [], }); - const token3 = await factory.issueToken({ claims: { sub: 'foo' } }); + const token3 = await factory.issueToken({ + claims: { sub: entityRef }, + }); expect(jwtKid(token3)).not.toBe(jwtKid(token2)); await expect(factory.listPublicKeys()).resolves.toEqual({ @@ -100,4 +113,20 @@ describe('TokenFactory', () => { ], }); }); + + it('should throw an error with a non entityRef sub claim', async () => { + const keyDurationSeconds = 5; + const factory = new TokenFactory({ + issuer: 'my-issuer', + keyStore: new MemoryKeyStore(), + keyDurationSeconds, + logger, + }); + + await expect(() => { + return factory.issueToken({ + claims: { sub: 'UserId' }, + }); + }).rejects.toThrowError(); + }); }); diff --git a/plugins/auth-backend/src/identity/TokenFactory.ts b/plugins/auth-backend/src/identity/TokenFactory.ts index 041a292d44..cb2a8b453d 100644 --- a/plugins/auth-backend/src/identity/TokenFactory.ts +++ b/plugins/auth-backend/src/identity/TokenFactory.ts @@ -19,6 +19,7 @@ import { JSONWebKey, JWK, JWS } from 'jose'; import { Logger } from 'winston'; import { v4 as uuid } from 'uuid'; import { DateTime } from 'luxon'; +import { parseEntityRef } from '@backstage/catalog-model'; const MS_IN_S = 1000; @@ -72,6 +73,15 @@ export class TokenFactory implements TokenIssuer { const iat = Math.floor(Date.now() / MS_IN_S); const exp = iat + this.keyDurationSeconds; + // Validate that the subject claim is a valid EntityRef + try { + parseEntityRef(sub); + } catch (error) { + throw new Error( + '"sub" claim provided by the auth resolver is not a valid EntityRef.', + ); + } + this.logger.info(`Issuing token for ${sub}, with entities ${ent ?? []}`); return JWS.sign({ iss, sub, aud, iat, exp, ent }, key, { diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts index d2cee7bd89..08c46f2c21 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.test.ts @@ -26,6 +26,7 @@ import { CatalogIdentityClient } from './CatalogIdentityClient'; describe('CatalogIdentityClient', () => { const catalogApi: jest.Mocked = { getLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getEntities: jest.fn(), addLocation: jest.fn(), diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts index d948af2e05..502da913af 100644 --- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts +++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts @@ -18,7 +18,7 @@ import { Logger } from 'winston'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { CatalogApi } from '@backstage/catalog-client'; import { - EntityName, + CompoundEntityRef, parseEntityRef, RELATION_MEMBER_OF, stringifyEntityRef, @@ -96,7 +96,7 @@ export class CatalogIdentityClient { return null; } }) - .filter((ref): ref is EntityName => ref !== null); + .filter((ref): ref is CompoundEntityRef => ref !== null); const filter = resolvedEntityRefs.map(ref => ({ kind: ref.kind, diff --git a/plugins/badges-backend/src/service/router.test.ts b/plugins/badges-backend/src/service/router.test.ts index 56dbfb0655..c492449cb3 100644 --- a/plugins/badges-backend/src/service/router.test.ts +++ b/plugins/badges-backend/src/service/router.test.ts @@ -60,6 +60,7 @@ describe('createRouter', () => { catalog = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -103,7 +104,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge-specs', () => { it('returns all badge specs for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.getBadges.mockResolvedValueOnce([{ id: badge.id }]); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); @@ -115,8 +116,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.text).toEqual(JSON.stringify([badge], null, 2)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -142,7 +143,7 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/badge/test-badge', () => { it('returns badge for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); const image = '...'; badgeBuilder.createBadgeSvg.mockResolvedValueOnce(image); @@ -154,8 +155,8 @@ describe('createRouter', () => { expect(response.status).toEqual(200); expect(response.body).toEqual(Buffer.from(image)); - expect(catalog.getEntityByName).toHaveBeenCalledTimes(1); - expect(catalog.getEntityByName).toHaveBeenCalledWith( + expect(catalog.getEntityByRef).toHaveBeenCalledTimes(1); + expect(catalog.getEntityByRef).toHaveBeenCalledWith( { namespace: 'default', kind: 'service', @@ -179,7 +180,7 @@ describe('createRouter', () => { }); it('returns badge spec for entity', async () => { - catalog.getEntityByName.mockResolvedValueOnce(entity); + catalog.getEntityByRef.mockResolvedValueOnce(entity); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); const url = '/entity/default/service/test/badge/test-badge?format=json'; @@ -192,7 +193,7 @@ describe('createRouter', () => { describe('Errors', () => { it('returns 404 for unknown entities', async () => { - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); async function testUrl(url: string) { const response = await request(app).get(url); expect(response.status).toEqual(404); diff --git a/plugins/badges-backend/src/service/router.ts b/plugins/badges-backend/src/service/router.ts index 196ddf36da..b7211a0aad 100644 --- a/plugins/badges-backend/src/service/router.ts +++ b/plugins/badges-backend/src/service/router.ts @@ -46,7 +46,7 @@ export async function createRouter( router.get('/entity/:namespace/:kind/:name/badge-specs', async (req, res) => { const { namespace, kind, name } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), @@ -84,7 +84,7 @@ export async function createRouter( '/entity/:namespace/:kind/:name/badge/:badgeId', async (req, res) => { const { namespace, kind, name, badgeId } = req.params; - const entity = await catalog.getEntityByName( + const entity = await catalog.getEntityByRef( { namespace, kind, name }, { token: getBearerToken(req.headers.authorization), diff --git a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts index e5e72d5fa8..1018e49a41 100644 --- a/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts +++ b/plugins/catalog-backend-module-aws/src/processors/AwsOrganizationCloudAccountProcessor.ts @@ -20,7 +20,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import AWS, { Credentials, Organizations } from 'aws-sdk'; import { Account, ListAccountsResponse } from 'aws-sdk/clients/organizations'; @@ -112,7 +112,7 @@ export class AwsOrganizationCloudAccountProcessor implements CatalogProcessor { return true; }) .forEach(entity => { - emit(results.entity(location, entity)); + emit(processingResult.entity(location, entity)); }); return true; diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts index 6c31d5a27d..bec24a1744 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgReaderProcessor.ts @@ -28,7 +28,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; /** @@ -119,10 +119,10 @@ export class LdapOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index 0e088d3a4e..a253359471 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -35,17 +35,38 @@ catalog: # the App registration in the Microsoft Azure Portal. clientId: ${MICROSOFT_GRAPH_CLIENT_ID} clientSecret: ${MICROSOFT_GRAPH_CLIENT_SECRET_TOKEN} + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + userExpand: manager # Optional filter for user, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/user?view=graph-rest-1.0#properties # and for the syntax https://docs.microsoft.com/en-us/graph/query-parameters#filter-parameter # This and userGroupMemberFilter are mutually exclusive, only one can be specified userFilter: accountEnabled eq true and userType eq 'member' # Optional filter for users, use group membership to get users. + # (Filtered groups and fetch their members.) # This and userFilter are mutually exclusive, only one can be specified + # See https://docs.microsoft.com/en-us/graph/search-query-parameter userGroupMemberFilter: "displayName eq 'Backstage Users'" + # Optional parameter to include the expanded resource or collection referenced + # by a single relationship (navigation property) in your results. + # Only one relationship can be expanded in a single request. + # See https://docs.microsoft.com/en-us/graph/query-parameters#expand-parameter + # Can be combined with userGroupMember[...] instead of userFilter. + groupExpand: member + # Optional search for users, use group membership to get users. + # (Search for groups and fetch their members.) + # This and userFilter are mutually exclusive, only one can be specified + userGroupMemberSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' # Optional filter for group, see Microsoft Graph API for the syntax # See https://docs.microsoft.com/en-us/graph/api/resources/group?view=graph-rest-1.0#properties groupFilter: securityEnabled eq false and mailEnabled eq true and groupTypes/any(c:c+eq+'Unified') + # Optional search for groups, see Microsoft Graph API for the syntax + # See https://docs.microsoft.com/en-us/graph/search-query-parameter + groupSearch: '"description:One" AND ("displayName:Video" OR "displayName:Drive")' ``` `userFilter` and `userGroupMemberFilter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown. diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 7ad603e6ca..8df06f54b1 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -78,7 +78,10 @@ export class MicrosoftGraphClient { userId: string, maxSize: number, ): Promise; - getUserProfile(userId: string): Promise; + getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise; getUsers(query?: ODataQuery): AsyncIterable; requestApi( path: string, @@ -158,9 +161,10 @@ export type MicrosoftGraphProviderConfig = { clientId: string; clientSecret: string; userFilter?: string; - userExpand?: string[]; + userExpand?: string; userGroupMemberFilter?: string; userGroupMemberSearch?: string; + groupExpand?: string; groupFilter?: string; groupSearch?: string; }; @@ -172,7 +176,7 @@ export function normalizeEntityName(name: string): string; export type ODataQuery = { search?: string; filter?: string; - expand?: string[]; + expand?: string; select?: string[]; }; @@ -191,10 +195,11 @@ export function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts index 6413a97c15..94a126e1a7 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.test.ts @@ -85,7 +85,7 @@ describe('MicrosoftGraphClient', () => { const response = await client.requestApi('users', { filter: 'test eq true', - expand: ['children'], + expand: 'children', select: ['id', 'children'], }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts index db749a536e..cba5d9f640 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/client.ts @@ -38,7 +38,7 @@ export type ODataQuery = { /** * specifies the related resources or media streams to be included in line with retrieved resources */ - expand?: string[]; + expand?: string; /** * request a specific set of properties for each entity or complex type */ @@ -155,7 +155,7 @@ export class MicrosoftGraphClient { $search: query?.search, $filter: query?.filter, $select: query?.select?.join(','), - $expand: query?.expand?.join(','), + $expand: query?.expand, }, { addQueryPrefix: true, @@ -203,10 +203,14 @@ export class MicrosoftGraphClient { * * @public * @param userId - The unique identifier for the `User` resource + * @param query - OData Query {@link ODataQuery} * */ - async getUserProfile(userId: string): Promise { - const response = await this.requestApi(`users/${userId}`); + async getUserProfile( + userId: string, + query?: ODataQuery, + ): Promise { + const response = await this.requestApi(`users/${userId}`, query); if (response.status !== 200) { await this.handleError('user profile', response); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts index efbbca1f5c..cfc5c1cbb0 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts @@ -53,7 +53,9 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com/', + userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ], @@ -66,7 +68,9 @@ describe('readMicrosoftGraphConfig', () => { clientId: 'clientId', clientSecret: 'clientSecret', authority: 'https://login.example.com', + userExpand: 'manager', userFilter: 'accountEnabled eq true', + groupExpand: 'member', groupFilter: 'securityEnabled eq false', }, ]; diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts index 6809144f80..b7fbcfb8a9 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts @@ -57,7 +57,7 @@ export type MicrosoftGraphProviderConfig = { * * E.g. "manager" */ - userExpand?: string[]; + userExpand?: string; /** * The filter to apply to extract users by groups memberships. * @@ -70,6 +70,12 @@ export type MicrosoftGraphProviderConfig = { * E.g. "\"displayName:-team\"" would only match groups which contain '-team' */ userGroupMemberSearch?: string; + /** + * The "expand" argument to apply to groups. + * + * E.g. "member" + */ + groupExpand?: string; /** * The filter to apply to extract groups. * @@ -106,6 +112,8 @@ export function readMicrosoftGraphConfig( const tenantId = providerConfig.getString('tenantId'); const clientId = providerConfig.getString('clientId'); const clientSecret = providerConfig.getString('clientSecret'); + + const userExpand = providerConfig.getOptionalString('userExpand'); const userFilter = providerConfig.getOptionalString('userFilter'); const userGroupMemberFilter = providerConfig.getOptionalString( 'userGroupMemberFilter', @@ -113,6 +121,7 @@ export function readMicrosoftGraphConfig( const userGroupMemberSearch = providerConfig.getOptionalString( 'userGroupMemberSearch', ); + const groupExpand = providerConfig.getOptionalString('groupExpand'); const groupFilter = providerConfig.getOptionalString('groupFilter'); const groupSearch = providerConfig.getOptionalString('groupSearch'); @@ -133,9 +142,11 @@ export function readMicrosoftGraphConfig( tenantId, clientId, clientSecret, + userExpand, userFilter, userGroupMemberFilter, userGroupMemberSearch, + groupExpand, groupFilter, groupSearch, }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index b0273e1f58..e3d6897d66 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -118,7 +118,7 @@ describe('read microsoft graph', () => { expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand and custom transformer', async () => { async function* getExampleUsers() { yield { id: 'userid', @@ -133,6 +133,7 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsers(client, { + userExpand: 'manager', userFilter: 'accountEnabled eq true', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', @@ -154,6 +155,7 @@ describe('read microsoft graph', () => { expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); @@ -227,12 +229,14 @@ describe('read microsoft graph', () => { expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: undefined, + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); - it('should read users with custom transformer', async () => { + it('should read users with userExpand, groupExpand and custom transformer', async () => { async function* getExampleGroups() { yield { id: 'groupid', @@ -266,7 +270,9 @@ describe('read microsoft graph', () => { ); const { users } = await readMicrosoftGraphUsersInGroups(client, { + userExpand: 'manager', userGroupMemberFilter: 'securityEnabled eq true', + groupExpand: 'member', transformer: async () => ({ apiVersion: 'backstage.io/v1alpha1', kind: 'User', @@ -287,13 +293,16 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledTimes(1); expect(client.getGroups).toBeCalledWith({ + expand: 'member', filter: 'securityEnabled eq true', }); expect(client.getGroupMembers).toBeCalledTimes(1); expect(client.getGroupMembers).toBeCalledWith('groupid'); expect(client.getUserProfile).toBeCalledTimes(1); - expect(client.getUserProfile).toBeCalledWith('userid'); + expect(client.getUserProfile).toBeCalledWith('userid', { + expand: 'manager', + }); expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); @@ -446,6 +455,100 @@ describe('read microsoft graph', () => { // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); }); + it('should read groups with groupExpand', async () => { + async function* getExampleGroups() { + yield { + id: 'groupid', + displayName: 'Group Name', + description: 'Group Description', + mail: 'group@example.com', + }; + } + + async function* getExampleGroupMembers(): AsyncIterable { + yield { + '@odata.type': '#microsoft.graph.group', + id: 'childgroupid', + }; + yield { + '@odata.type': '#microsoft.graph.user', + id: 'userid', + }; + } + + client.getGroups.mockImplementation(getExampleGroups); + client.getGroupMembers.mockImplementation(getExampleGroupMembers); + client.getOrganization.mockResolvedValue({ + id: 'tenantid', + displayName: 'Organization Name', + }); + client.getGroupPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { groups, groupMember, groupMemberOf, rootGroup } = + await readMicrosoftGraphGroups(client, 'tenantid', { + groupExpand: 'member', + groupFilter: 'securityEnabled eq false', + }); + + const expectedRootGroup = group({ + metadata: { + annotations: { + 'graph.microsoft.com/tenant-id': 'tenantid', + }, + name: 'organization_name', + description: 'Organization Name', + }, + spec: { + type: 'root', + profile: { + displayName: 'Organization Name', + }, + children: [], + }, + }); + expect(groups).toEqual([ + expectedRootGroup, + group({ + metadata: { + annotations: { + 'graph.microsoft.com/group-id': 'groupid', + }, + name: 'group_name', + description: 'Group Description', + }, + spec: { + type: 'team', + profile: { + displayName: 'Group Name', + email: 'group@example.com', + // TODO: Loading groups photos doesn't work right now as Microsoft + // Graph doesn't allows this yet + /* picture: 'data:image/jpeg;base64,...',*/ + }, + children: [], + }, + }), + ]); + expect(rootGroup).toEqual(expectedRootGroup); + expect(groupMember.get('groupid')).toEqual(new Set(['childgroupid'])); + expect(groupMemberOf.get('userid')).toEqual(new Set(['groupid'])); + expect(groupMember.get('organization_name')).toEqual(new Set()); + + expect(client.getGroups).toBeCalledTimes(1); + expect(client.getGroups).toBeCalledWith({ + expand: 'member', + filter: 'securityEnabled eq false', + }); + expect(client.getGroupMembers).toBeCalledTimes(1); + expect(client.getGroupMembers).toBeCalledWith('groupid'); + // TODO: Loading groups photos doesn't work right now as Microsoft Graph + // doesn't allows this yet + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledTimes(1); + // expect(client.getGroupPhotoWithSizeLimit).toBeCalledWith('groupid', 120); + }); + it('should read security groups', async () => { async function* getExampleGroups() { yield { @@ -634,6 +737,14 @@ describe('read microsoft graph', () => { }; } + async function getExampleUserProfile(userId: string) { + return { + id: userId, + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + async function* getExampleGroups() { yield { id: 'groupid', @@ -686,7 +797,7 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userFilter', async () => { + it('should read users using userExpand and userFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', @@ -705,12 +816,14 @@ describe('read microsoft graph', () => { await readMicrosoftGraphOrg(client, 'tenantid', { logger: getVoidLogger(), + userExpand: 'manager', userFilter: 'accountEnabled eq true', groupFilter: 'securityEnabled eq false', }); expect(client.getUsers).toBeCalledTimes(1); expect(client.getUsers).toBeCalledWith({ + expand: 'manager', filter: 'accountEnabled eq true', }); expect(client.getGroups).toBeCalledTimes(1); @@ -719,13 +832,14 @@ describe('read microsoft graph', () => { }); }); - it('should read users using userGroupMemberFilter', async () => { + it('should read users using userExpand and userGroupMemberFilter', async () => { client.getOrganization.mockResolvedValue({ id: 'tenantid', displayName: 'Organization Name', }); client.getUsers.mockImplementation(getExampleUsers); + client.getUserProfile.mockImplementation(getExampleUserProfile); client.getUserPhotoWithSizeLimit.mockResolvedValue( 'data:image/jpeg;base64,...', ); @@ -750,6 +864,8 @@ describe('read microsoft graph', () => { expect(client.getGroups).toBeCalledWith({ filter: 'securityEnabled eq false', }); + expect(client.getUserProfile).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); }); }); }); diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index f862fd5ace..ed6ca76e68 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -85,7 +85,7 @@ export async function readMicrosoftGraphUsers( client: MicrosoftGraphClient, options: { userFilter?: string; - userExpand?: string[]; + userExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -137,8 +137,10 @@ export async function readMicrosoftGraphUsers( export async function readMicrosoftGraphUsersInGroups( client: MicrosoftGraphClient, options: { + userExpand?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; transformer?: UserTransformer; logger: Logger; }, @@ -149,15 +151,16 @@ export async function readMicrosoftGraphUsersInGroups( const limiter = limiterFactory(10); - const transformer = options?.transformer ?? defaultUserTransformer; + const transformer = options.transformer ?? defaultUserTransformer; const userGroupMemberPromises: Promise[] = []; const userPromises: Promise[] = []; const groupMemberUsers: Set = new Set(); for await (const group of client.getGroups({ - search: options?.userGroupMemberSearch, - filter: options?.userGroupMemberFilter, + expand: options.groupExpand, + search: options.userGroupMemberSearch, + filter: options.userGroupMemberFilter, })) { // Process all groups in parallel, otherwise it can take quite some time userGroupMemberPromises.push( @@ -186,7 +189,9 @@ export async function readMicrosoftGraphUsersInGroups( let user; let userPhoto; try { - user = await client.getUserProfile(userId); + user = await client.getUserProfile(userId, { + expand: options.userExpand, + }); } catch (e) { options.logger.warn(`Unable to load user for ${userId}`); } @@ -326,8 +331,9 @@ export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, options?: { - groupSearch?: string; + groupExpand?: string; groupFilter?: string; + groupSearch?: string; groupTransformer?: GroupTransformer; organizationTransformer?: OrganizationTransformer; }, @@ -354,6 +360,7 @@ export async function readMicrosoftGraphGroups( const promises: Promise[] = []; for await (const group of client.getGroups({ + expand: options?.groupExpand, search: options?.groupSearch, filter: options?.groupFilter, })) { @@ -506,10 +513,11 @@ export async function readMicrosoftGraphOrg( client: MicrosoftGraphClient, tenantId: string, options: { - userExpand?: string[]; + userExpand?: string; userFilter?: string; userGroupMemberSearch?: string; userGroupMemberFilter?: string; + groupExpand?: string; groupSearch?: string; groupFilter?: string; userTransformer?: UserTransformer; diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 4db3a9b1a0..9037bd8728 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -19,7 +19,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { @@ -109,6 +109,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { userFilter: provider.userFilter, userGroupMemberFilter: provider.userGroupMemberFilter, userGroupMemberSearch: provider.userGroupMemberSearch, + groupExpand: provider.groupExpand, groupFilter: provider.groupFilter, groupSearch: provider.groupSearch, userTransformer: this.userTransformer, @@ -125,10 +126,10 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 9b4db418b3..0d9f306009 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -7,18 +7,18 @@ import { BitbucketIntegration } from '@backstage/integration'; import { CatalogApi } from '@backstage/catalog-client'; +import { CatalogEntityDocument as CatalogEntityDocument_2 } from '@backstage/plugin-catalog-common'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConditionalPolicyDecision } from '@backstage/plugin-permission-node'; import { Conditions } from '@backstage/plugin-permission-node'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { EntityPolicy } from '@backstage/catalog-model'; import express from 'express'; import { GetEntitiesRequest } from '@backstage/catalog-client'; import { GithubCredentialsProvider } from '@backstage/integration'; import { GitHubIntegrationConfig } from '@backstage/integration'; -import { IndexableDocument } from '@backstage/search-common'; import { JsonObject } from '@backstage/types'; import { JsonValue } from '@backstage/types'; import { Location as Location_2 } from '@backstage/catalog-client'; @@ -30,6 +30,7 @@ import { PermissionCriteria } from '@backstage/plugin-permission-common'; import { PermissionRule } from '@backstage/plugin-permission-node'; import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Readable } from 'stream'; import { Router } from 'express'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { TokenManager } from '@backstage/backend-common'; @@ -137,14 +138,22 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { export class BitbucketDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrationRegistry; - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger_2; + }) => AsyncIterable; logger: Logger_2; }); // (undocumented) static fromConfig( config: Config, options: { - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger_2; + }) => AsyncIterable; logger: Logger_2; }, ): BitbucketDiscoveryProcessor; @@ -158,7 +167,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) export type BitbucketRepositoryParser = (options: { integration: BitbucketIntegration; target: string; @@ -209,11 +218,17 @@ export class CatalogBuilder { key: string, resolver: PlaceholderResolver, ): CatalogBuilder; + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder; + setProcessingIntervalSeconds(seconds: number): CatalogBuilder; + // @deprecated setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder; + // @deprecated setRefreshIntervalSeconds(seconds: number): CatalogBuilder; } -// @public +// @alpha export const catalogConditions: Conditions<{ hasAnnotation: PermissionRule< Entity, @@ -239,19 +254,8 @@ export const catalogConditions: Conditions<{ >; }>; -// @public (undocumented) -export interface CatalogEntityDocument extends IndexableDocument { - // (undocumented) - componentType: string; - // (undocumented) - kind: string; - // (undocumented) - lifecycle: string; - // (undocumented) - namespace: string; - // (undocumented) - owner: string; -} +// @public @deprecated (undocumented) +export type CatalogEntityDocument = CatalogEntityDocument_2; // @public (undocumented) export type CatalogEnvironment = { @@ -300,11 +304,6 @@ export type CatalogProcessor = { emit: CatalogProcessorEmit, cache: CatalogProcessorCache, ): Promise; - handleError?( - error: Error, - location: LocationSpec, - emit: CatalogProcessorEmit, - ): Promise; }; // @public @@ -347,7 +346,6 @@ export type CatalogProcessorParser = (options: { export type CatalogProcessorRelationResult = { type: 'relation'; relation: EntityRelationSpec; - entityRef?: string; }; // @public (undocumented) @@ -394,17 +392,23 @@ export class CodeOwnersProcessor implements CatalogProcessor { preProcessEntity(entity: Entity, location: LocationSpec): Promise; } -// @public +// @alpha export const createCatalogPermissionRule: ( rule: PermissionRule, ) => PermissionRule; -// @public +// @alpha export const createCatalogPolicyDecision: ( conditions: PermissionCriteria>, ) => ConditionalPolicyDecision; // @public +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction; + +// @public @deprecated export function createRandomRefreshInterval(options: { minSeconds: number; maxSeconds: number; @@ -413,8 +417,8 @@ export function createRandomRefreshInterval(options: { // @public export function createRouter(options: RouterOptions): Promise; -// @public (undocumented) -export class DefaultCatalogCollator implements DocumentCollator { +// @public @deprecated (undocumented) +export class DefaultCatalogCollator { constructor(options: { discovery: PluginEndpointDiscovery; tokenManager: TokenManager; @@ -432,7 +436,7 @@ export class DefaultCatalogCollator implements DocumentCollator { // (undocumented) protected discovery: PluginEndpointDiscovery; // (undocumented) - execute(): Promise; + execute(): Promise; // (undocumented) protected filter?: GetEntitiesRequest['filter']; // (undocumented) @@ -454,6 +458,31 @@ export class DefaultCatalogCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public (undocumented) +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ): DefaultCatalogCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public (undocumented) +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + // @public (undocumented) export class DefaultCatalogProcessingOrchestrator implements CatalogProcessingOrchestrator @@ -522,7 +551,7 @@ export type EntitiesSearchFilter = { values?: string[]; }; -// @public (undocumented) +// @public @deprecated (undocumented) function entity( atLocation: LocationSpec, newEntity: Entity, @@ -621,9 +650,9 @@ export type EntityProviderMutation = // @public export type EntityRelationSpec = { - source: EntityName; + source: CompoundEntityRef; type: string; - target: EntityName; + target: CompoundEntityRef; }; // @public (undocumented) @@ -639,7 +668,7 @@ export class FileReaderProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function generalError( atLocation: LocationSpec, message: string, @@ -774,13 +803,13 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { ): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function inputError( atLocation: LocationSpec, message: string, ): CatalogProcessorResult; -// @public (undocumented) +// @public @deprecated (undocumented) function location_2( newLocation: LocationSpec, optional?: boolean, @@ -874,7 +903,7 @@ export interface LocationStore { listLocations(): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function notFoundError( atLocation: LocationSpec, message: string, @@ -896,7 +925,7 @@ export function parseEntityYaml( location: LocationSpec, ): Iterable; -// @public +// @alpha export const permissionRules: { hasAnnotation: PermissionRule< Entity, @@ -961,6 +990,34 @@ export type PlaceholderResolverResolveUrl = ( base: string, ) => string; +// @public +export type ProcessingIntervalFunction = () => number; + +// @public +export const processingResult: Readonly<{ + readonly notFoundError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly inputError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly generalError: ( + atLocation: LocationSpec, + message: string, + ) => CatalogProcessorResult; + readonly location: ( + newLocation: LocationSpec, + optional?: boolean | undefined, + ) => CatalogProcessorResult; + readonly entity: ( + atLocation: LocationSpec, + newEntity: Entity, + ) => CatalogProcessorResult; + readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult; +}>; + // @public export type RecursivePartial = { [P in keyof T]?: T[P] extends (infer U)[] @@ -970,7 +1027,7 @@ export type RecursivePartial = { : T[P]; }; -// @public +// @public @deprecated export type RefreshIntervalFunction = () => number; // @public @@ -984,7 +1041,7 @@ export interface RefreshService { refresh(options: RefreshOptions): Promise; } -// @public (undocumented) +// @public @deprecated (undocumented) function relation(spec: EntityRelationSpec): CatalogProcessorResult; declare namespace results { diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 55c92c92be..f7d946c531 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -9,7 +9,8 @@ "publishConfig": { "access": "public", "main": "dist/index.cjs.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "backend-plugin" @@ -25,7 +26,7 @@ ], "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -72,6 +73,7 @@ "@backstage/backend-test-utils": "^0.1.19", "@backstage/cli": "^0.14.1", "@backstage/plugin-permission-common": "^0.5.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/core-js": "^2.5.4", "@types/git-url-parse": "^9.0.0", @@ -87,6 +89,7 @@ }, "files": [ "dist", + "alpha", "migrations/**/*.{js,d.ts}", "config.d.ts" ], diff --git a/plugins/catalog-backend/src/api/common.ts b/plugins/catalog-backend/src/api/common.ts new file mode 100644 index 0000000000..f3b4a387ba --- /dev/null +++ b/plugins/catalog-backend/src/api/common.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CompoundEntityRef } from '@backstage/catalog-model'; + +/** + * Holds the entity location information. + * + * @remarks + * + * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. + * This flag is then set to indicate that the file can be not present. + * default value: 'required'. + * + * @public + */ +export type LocationSpec = { + type: string; + target: string; + presence?: 'optional' | 'required'; +}; + +/** + * Holds the relation data for entities. + * + * @public + */ +export type EntityRelationSpec = { + /** + * The source entity of this relation. + */ + source: CompoundEntityRef; + + /** + * The type of the relation. + */ + type: string; + + /** + * The target entity of this relation. + */ + target: CompoundEntityRef; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/results.ts b/plugins/catalog-backend/src/api/deprecatedResult.ts similarity index 69% rename from plugins/catalog-backend/src/ingestion/processors/results.ts rename to plugins/catalog-backend/src/api/deprecatedResult.ts index 01583678df..ecacc62c65 100644 --- a/plugins/catalog-backend/src/ingestion/processors/results.ts +++ b/plugins/catalog-backend/src/api/deprecatedResult.ts @@ -16,10 +16,15 @@ import { InputError, NotFoundError } from '@backstage/errors'; import { Entity } from '@backstage/catalog-model'; -import { CatalogProcessorResult, LocationSpec } from './types'; -import { EntityRelationSpec } from '../../processing/types'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; -/** @public */ +// NOTE: This entire file is deprecated and should be eventually removed along with the `result` export + +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function notFoundError( atLocation: LocationSpec, message: string, @@ -31,7 +36,10 @@ export function notFoundError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function inputError( atLocation: LocationSpec, message: string, @@ -43,7 +51,10 @@ export function inputError( }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function generalError( atLocation: LocationSpec, message: string, @@ -51,7 +62,10 @@ export function generalError( return { type: 'error', location: atLocation, error: new Error(message) }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function location( newLocation: LocationSpec, optional?: boolean, @@ -59,7 +73,10 @@ export function location( return { type: 'location', location: newLocation, optional }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function entity( atLocation: LocationSpec, newEntity: Entity, @@ -67,7 +84,10 @@ export function entity( return { type: 'entity', location: atLocation, entity: newEntity }; } -/** @public */ +/** + * @public + * @deprecated import the processingResult symbol instead and use its fields + */ export function relation(spec: EntityRelationSpec): CatalogProcessorResult { return { type: 'relation', relation: spec }; } diff --git a/plugins/catalog-backend/src/api/index.ts b/plugins/catalog-backend/src/api/index.ts new file mode 100644 index 0000000000..3f1c627743 --- /dev/null +++ b/plugins/catalog-backend/src/api/index.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as results from './deprecatedResult'; + +export { results }; + +export { processingResult } from './processingResult'; +export type { EntityRelationSpec, LocationSpec } from './common'; +export type { + CatalogProcessor, + CatalogProcessorParser, + CatalogProcessorCache, + CatalogProcessorEmit, + CatalogProcessorLocationResult, + CatalogProcessorEntityResult, + CatalogProcessorRelationResult, + CatalogProcessorErrorResult, + CatalogProcessorResult, +} from './processor'; +export type { + EntityProvider, + EntityProviderConnection, + EntityProviderMutation, +} from './provider'; diff --git a/plugins/catalog-backend/src/api/processingResult.ts b/plugins/catalog-backend/src/api/processingResult.ts new file mode 100644 index 0000000000..5ad908dc8b --- /dev/null +++ b/plugins/catalog-backend/src/api/processingResult.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { InputError, NotFoundError } from '@backstage/errors'; +import { Entity } from '@backstage/catalog-model'; +import { CatalogProcessorResult } from './processor'; +import { EntityRelationSpec, LocationSpec } from './common'; + +/** + * Factory functions for the standard processing result types. + * + * @public + */ +export const processingResult = Object.freeze({ + notFoundError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new NotFoundError(message), + }; + }, + + inputError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { + type: 'error', + location: atLocation, + error: new InputError(message), + }; + }, + + generalError( + atLocation: LocationSpec, + message: string, + ): CatalogProcessorResult { + return { type: 'error', location: atLocation, error: new Error(message) }; + }, + + location( + newLocation: LocationSpec, + optional?: boolean, + ): CatalogProcessorResult { + return { type: 'location', location: newLocation, optional }; + }, + + entity(atLocation: LocationSpec, newEntity: Entity): CatalogProcessorResult { + return { type: 'entity', location: atLocation, entity: newEntity }; + }, + + relation(spec: EntityRelationSpec): CatalogProcessorResult { + return { type: 'relation', relation: spec }; + }, +} as const); diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/api/processor.ts similarity index 87% rename from plugins/catalog-backend/src/ingestion/processors/types.ts rename to plugins/catalog-backend/src/api/processor.ts index 879b4d32a8..12a008edc7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/api/processor.ts @@ -16,24 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; -import { EntityRelationSpec } from '../../processing/types'; - -/** - * Holds the entity location information. - * - * @remarks - * - * `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch. - * This flag is then set to indicate that the file can be not present. - * default value: 'required'. - * - * @public - */ -export type LocationSpec = { - type: string; - target: string; - presence?: 'optional' | 'required'; -}; +import { EntityRelationSpec, LocationSpec } from './common'; /** * @public @@ -116,20 +99,6 @@ export type CatalogProcessor = { emit: CatalogProcessorEmit, cache: CatalogProcessorCache, ): Promise; - - /** - * Handles an emitted error. - * - * @param error - The error - * @param location - The location where the error occurred - * @param emit - A sink for items resulting from this handling - * @returns Nothing - */ - handleError?( - error: Error, - location: LocationSpec, - emit: CatalogProcessorEmit, - ): Promise; }; /** @@ -193,7 +162,6 @@ export type CatalogProcessorEntityResult = { export type CatalogProcessorRelationResult = { type: 'relation'; relation: EntityRelationSpec; - entityRef?: string; }; /** @public */ diff --git a/plugins/catalog-backend/src/providers/types.ts b/plugins/catalog-backend/src/api/provider.ts similarity index 96% rename from plugins/catalog-backend/src/providers/types.ts rename to plugins/catalog-backend/src/api/provider.ts index 226c021e3c..fa7659ee32 100644 --- a/plugins/catalog-backend/src/providers/types.ts +++ b/plugins/catalog-backend/src/api/provider.ts @@ -44,7 +44,7 @@ export interface EntityProviderConnection { * @public */ export interface EntityProvider { - /** Unique name provider name used internally for caching. */ + /** Unique provider name used internally for caching. */ getProviderName(): string; /** Connect is called upon initialization by the catalog engine. */ connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts index 729be442aa..c7cd6b7026 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts @@ -28,7 +28,7 @@ import { DbRefreshStateRow, DbRelationsRow, } from './tables'; -import { createRandomRefreshInterval } from '../processing/refresh'; +import { createRandomProcessingInterval } from '../processing/refresh'; import { timestampToDateTime } from './conversion'; import { generateStableHash } from './util'; @@ -49,7 +49,7 @@ describe('Default Processing Database', () => { db: new DefaultProcessingDatabase({ database: knex, logger, - refreshInterval: createRandomRefreshInterval({ + refreshInterval: createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }), diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts index 08201323ea..1affb14b45 100644 --- a/plugins/catalog-backend/src/database/types.ts +++ b/plugins/catalog-backend/src/database/types.ts @@ -17,7 +17,8 @@ import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; import { DateTime } from 'luxon'; -import { DeferredEntity, EntityRelationSpec } from '../processing/types'; +import { EntityRelationSpec } from '../api'; +import { DeferredEntity } from '../processing/types'; /** * An abstraction for transactions of the underlying database technology. diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts index efdaa4b8c5..4c1e753300 100644 --- a/plugins/catalog-backend/src/index.ts +++ b/plugins/catalog-backend/src/index.ts @@ -20,11 +20,12 @@ * @packageDocumentation */ +export * from './api'; export * from './catalog'; export * from './ingestion'; +export * from './modules'; export * from './search'; export * from './util'; export * from './processing'; -export * from './providers'; export * from './service'; export * from './permissions'; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index d29a445a8c..13430f51dd 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultCatalogRulesEnforcer } from './CatalogRules'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; const entity = { user: { diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index b4ad394168..e4bfdb8d0f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -17,7 +17,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** * Rules to apply to catalog entities. diff --git a/plugins/catalog-backend/src/ingestion/index.ts b/plugins/catalog-backend/src/ingestion/index.ts index fcb634914b..1f4729b041 100644 --- a/plugins/catalog-backend/src/ingestion/index.ts +++ b/plugins/catalog-backend/src/ingestion/index.ts @@ -16,8 +16,6 @@ export { DefaultCatalogRulesEnforcer } from './CatalogRules'; export type { CatalogRule, CatalogRulesEnforcer } from './CatalogRules'; -export * from './processors'; -export * from './providers'; export type { AnalyzeLocationEntityField, AnalyzeLocationExistingEntity, diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index eccf583e4b..37d12d12b1 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { RecursivePartial } from '../util/RecursivePartial'; -import { LocationSpec } from './processors'; +import { LocationSpec } from '../api'; /** @public */ export type LocationAnalyzer = { diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts index 064a2ad724..249de71591 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.test.ts @@ -13,11 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger, UrlReaders } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -import { CatalogProcessorEntityResult, CatalogProcessorResult } from './types'; -import { defaultEntityDataParser } from './util/parse'; +import { + CatalogProcessorEntityResult, + CatalogProcessorResult, +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import AWSMock from 'aws-sdk-mock'; import aws from 'aws-sdk'; import path from 'path'; @@ -36,10 +40,7 @@ AWSMock.mock( 'getObject', Buffer.from( require('fs').readFileSync( - path.resolve( - __dirname, - '__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt', - ), + path.resolve(__dirname, '__fixtures__/awsS3-mock-object.txt'), ), ), ); diff --git a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts index 0dc44cc157..737f8f7c2f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AwsS3DiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/aws/AwsS3DiscoveryProcessor.ts @@ -17,13 +17,13 @@ import { UrlReader } from '@backstage/backend-common'; import { isError } from '@backstage/errors'; import limiterFactory from 'p-limit'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; /** @public */ export class AwsS3DiscoveryProcessor implements CatalogProcessor { @@ -58,10 +58,10 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { if (isError(error) && error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt rename to plugins/catalog-backend/src/modules/aws/__fixtures__/awsS3-mock-object.txt diff --git a/packages/core-app-api/src/apis/implementations/auth/auth0/index.ts b/plugins/catalog-backend/src/modules/aws/index.ts similarity index 83% rename from packages/core-app-api/src/apis/implementations/auth/auth0/index.ts rename to plugins/catalog-backend/src/modules/aws/index.ts index 9daed7d13e..9477d4104d 100644 --- a/packages/core-app-api/src/apis/implementations/auth/auth0/index.ts +++ b/plugins/catalog-backend/src/modules/aws/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export { default as Auth0Auth } from './Auth0Auth'; +export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts index a406feaa74..450a0482ed 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.test.ts @@ -16,14 +16,14 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { codeSearch } from './azure'; +import { codeSearch } from './lib'; import { AzureDevOpsDiscoveryProcessor, parseUrl, } from './AzureDevOpsDiscoveryProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; -jest.mock('./azure'); +jest.mock('./lib'); const mockCodeSearch = codeSearch as jest.MockedFunction; describe('AzureDevOpsDiscoveryProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts index bc74621f1d..8e36e810b8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/azure/AzureDevOpsDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { codeSearch } from './azure'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { codeSearch } from './lib'; /** * Extracts repositories out of an Azure DevOps org. @@ -102,7 +106,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { for (const file of files) { emit( - results.location({ + processingResult.location({ type: 'url', target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/modules/azure/index.ts b/plugins/catalog-backend/src/modules/azure/index.ts new file mode 100644 index 0000000000..9c76336804 --- /dev/null +++ b/plugins/catalog-backend/src/modules/azure/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/modules/azure/lib/azure.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/azure.ts rename to plugins/catalog-backend/src/modules/azure/lib/azure.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/modules/azure/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/azure/index.ts rename to plugins/catalog-backend/src/modules/azure/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts index 9ee8bf1b57..857f999a70 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts @@ -13,19 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; import { ConfigReader } from '@backstage/config'; -import { - BitbucketRepository20, - BitbucketRepositoryParser, - PagedResponse, - PagedResponse20, -} from './bitbucket'; -import { LocationSpec } from './types'; -import { results } from './index'; import { RequestHandler, rest } from 'msw'; import { setupServer } from 'msw/node'; +import { BitbucketRepository20, PagedResponse, PagedResponse20 } from './lib'; +import { LocationSpec, processingResult } from '../../api'; const server = setupServer(); @@ -742,15 +737,6 @@ describe('BitbucketDiscoveryProcessor', () => { }); describe('Custom repository parser', () => { - const customRepositoryParser: BitbucketRepositoryParser = - async function* customRepositoryParser({}) { - yield results.location({ - type: 'custom-location-type', - target: 'custom-target', - presence: 'optional', - }); - }; - const processor = BitbucketDiscoveryProcessor.fromConfig( new ConfigReader({ integrations: { @@ -763,7 +749,16 @@ describe('BitbucketDiscoveryProcessor', () => { ], }, }), - { parser: customRepositoryParser, logger: getVoidLogger() }, + { + parser: async function* customRepositoryParser({}) { + yield processingResult.location({ + type: 'custom-location-type', + target: 'custom-target', + presence: 'optional', + }); + }, + logger: getVoidLogger(), + }, ); it('use custom repository parser', async () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts index 8fcf64dc55..daf24e2427 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Logger } from 'winston'; import { Config } from '@backstage/config'; @@ -22,15 +23,19 @@ import { ScmIntegrations, } from '@backstage/integration'; import { - BitbucketRepositoryParser, BitbucketClient, defaultRepositoryParser, paginated, paginated20, BitbucketRepository, BitbucketRepository20, -} from './bitbucket'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorResult, + LocationSpec, +} from '../../api'; const DEFAULT_BRANCH = 'master'; const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml'; @@ -39,12 +44,23 @@ const EMPTY_CATALOG_LOCATION = '/'; /** @public */ export class BitbucketDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrationRegistry; - private readonly parser: BitbucketRepositoryParser; + private readonly parser: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; private readonly logger: Logger; static fromConfig( config: Config, - options: { parser?: BitbucketRepositoryParser; logger: Logger }, + options: { + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; + logger: Logger; + }, ) { const integrations = ScmIntegrations.fromConfig(config); @@ -56,7 +72,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor { constructor(options: { integrations: ScmIntegrationRegistry; - parser?: BitbucketRepositoryParser; + parser?: (options: { + integration: BitbucketIntegration; + target: string; + logger: Logger; + }) => AsyncIterable; logger: Logger; }) { this.integrations = options.integrations; diff --git a/plugins/catalog-backend/src/modules/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/index.ts new file mode 100644 index 0000000000..0e39083520 --- /dev/null +++ b/plugins/catalog-backend/src/modules/bitbucket/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; +export type { BitbucketRepositoryParser } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts similarity index 83% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts index 35e24ed4cb..9f2d8b2daa 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.test.ts @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import { processingResult } from '../../../api'; import { defaultRepositoryParser } from './BitbucketRepositoryParser'; -import { results } from '../index'; -import { getVoidLogger } from '@backstage/backend-common'; -import { BitbucketIntegration } from '@backstage/integration'; describe('BitbucketRepositoryParser', () => { describe('defaultRepositoryParser', () => { @@ -25,16 +24,14 @@ describe('BitbucketRepositoryParser', () => { 'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse'; const path = '/catalog-info.yaml'; const expected = [ - results.location({ + processingResult.location({ type: 'url', target: `${browseUrl}${path}`, presence: 'optional', }), ]; const actual = await defaultRepositoryParser({ - integration: {} as BitbucketIntegration, target: `${browseUrl}${path}`, - logger: getVoidLogger(), }); let i = 0; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts similarity index 79% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts index 9eaec221b5..c72e560c21 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts @@ -13,23 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { CatalogProcessorResult } from '../types'; -import { results } from '../index'; -import { Logger } from 'winston'; -import { BitbucketIntegration } from '@backstage/integration'; -/** @public */ +import { BitbucketIntegration } from '@backstage/integration'; +import { Logger } from 'winston'; +import { CatalogProcessorResult, processingResult } from '../../../api'; + +/** + * @public + * @deprecated type inlined. + */ export type BitbucketRepositoryParser = (options: { integration: BitbucketIntegration; target: string; logger: Logger; }) => AsyncIterable; -export const defaultRepositoryParser: BitbucketRepositoryParser = - async function* defaultRepositoryParser({ target }) { - yield results.location({ +export const defaultRepositoryParser = + async function* defaultRepositoryParser(options: { target: string }) { + yield processingResult.location({ type: 'url', - target: target, + target: options.target, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting // an error if it couldn't. diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/client.ts index 0c132bb61b..65cd28d5ab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/client.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fetch from 'node-fetch'; +import fetch from 'node-fetch'; import { BitbucketIntegrationConfig, getBitbucketRequestOptions, diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/index.ts index 4b28c7f115..a819bb3c64 100644 --- a/plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts +++ b/plugins/catalog-backend/src/modules/bitbucket/lib/index.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export { BitbucketClient, paginated, paginated20 } from './client'; export { defaultRepositoryParser } from './BitbucketRepositoryParser'; export type { PagedResponse, PagedResponse20 } from './client'; diff --git a/plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts b/plugins/catalog-backend/src/modules/bitbucket/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts rename to plugins/catalog-backend/src/modules/bitbucket/lib/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts index 8d6d1f8ff7..8cf250de2a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.test.ts @@ -17,7 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { CodeOwnersProcessor } from './CodeOwnersProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; const mockCodeOwnersText = () => ` * @acme/team-foo @acme/team-bar diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts rename to plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts index 93e3c1618a..6f6aee7161 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts +++ b/plugins/catalog-backend/src/modules/codeowners/CodeOwnersProcessor.ts @@ -22,11 +22,10 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import { findCodeOwnerByTarget } from './codeowners'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; +import { findCodeOwnerByTarget } from './lib'; const ALLOWED_KINDS = ['API', 'Component', 'Domain', 'Resource', 'System']; - const ALLOWED_LOCATION_TYPES = ['url']; /** @public */ diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts b/plugins/catalog-backend/src/modules/codeowners/index.ts similarity index 84% rename from plugins/scaffolder/src/components/FavouriteTemplate/index.ts rename to plugins/catalog-backend/src/modules/codeowners/index.ts index 5955e88c01..6b7569ff07 100644 --- a/plugins/scaffolder/src/components/FavouriteTemplate/index.ts +++ b/plugins/catalog-backend/src/modules/codeowners/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export * from './FavouriteTemplate'; +export { CodeOwnersProcessor } from './CodeOwnersProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts b/plugins/catalog-backend/src/modules/codeowners/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/index.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts b/plugins/catalog-backend/src/modules/codeowners/lib/read.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/read.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/read.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.test.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts b/plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/resolve.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/resolve.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts b/plugins/catalog-backend/src/modules/codeowners/lib/scm.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/codeowners/scm.ts rename to plugins/catalog-backend/src/modules/codeowners/lib/scm.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts index 5f5b20d1c2..3911f7f743 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.test.ts @@ -17,7 +17,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; import { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; describe('AnnotateLocationEntityProcessor', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts index b8c31ed5a2..4af0b8cf61 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateLocationEntityProcessor.ts @@ -25,7 +25,11 @@ import { } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** @public */ export class AnnotateLocationEntityProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts index 311dd13789..1e553a44f8 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.test.ts @@ -16,7 +16,7 @@ import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('AnnotateScmSlugEntityProcessor', () => { describe('github', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts index 4d3b3aff6c..e44db4d44d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AnnotateScmSlugEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/AnnotateScmSlugEntityProcessor.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import parseGitUrl from 'git-url-parse'; import { identity, merge, pickBy } from 'lodash'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug'; diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts index 89b0bb1bb5..9e420b9d65 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/BuiltinKindsEntityProcessor.ts @@ -22,7 +22,7 @@ import { DomainEntity, domainEntityV1alpha1Validator, Entity, - getEntityName, + getCompoundEntityRef, GroupEntity, groupEntityV1alpha1Validator, locationEntityV1alpha1Validator, @@ -52,8 +52,12 @@ import { TemplateEntityV1beta2, templateEntityV1beta2Validator, } from '@backstage/plugin-scaffolder-common'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** @public */ export class BuiltinKindsEntityProcessor implements CatalogProcessor { @@ -89,7 +93,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); /* * Utilities @@ -107,7 +111,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { for (const target of [targets].flat()) { const targetRef = parseEntityRef(target, context); emit( - result.relation({ + processingResult.relation({ source: selfRef, type: outgoingRelation, target: { @@ -118,7 +122,7 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { }), ); emit( - result.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace, diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts index 0663ad8d04..d61342e3d0 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.test.ts @@ -17,7 +17,7 @@ import { ConfigReader } from '@backstage/config'; import path from 'path'; import { ConfigLocationEntityProvider } from './ConfigLocationEntityProvider'; -import { EntityProviderConnection } from './types'; +import { EntityProviderConnection } from '../../api'; describe('ConfigLocationEntityProvider', () => { it('should apply mutation with the correct paths in the config', async () => { diff --git a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts similarity index 90% rename from plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts rename to plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts index 389cd8d9ff..9529a5b608 100644 --- a/plugins/catalog-backend/src/providers/ConfigLocationEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/core/ConfigLocationEntityProvider.ts @@ -16,9 +16,9 @@ import { Config } from '@backstage/config'; import path from 'path'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; export class ConfigLocationEntityProvider implements EntityProvider { constructor(private readonly config: Config) {} diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts similarity index 98% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts index b9d129de5f..dfbc37f6c9 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.test.ts @@ -15,7 +15,7 @@ */ import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; -import { applyDatabaseMigrations } from '../database/migrations'; +import { applyDatabaseMigrations } from '../../database/migrations'; import { DefaultLocationStore } from './DefaultLocationStore'; describe('DefaultLocationStore', () => { diff --git a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts similarity index 93% rename from plugins/catalog-backend/src/providers/DefaultLocationStore.ts rename to plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts index 9d5971508f..d637f4eda0 100644 --- a/plugins/catalog-backend/src/providers/DefaultLocationStore.ts +++ b/plugins/catalog-backend/src/modules/core/DefaultLocationStore.ts @@ -18,11 +18,11 @@ import { Location } from '@backstage/catalog-client'; import { ConflictError, NotFoundError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DbLocationsRow } from '../database/tables'; -import { getEntityLocationRef } from '../processing/util'; -import { EntityProvider, EntityProviderConnection } from './types'; -import { locationSpecToLocationEntity } from '../util/conversion'; -import { LocationInput, LocationStore } from '../service'; +import { DbLocationsRow } from '../../database/tables'; +import { getEntityLocationRef } from '../../processing/util'; +import { EntityProvider, EntityProviderConnection } from '../../api'; +import { locationSpecToLocationEntity } from '../../util/conversion'; +import { LocationInput, LocationStore } from '../../service'; export class DefaultLocationStore implements LocationStore, EntityProvider { private _connection: EntityProviderConnection | undefined; diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts similarity index 94% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts index c2cce66bc6..a3eb5e4542 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts @@ -19,12 +19,16 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; import path from 'path'; -import { defaultEntityDataParser } from './util/parse'; +import { defaultEntityDataParser } from '../util/parse'; describe('FileReaderProcessor', () => { - const fixturesRoot = path.join(__dirname, '__fixtures__/fileReaderProcessor'); + const fixturesRoot = path.join( + __dirname, + '__fixtures__', + 'fileReaderProcessor', + ); it('should load from file', async () => { const processor = new FileReaderProcessor(); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts index 996beff24a..66c6479fa7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts @@ -18,13 +18,13 @@ import fs from 'fs-extra'; import g from 'glob'; import path from 'path'; import { promisify } from 'util'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const glob = promisify(g); @@ -65,11 +65,11 @@ export class FileReaderProcessor implements CatalogProcessor { } } else if (!optional) { const message = `${location.type} ${location.target} does not exist`; - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } catch (e) { const message = `${location.type} ${location.target} could not be read, ${e}`; - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts index 83f1ca1925..92ba1fcebf 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.test.ts @@ -21,7 +21,7 @@ import { } from '@backstage/integration'; import path from 'path'; import { toAbsoluteUrl } from './LocationEntityProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; describe('LocationEntityProcessor', () => { describe('toAbsoluteUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts rename to plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts index 533a366f57..c168f54bf1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/LocationEntityProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/LocationEntityProcessor.ts @@ -17,8 +17,12 @@ import { Entity, LocationEntity } from '@backstage/catalog-model'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; export function toAbsoluteUrl( integrations: ScmIntegrationRegistry, @@ -62,7 +66,7 @@ export class LocationEntityProcessor implements CatalogProcessor { const type = locationEntity.spec.type || location.type; if (type === 'file' && location.target.endsWith(path.sep)) { emit( - result.inputError( + processingResult.inputError( location, `LocationEntityProcessor cannot handle ${type} type location with target ${location.target} that ends with a path separator`, ), @@ -83,7 +87,7 @@ export class LocationEntityProcessor implements CatalogProcessor { location, maybeRelativeTarget, ); - emit(result.location({ type, target })); + emit(processingResult.location({ type, target })); } } diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts rename to plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts index 0d572a024d..ddd5f951db 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts @@ -19,7 +19,7 @@ import { Entity } from '@backstage/catalog-model'; import { JsonValue } from '@backstage/types'; import { ScmIntegrationRegistry } from '@backstage/integration'; import yaml from 'yaml'; -import { CatalogProcessor, LocationSpec } from './types'; +import { CatalogProcessor, LocationSpec } from '../../api'; /** @public */ export type PlaceholderResolverRead = (url: string) => Promise; diff --git a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts rename to plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts index 1c2c27f425..e22d011695 100644 --- a/plugins/catalog-backend/src/ingestion/processors/StaticLocationProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/StaticLocationProcessor.ts @@ -15,8 +15,11 @@ */ import { Config } from '@backstage/config'; -import * as result from './results'; -import { CatalogProcessorEmit, LocationSpec } from './types'; +import { + processingResult, + CatalogProcessorEmit, + LocationSpec, +} from '../../api'; /** * @deprecated no longer in use, replaced by the ConfigLocationEntityProvider. @@ -48,7 +51,7 @@ export class StaticLocationProcessor implements StaticLocationProcessor { } for (const staticLocation of this.staticLocations) { - emit(result.location(staticLocation)); + emit(processingResult.location(staticLocation)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts similarity index 98% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts index b7dc0084f2..a96d392c52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts @@ -28,9 +28,9 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorResult, -} from './types'; +} from '../../api'; +import { defaultEntityDataParser } from '../util/parse'; import { UrlReaderProcessor } from './UrlReaderProcessor'; -import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts rename to plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts index c2be454bcf..7f27bcd19d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts @@ -20,7 +20,6 @@ import { assertError } from '@backstage/errors'; import parseGitUrl from 'git-url-parse'; import limiterFactory from 'p-limit'; import { Logger } from 'winston'; -import * as result from './results'; import { CatalogProcessor, CatalogProcessorCache, @@ -29,7 +28,8 @@ import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from './types'; + processingResult, +} from '../../api'; const CACHE_KEY = 'v1'; @@ -102,10 +102,10 @@ export class UrlReaderProcessor implements CatalogProcessor { } } else if (error.name === 'NotFoundError') { if (!optional) { - emit(result.notFoundError(location, message)); + emit(processingResult.notFoundError(location, message)); } } else { - emit(result.generalError(location, message)); + emit(processingResult.generalError(location, message)); } } diff --git a/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt new file mode 100644 index 0000000000..7470c0e8a3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/awsS3/awsS3-mock-object.txt @@ -0,0 +1 @@ +site_name: Test diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt b/plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt rename to plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/test.txt diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/modules/core/index.ts similarity index 63% rename from plugins/catalog-backend/src/ingestion/processors/index.ts rename to plugins/catalog-backend/src/modules/core/index.ts index 84a92fb1ba..9d582bfb34 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/modules/core/index.ts @@ -14,20 +14,10 @@ * limitations under the License. */ -import * as results from './results'; - export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { AnnotateScmSlugEntityProcessor } from './AnnotateScmSlugEntityProcessor'; -export { AwsS3DiscoveryProcessor } from './AwsS3DiscoveryProcessor'; -export { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; -export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; -export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; -export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; -export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; -export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export type { LocationEntityProcessorOptions } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; @@ -39,10 +29,5 @@ export type { PlaceholderResolverResolveUrl, } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; -export * from './types'; export { UrlReaderProcessor } from './UrlReaderProcessor'; -export { parseEntityYaml } from './util/parse'; -export { results }; - -export type { BitbucketRepositoryParser } from './bitbucket'; -export type { GithubMultiOrgConfig } from './github'; +export { parseEntityYaml } from '../util/parse'; diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts index e7aec1520d..dd3bf490fb 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.test.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.test.ts @@ -20,10 +20,12 @@ import { GithubCredentialsProvider, GitHubIntegrationConfig, } from '@backstage/integration'; -import { GitHubOrgEntityProvider } from '.'; -import { EntityProviderConnection } from '../../providers'; -import { withLocations } from './GitHubOrgEntityProvider'; import { graphql } from '@octokit/graphql'; +import { EntityProviderConnection } from '../../api'; +import { + GitHubOrgEntityProvider, + withLocations, +} from './GitHubOrgEntityProvider'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts similarity index 97% rename from plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts rename to plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts index a20f7fdb0b..76389968ae 100644 --- a/plugins/catalog-backend/src/ingestion/providers/GitHubOrgEntityProvider.ts +++ b/plugins/catalog-backend/src/modules/github/GitHubOrgEntityProvider.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -29,13 +30,13 @@ import { import { graphql } from '@octokit/graphql'; import { merge } from 'lodash'; import { Logger } from 'winston'; -import { EntityProvider, EntityProviderConnection } from '../../providers'; +import { EntityProvider, EntityProviderConnection } from '../../api'; import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from '../processors/github'; -import { assignGroupsToUsers, buildOrgHierarchy } from '../processors/util/org'; +} from './lib'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; // TODO: Consider supporting an (optional) webhook that reacts on org changes /** @public */ diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts index 55e5560451..5440539b27 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.test.ts @@ -15,16 +15,16 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; -import { getOrganizationRepositories } from './github'; -import { LocationSpec } from './types'; import { ConfigReader } from '@backstage/config'; import { - ScmIntegrations, DefaultGithubCredentialsProvider, + ScmIntegrations, } from '@backstage/integration'; +import { LocationSpec } from '../../api'; +import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor'; +import { getOrganizationRepositories } from './lib'; -jest.mock('./github'); +jest.mock('./lib'); const mockGetOrganizationRepositories = getOrganizationRepositories as jest.MockedFunction< typeof getOrganizationRepositories diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts similarity index 96% rename from plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts index e790134011..4aff470f52 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubDiscoveryProcessor.ts @@ -23,9 +23,13 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; -import { getOrganizationRepositories } from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; +import { getOrganizationRepositories } from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; /** * Extracts repositories out of a GitHub org. @@ -141,7 +145,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { const path = `/blob/${branchName}${catalogPath}`; emit( - results.location({ + processingResult.location({ type: 'url', target: `${repository.url}${path}`, // Not all locations may actually exist, since the user defined them as a wildcard pattern. diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts similarity index 95% rename from plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts index 383b82c019..339119137a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubMultiOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubMultiOrgReaderProcessor.ts @@ -30,10 +30,14 @@ import { getOrganizationUsers, GithubMultiOrgConfig, readGithubMultiOrgConfig, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { buildOrgHierarchy } from '../util/org'; /** * @alpha @@ -158,7 +162,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { buildOrgHierarchy(groups); for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } } catch (e) { this.logger.error( @@ -169,7 +173,7 @@ export class GithubMultiOrgReaderProcessor implements CatalogProcessor { const allUsers = Array.from(allUsersMap.values()); for (const user of allUsers) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts index c01db88d3a..9ce278ac44 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { @@ -21,7 +22,7 @@ import { } from '@backstage/integration'; import { graphql } from '@octokit/graphql'; import { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; -import { LocationSpec } from './types'; +import { LocationSpec } from '../../api'; jest.mock('@octokit/graphql'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts rename to plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts index 8b5914cc80..e85e7b874d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/modules/github/GithubOrgReaderProcessor.ts @@ -28,10 +28,14 @@ import { getOrganizationTeams, getOrganizationUsers, parseGitHubOrgUrl, -} from './github'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { assignGroupsToUsers, buildOrgHierarchy } from './util/org'; +} from './lib'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { assignGroupsToUsers, buildOrgHierarchy } from '../util/org'; type GraphQL = typeof graphql; @@ -106,10 +110,10 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { // Done! for (const group of groups) { - emit(results.entity(location, group)); + emit(processingResult.entity(location, group)); } for (const user of users) { - emit(results.entity(location, user)); + emit(processingResult.entity(location, user)); } return true; diff --git a/plugins/catalog-backend/src/modules/github/index.ts b/plugins/catalog-backend/src/modules/github/index.ts new file mode 100644 index 0000000000..7958818096 --- /dev/null +++ b/plugins/catalog-backend/src/modules/github/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; +export type { GithubMultiOrgConfig } from './lib'; diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.test.ts b/plugins/catalog-backend/src/modules/github/lib/config.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.test.ts rename to plugins/catalog-backend/src/modules/github/lib/config.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/config.ts b/plugins/catalog-backend/src/modules/github/lib/config.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/config.ts rename to plugins/catalog-backend/src/modules/github/lib/config.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/modules/github/lib/github.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.test.ts rename to plugins/catalog-backend/src/modules/github/lib/github.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/modules/github/lib/github.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/github.ts rename to plugins/catalog-backend/src/modules/github/lib/github.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/modules/github/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/github/index.ts rename to plugins/catalog-backend/src/modules/github/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts b/plugins/catalog-backend/src/modules/github/lib/util.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.test.ts rename to plugins/catalog-backend/src/modules/github/lib/util.test.ts index 02f54e0675..c73c11bd2d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.test.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.test.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { parseGitHubOrgUrl } from './util'; describe('parseGitHubOrgUrl', () => { diff --git a/plugins/catalog-backend/src/ingestion/processors/github/util.ts b/plugins/catalog-backend/src/modules/github/lib/util.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/github/util.ts rename to plugins/catalog-backend/src/modules/github/lib/util.ts index d8df376038..a38225ff98 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/util.ts +++ b/plugins/catalog-backend/src/modules/github/lib/util.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + export function parseGitHubOrgUrl(urlString: string): { org: string } { const path = new URL(urlString).pathname.substr(1).split('/'); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts similarity index 99% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts index 8261690e8f..cc0f368aee 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.test.ts @@ -19,8 +19,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from './gitlab'; -import { LocationSpec } from './types'; +import { GitLabProject } from './lib'; +import { LocationSpec } from '../../api'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts similarity index 92% rename from plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts rename to plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts index b534d50044..d3916f45d1 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/modules/gitlab/GitLabDiscoveryProcessor.ts @@ -20,9 +20,13 @@ import { ScmIntegrations, } from '@backstage/integration'; import { Logger } from 'winston'; -import * as results from './results'; -import { CatalogProcessor, CatalogProcessorEmit, LocationSpec } from './types'; -import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CatalogProcessor, + CatalogProcessorEmit, + LocationSpec, + processingResult, +} from '../../api'; +import { GitLabClient, GitLabProject, paginated } from './lib'; import { CacheClient, CacheManager, @@ -95,12 +99,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { page: 1, }); - const result: Result = { + const res: Result = { scanned: 0, matches: [], }; for await (const project of projects) { - result.scanned++; + res.scanned++; if (project.archived) { continue; @@ -110,14 +114,14 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { continue; } - result.matches.push(project); + res.matches.push(project); } - for (const project of result.matches) { + for (const project of res.matches) { const project_branch = branch === '*' ? project.default_branch : branch; emit( - results.location({ + processingResult.location({ type: 'url', // The format expected by the GitLabUrlReader: // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath @@ -133,7 +137,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); this.logger.debug( - `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + `Read ${res.scanned} GitLab repositories in ${duration} seconds`, ); return true; diff --git a/plugins/catalog-backend/src/modules/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/index.ts new file mode 100644 index 0000000000..f42a901f8f --- /dev/null +++ b/plugins/catalog-backend/src/modules/gitlab/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts index b249137740..a7c0d5161f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.test.ts +++ b/plugins/catalog-backend/src/modules/gitlab/lib/client.test.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { ConfigReader } from '@backstage/config'; import { setupRequestMockHandlers } from '@backstage/test-utils'; import { readGitLabIntegrationConfig } from '@backstage/integration'; import { getVoidLogger } from '@backstage/backend-common'; import { rest } from 'msw'; import { setupServer, SetupServerApi } from 'msw/node'; - import { GitLabClient, paginated } from './client'; const server = setupServer(); diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/modules/gitlab/lib/client.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/client.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/modules/gitlab/lib/index.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/modules/gitlab/lib/types.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts rename to plugins/catalog-backend/src/modules/gitlab/lib/types.ts diff --git a/plugins/catalog-backend/src/modules/index.ts b/plugins/catalog-backend/src/modules/index.ts new file mode 100644 index 0000000000..6acf057ca3 --- /dev/null +++ b/plugins/catalog-backend/src/modules/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './aws'; +export * from './azure'; +export * from './bitbucket'; +export * from './codeowners'; +export * from './core'; +export * from './github'; +export * from './gitlab'; diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.test.ts b/plugins/catalog-backend/src/modules/util/org.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.test.ts rename to plugins/catalog-backend/src/modules/util/org.test.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/org.ts b/plugins/catalog-backend/src/modules/util/org.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/org.ts rename to plugins/catalog-backend/src/modules/util/org.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts b/plugins/catalog-backend/src/modules/util/parse.test.ts similarity index 91% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts rename to plugins/catalog-backend/src/modules/util/parse.test.ts index a42d160725..e87de1c9a3 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts +++ b/plugins/catalog-backend/src/modules/util/parse.test.ts @@ -15,7 +15,7 @@ */ import { parseEntityYaml } from './parse'; -import * as result from '../results'; +import { processingResult } from '../../api'; const testLoc = { target: 'my-loc-target', @@ -47,7 +47,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -92,7 +92,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -102,7 +102,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -137,7 +137,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -157,7 +157,7 @@ describe('parseEntityYaml', () => { // Parse errors are always per document expect(results).toEqual([ - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Plain value cannot start with reserved character `', ), @@ -186,7 +186,7 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.entity(testLoc, { + processingResult.entity(testLoc, { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { @@ -196,7 +196,7 @@ describe('parseEntityYaml', () => { type: 'website', }, }), - result.generalError( + processingResult.generalError( testLoc, 'YAML error at my-loc-type:my-loc-target, YAMLSemanticError: Nested mappings are not allowed in compact mappings', ), @@ -209,7 +209,10 @@ describe('parseEntityYaml', () => { ); expect(results).toEqual([ - result.generalError(testLoc, 'Expected object at root, got string'), + processingResult.generalError( + testLoc, + 'Expected object at root, got string', + ), ]); }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/modules/util/parse.ts similarity index 87% rename from plugins/catalog-backend/src/ingestion/processors/util/parse.ts rename to plugins/catalog-backend/src/modules/util/parse.ts index 0beaf1cd4c..65fad5986f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/modules/util/parse.ts @@ -17,12 +17,12 @@ import { Entity, stringifyLocationRef } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; -import * as result from '../results'; import { CatalogProcessorParser, CatalogProcessorResult, LocationSpec, -} from '../types'; + processingResult, +} from '../../api'; /** @public */ export function* parseEntityYaml( @@ -35,7 +35,7 @@ export function* parseEntityYaml( } catch (e) { const loc = stringifyLocationRef(location); const message = `Failed to parse YAML at ${loc}, ${e}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); return; } @@ -43,17 +43,17 @@ export function* parseEntityYaml( if (document.errors?.length) { const loc = stringifyLocationRef(location); const message = `YAML error at ${loc}, ${document.errors[0]}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } else { const json = document.toJSON(); if (lodash.isPlainObject(json)) { - yield result.entity(location, json as Entity); + yield processingResult.entity(location, json as Entity); } else if (json === null) { // Ignore null values, these happen if there is an empty document in the // YAML file, for example if --- is added to the end of the file. } else { const message = `Expected object at root, got ${typeof json}`; - yield result.generalError(location, message); + yield processingResult.generalError(location, message); } } } diff --git a/plugins/catalog-backend/src/permissions/conditionExports.ts b/plugins/catalog-backend/src/permissions/conditionExports.ts index 28e74c53c2..2846c8c3ec 100644 --- a/plugins/catalog-backend/src/permissions/conditionExports.ts +++ b/plugins/catalog-backend/src/permissions/conditionExports.ts @@ -27,7 +27,8 @@ const conditionExports = createConditionExports({ /** * These conditions are used when creating conditional decisions that are returned * by authorization policies. - * @public + * + * @alpha */ export const catalogConditions = conditionExports.conditions; @@ -50,7 +51,8 @@ export const catalogConditions = conditionExports.conditions; * } * } * ``` - * @public + * + * @alpha */ export const createCatalogPolicyDecision = conditionExports.createPolicyDecision; diff --git a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts index 81ebd52789..c6c5951153 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasAnnotation.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for the presence of an annotation on a given entity. * - * @public + * @alpha */ export const hasAnnotation = createCatalogPermissionRule({ name: 'HAS_ANNOTATION', diff --git a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts index 04b00d68fa..7296eecd89 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasLabel.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasLabel.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified label in its metadata. - * @public + * @alpha */ export const hasLabel = createCatalogPermissionRule({ name: 'HAS_LABEL', diff --git a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts index f5f25a5ecf..cf14faea14 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasMetadata.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasMetadata = createPropertyRule('metadata'); diff --git a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts index 891cf1d58c..63ae67d01c 100644 --- a/plugins/catalog-backend/src/permissions/rules/hasSpec.ts +++ b/plugins/catalog-backend/src/permissions/rules/hasSpec.ts @@ -23,6 +23,6 @@ import { createPropertyRule } from './createPropertyRule'; * * The key argument to the `apply` and `toQuery` methods can be nested, such as * 'field.nestedfield'. - * @public + * @alpha */ export const hasSpec = createPropertyRule('spec'); diff --git a/plugins/catalog-backend/src/permissions/rules/index.ts b/plugins/catalog-backend/src/permissions/rules/index.ts index 4eec796c74..f9d192781e 100644 --- a/plugins/catalog-backend/src/permissions/rules/index.ts +++ b/plugins/catalog-backend/src/permissions/rules/index.ts @@ -24,7 +24,8 @@ import { hasSpec } from './hasSpec'; /** * These permission rules can be used to conditionally filter catalog entities * or describe a user's access to the entities. - * @public + * + * @alpha */ export const permissionRules = { hasAnnotation, diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts index 6356c94dc4..c330c903d9 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityKind.ts @@ -20,7 +20,7 @@ import { createCatalogPermissionRule } from './util'; /** * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified kind. - * @public + * @alpha */ export const isEntityKind = createCatalogPermissionRule({ name: 'IS_ENTITY_KIND', diff --git a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts index a6dda13c29..23f118abc4 100644 --- a/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts +++ b/plugins/catalog-backend/src/permissions/rules/isEntityOwner.ts @@ -21,7 +21,7 @@ import { createCatalogPermissionRule } from './util'; * A catalog {@link @backstage/plugin-permission-node#PermissionRule} which * filters for entities with a specified owner. * - * @public + * @alpha */ export const isEntityOwner = createCatalogPermissionRule({ name: 'IS_ENTITY_OWNER', diff --git a/plugins/catalog-backend/src/permissions/rules/util.ts b/plugins/catalog-backend/src/permissions/rules/util.ts index 7ef0ca3546..a1316c7fb2 100644 --- a/plugins/catalog-backend/src/permissions/rules/util.ts +++ b/plugins/catalog-backend/src/permissions/rules/util.ts @@ -23,7 +23,7 @@ import { EntitiesSearchFilter } from '../../catalog/types'; * {@link @backstage/plugin-permission-node#PermissionRule}s for the * catalog-backend. * - * @public + * @alpha */ export const createCatalogPermissionRule = makeCreatePermissionRule< Entity, diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts index 0853484408..6e494d784e 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts @@ -29,11 +29,11 @@ import { CatalogProcessorEmit, CatalogProcessorParser, LocationSpec, - results, -} from '../ingestion'; + processingResult, +} from '../api'; import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; import { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { ConfigReader } from '@backstage/config'; class FooBarProcessor implements CatalogProcessor { @@ -51,7 +51,7 @@ class FooBarProcessor implements CatalogProcessor { ) { if (await cache.get('emit')) { emit( - results.entity( + processingResult.entity( { type: 'url', target: './new-place' }, { apiVersion: 'my-api/v1', @@ -63,7 +63,7 @@ class FooBarProcessor implements CatalogProcessor { ), ); emit( - results.relation({ + processingResult.relation({ type: 'my-type', source: { kind: 'foobar', name: 'my-source', namespace: 'default' }, target: { kind: 'foobar', name: 'my-target', namespace: 'default' }, @@ -211,7 +211,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => { getProcessorName: jest.fn(), validateEntityKind: jest.fn(async () => true), readLocation: jest.fn(async (_l, _o, emit) => { - emit(results.entity({ type: 't', target: 't' }, entity)); + emit(processingResult.entity({ type: 't', target: 't' }, entity)); return true; }), }; diff --git a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts index 9649bc937b..c050779a8b 100644 --- a/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.ts @@ -36,8 +36,8 @@ import { CatalogProcessor, CatalogProcessorParser, LocationSpec, -} from '../ingestion/processors'; -import * as results from '../ingestion/processors/results'; + processingResult, +} from '../api'; import { CatalogProcessingOrchestrator, EntityProcessingRequest, @@ -178,13 +178,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.preProcessEntity) { try { - result = await processor.preProcessEntity( - result, + res = await processor.preProcessEntity( + res, context.location, context.collector.onEmit, context.originLocation, @@ -199,7 +199,7 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } /** @@ -295,7 +295,7 @@ export class DefaultCatalogProcessingOrchestrator for (const maybeRelativeTarget of targets) { if (type === 'file' && maybeRelativeTarget.endsWith(path.sep)) { context.collector.onEmit( - results.inputError( + processingResult.inputError( context.location, `LocationEntityProcessor cannot handle ${type} type location with target ${context.location.target} that ends with a path separator`, ), @@ -351,13 +351,13 @@ export class DefaultCatalogProcessingOrchestrator entity: Entity, context: Context, ): Promise { - let result = entity; + let res = entity; for (const processor of this.options.processors) { if (processor.postProcessEntity) { try { - result = await processor.postProcessEntity( - result, + res = await processor.postProcessEntity( + res, context.location, context.collector.onEmit, context.cache.forProcessor(processor), @@ -371,6 +371,6 @@ export class DefaultCatalogProcessingOrchestrator } } - return result; + return res; } } diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts index 71631b13d8..602557a224 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { CatalogProcessor } from '../ingestion/processors'; +import { CatalogProcessor } from '../api'; import { ProcessorCacheManager } from './ProcessorCacheManager'; class MyProcessor implements CatalogProcessor { diff --git a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts index ab95fcedb7..e54641f572 100644 --- a/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts +++ b/plugins/catalog-backend/src/processing/ProcessorCacheManager.ts @@ -15,8 +15,7 @@ */ import { JsonObject, JsonValue } from '@backstage/types'; -import { CatalogProcessor } from '../ingestion/processors'; -import { CatalogProcessorCache } from '../ingestion/processors/types'; +import { CatalogProcessor, CatalogProcessorCache } from '../api'; import { isObject } from './util'; class SingleProcessorSubCache implements CatalogProcessorCache { diff --git a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts index 351e68882b..c3cc6e288b 100644 --- a/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts +++ b/plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts @@ -22,9 +22,9 @@ import { } from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { Logger } from 'winston'; -import { CatalogProcessorResult } from '../ingestion'; +import { CatalogProcessorResult, EntityRelationSpec } from '../api'; import { locationSpecToLocationEntity } from '../util/conversion'; -import { DeferredEntity, EntityRelationSpec } from './types'; +import { DeferredEntity } from './types'; import { getEntityLocationRef, getEntityOriginLocationRef, diff --git a/plugins/catalog-backend/src/processing/connectEntityProviders.ts b/plugins/catalog-backend/src/processing/connectEntityProviders.ts index 02bdc2aee3..d7015ea52e 100644 --- a/plugins/catalog-backend/src/processing/connectEntityProviders.ts +++ b/plugins/catalog-backend/src/processing/connectEntityProviders.ts @@ -23,7 +23,7 @@ import { EntityProvider, EntityProviderConnection, EntityProviderMutation, -} from '../providers/types'; +} from '../api'; class Connection implements EntityProviderConnection { readonly validateEntityEnvelope = entityEnvelopeSchemaValidator(); diff --git a/plugins/catalog-backend/src/processing/index.ts b/plugins/catalog-backend/src/processing/index.ts index 83f8a9f566..aec6a5d71c 100644 --- a/plugins/catalog-backend/src/processing/index.ts +++ b/plugins/catalog-backend/src/processing/index.ts @@ -19,10 +19,15 @@ export type { CatalogProcessingEngine, EntityProcessingRequest, EntityProcessingResult, - EntityRelationSpec, DeferredEntity, } from './types'; export { DefaultCatalogProcessingOrchestrator } from './DefaultCatalogProcessingOrchestrator'; -export { createRandomRefreshInterval } from './refresh'; -export type { RefreshIntervalFunction } from './refresh'; +export { + createRandomRefreshInterval, + createRandomProcessingInterval, +} from './refresh'; +export type { + RefreshIntervalFunction, + ProcessingIntervalFunction, +} from './refresh'; diff --git a/plugins/catalog-backend/src/processing/refresh.ts b/plugins/catalog-backend/src/processing/refresh.ts index 3e03e3add8..941c338912 100644 --- a/plugins/catalog-backend/src/processing/refresh.ts +++ b/plugins/catalog-backend/src/processing/refresh.ts @@ -16,13 +16,21 @@ /** * Function that returns the catalog refresh interval in seconds. + * @deprecated use {@link ProcessingIntervalFunction} instead * @public */ export type RefreshIntervalFunction = () => number; +/** + * Function that returns the catalog processing interval in seconds. + * @public + */ +export type ProcessingIntervalFunction = () => number; + /** * Creates a function that returns a random refresh interval between minSeconds and maxSeconds. * @returns A {@link RefreshIntervalFunction} that provides the next refresh interval + * @deprecated use {@link createRandomProcessingInterval} instead * @public */ export function createRandomRefreshInterval(options: { @@ -34,3 +42,18 @@ export function createRandomRefreshInterval(options: { return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } + +/** + * Creates a function that returns a random processing interval between minSeconds and maxSeconds. + * @returns A {@link ProcessingIntervalFunction} that provides the next processing interval + * @public + */ +export function createRandomProcessingInterval(options: { + minSeconds: number; + maxSeconds: number; +}): ProcessingIntervalFunction { + const { minSeconds, maxSeconds } = options; + return () => { + return Math.random() * (maxSeconds - minSeconds) + minSeconds; + }; +} diff --git a/plugins/catalog-backend/src/processing/types.ts b/plugins/catalog-backend/src/processing/types.ts index b732df297e..484cd73939 100644 --- a/plugins/catalog-backend/src/processing/types.ts +++ b/plugins/catalog-backend/src/processing/types.ts @@ -14,30 +14,9 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/types'; - -/** - * Holds the relation data for entities. - * - * @public - */ -export type EntityRelationSpec = { - /** - * The source entity of this relation. - */ - source: EntityName; - - /** - * The type of the relation. - */ - type: string; - - /** - * The target entity of this relation. - */ - target: EntityName; -}; +import { EntityRelationSpec } from '../api'; /** * The request to process an entity. diff --git a/plugins/catalog-backend/src/processing/util.ts b/plugins/catalog-backend/src/processing/util.ts index 94634f794a..b1948cf559 100644 --- a/plugins/catalog-backend/src/processing/util.ts +++ b/plugins/catalog-backend/src/processing/util.ts @@ -27,7 +27,7 @@ import { JsonObject, JsonValue } from '@backstage/types'; import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function isLocationEntity(entity: Entity): entity is LocationEntity { return entity.kind === 'Location'; diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts index 96704a6c1a..ac9a33bf62 100644 --- a/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollator.ts @@ -23,26 +23,23 @@ import { stringifyEntityRef, UserEntity, } from '@backstage/catalog-model'; -import { IndexableDocument, DocumentCollator } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { CatalogApi, CatalogClient, GetEntitiesRequest, } from '@backstage/catalog-client'; -import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; -/** @public */ -export interface CatalogEntityDocument extends IndexableDocument { - componentType: string; - namespace: string; - kind: string; - lifecycle: string; - owner: string; -} - -/** @public */ -export class DefaultCatalogCollator implements DocumentCollator { +/** + * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultCatalogCollatorFactory` instead. + */ +export class DefaultCatalogCollator { protected discovery: PluginEndpointDiscovery; protected locationTemplate: string; protected filter?: GetEntitiesRequest['filter']; @@ -130,6 +127,7 @@ export class DefaultCatalogCollator implements DocumentCollator { }), text: this.getDocumentText(entity), componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', namespace: entity.metadata.namespace || 'default', kind: entity.kind, lifecycle: (entity.spec?.lifecycle as string) || '', diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts new file mode 100644 index 0000000000..ccb61b8680 --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.test.ts @@ -0,0 +1,213 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; + +const server = setupServer(); + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-entity', + description: 'The expected description', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity', + name: 'test-entity-2', + description: 'The expected description 2', + }, + spec: { + type: 'some-type', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultCatalogCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + }; + + beforeAll(() => { + server.listen(); + }); + + beforeEach(() => { + server.use( + rest.get('http://localhost:7007/entities', (req, res, ctx) => { + if (req.url.searchParams.has('filter')) { + const filter = req.url.searchParams.get('filter'); + if (filter === 'kind=Foo,kind=Bar') { + // When filtering on the 'Foo,Bar' kinds we simply return no items, to simulate a filter + return res(ctx.json([])); + } + throw new Error('Unexpected filter parameter'); + } + + // Imitate offset/limit pagination. + const offset = parseInt(req.url.searchParams.get('offset') || '0', 10); + const limit = parseInt(req.url.searchParams.get('limit') || '500', 10); + return res(ctx.json(expectedEntities.slice(offset, limit + offset))); + }), + ); + }); + + afterAll(() => { + server.close(); + }); + + afterEach(() => server.resetHandlers()); + + it('has expected type', () => { + const factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('software-catalog'); + }); + + describe('getCollator', () => { + let factory: DefaultCatalogCollatorFactory; + let collator: Readable; + + beforeEach(async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog service', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(documents).toHaveLength(expectedEntities.length); + }); + + it('maps a returned entity to an expected CatalogEntityDocument', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + title: expectedEntities[0].metadata.name, + location: '/catalog/default/component/test-entity', + text: expectedEntities[0].metadata.description, + namespace: 'default', + componentType: expectedEntities[0]!.spec!.type, + lifecycle: expectedEntities[0]!.spec!.lifecycle, + owner: expectedEntities[0]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity', + }, + }); + expect(documents[1]).toMatchObject({ + title: expectedEntities[1].metadata.title, + location: '/catalog/default/component/test-entity-2', + text: expectedEntities[1].metadata.description, + namespace: 'default', + componentType: expectedEntities[1]!.spec!.type, + lifecycle: expectedEntities[1]!.spec!.lifecycle, + owner: expectedEntities[1]!.spec!.owner, + authorization: { + resourceRef: 'component:default/test-entity-2', + }, + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(documents[0]).toMatchObject({ + location: '/software/test-entity', + }); + }); + + it('allows filtering of the retrieved catalog entities', async () => { + // Provide a custom filter. + factory = DefaultCatalogCollatorFactory.fromConfig(new ConfigReader({}), { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + filter: { + kind: ['Foo', 'Bar'], + }, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // The simulated 'Foo,Bar' filter should return in an empty list + expect(documents).toHaveLength(0); + }); + + it('paginates through catalog entities using batchSize', async () => { + factory = DefaultCatalogCollatorFactory.fromConfig(config, { + ...options, + batchSize: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents).toHaveLength(expectedEntities.length); + expect(documents[0].location).toBe( + '/catalog/default/component/test-entity', + ); + expect(documents[1].location).toBe( + '/catalog/default/component/test-entity-2', + ); + }); + }); +}); diff --git a/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts new file mode 100644 index 0000000000..440a8c8ebb --- /dev/null +++ b/plugins/catalog-backend/src/search/DefaultCatalogCollatorFactory.ts @@ -0,0 +1,165 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + GetEntitiesRequest, +} from '@backstage/catalog-client'; +import { + Entity, + stringifyEntityRef, + UserEntity, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { + catalogEntityReadPermission, + CatalogEntityDocument, +} from '@backstage/plugin-catalog-common'; +import { Readable } from 'stream'; + +/** @public */ +export type DefaultCatalogCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + tokenManager: TokenManager; + locationTemplate?: string; + filter?: GetEntitiesRequest['filter']; + batchSize?: number; + catalogClient?: CatalogApi; +}; + +/** @public */ +export class DefaultCatalogCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'software-catalog'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private locationTemplate: string; + private filter?: GetEntitiesRequest['filter']; + private batchSize: number; + private readonly catalogClient: CatalogApi; + private tokenManager: TokenManager; + + static fromConfig( + _config: Config, + options: DefaultCatalogCollatorFactoryOptions, + ) { + return new DefaultCatalogCollatorFactory(options); + } + + private constructor(options: DefaultCatalogCollatorFactoryOptions) { + const { + batchSize, + discovery, + locationTemplate, + filter, + catalogClient, + tokenManager, + } = options; + + this.locationTemplate = + locationTemplate || '/catalog/:namespace/:kind/:name'; + this.filter = filter; + this.batchSize = batchSize || 500; + this.catalogClient = + catalogClient || new CatalogClient({ discoveryApi: discovery }); + this.tokenManager = tokenManager; + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted.toLowerCase(); + } + + private isUserEntity(entity: Entity): entity is UserEntity { + return entity.kind.toLocaleUpperCase('en-US') === 'USER'; + } + + private getDocumentText(entity: Entity): string { + let documentText = entity.metadata.description || ''; + if (this.isUserEntity(entity)) { + if (entity.spec?.profile?.displayName && documentText) { + // combine displayName and description + const displayName = entity.spec?.profile?.displayName; + documentText = displayName.concat(' : ', documentText); + } else { + documentText = entity.spec?.profile?.displayName || documentText; + } + } + return documentText; + } + + private async *execute(): AsyncGenerator { + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: this.filter, + limit: this.batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === this.batchSize; + entitiesRetrieved += entities.length; + + for (const entity of entities) { + yield { + title: entity.metadata.title ?? entity.metadata.name, + location: this.applyArgsToFormat(this.locationTemplate, { + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + name: entity.metadata.name, + }), + text: this.getDocumentText(entity), + componentType: entity.spec?.type?.toString() || 'other', + type: entity.spec?.type?.toString() || 'other', + namespace: entity.metadata.namespace || 'default', + kind: entity.kind, + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: (entity.spec?.owner as string) || '', + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + }; + } + } + } +} diff --git a/plugins/catalog-backend/src/search/index.ts b/plugins/catalog-backend/src/search/index.ts index 5641670333..eb5b1e7c22 100644 --- a/plugins/catalog-backend/src/search/index.ts +++ b/plugins/catalog-backend/src/search/index.ts @@ -14,5 +14,18 @@ * limitations under the License. */ +export { DefaultCatalogCollatorFactory } from './DefaultCatalogCollatorFactory'; +export type { DefaultCatalogCollatorFactoryOptions } from './DefaultCatalogCollatorFactory'; + +import { CatalogEntityDocument as CatalogEntityDocumentType } from '@backstage/plugin-catalog-common'; + +/** + * @deprecated import from `@backstage/plugin-catalog-common` instead + * @public + */ +export type CatalogEntityDocument = CatalogEntityDocumentType; + +/** + * todo(backstage/techdocs-core): stop exporting this in a future release. + */ export { DefaultCatalogCollator } from './DefaultCatalogCollator'; -export type { CatalogEntityDocument } from './DefaultCatalogCollator'; diff --git a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts index 819451d854..17dfb58c54 100644 --- a/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts +++ b/plugins/catalog-backend/src/service/AuthorizedRefreshService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { NotAllowedError } from '@backstage/errors'; import { catalogEntityRefreshPermission } from '@backstage/plugin-catalog-common'; import { diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 9004cb4c56..b53335c469 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -38,12 +38,15 @@ import { Router } from 'express'; import lodash, { keyBy } from 'lodash'; import { EntitiesCatalog, EntitiesSearchFilter } from '../catalog'; +import { + CatalogProcessor, + CatalogProcessorParser, + EntityProvider, +} from '../api'; import { AnnotateLocationEntityProcessor, BitbucketDiscoveryProcessor, BuiltinKindsEntityProcessor, - CatalogProcessor, - CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, AzureDevOpsDiscoveryProcessor, @@ -53,29 +56,29 @@ import { PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, -} from '../ingestion'; +} from '../modules'; +import { ConfigLocationEntityProvider } from '../modules/core/ConfigLocationEntityProvider'; +import { DefaultLocationStore } from '../modules/core/DefaultLocationStore'; import { RepoLocationAnalyzer } from '../ingestion/LocationAnalyzer'; import { jsonPlaceholderResolver, textPlaceholderResolver, yamlPlaceholderResolver, -} from '../ingestion/processors/PlaceholderProcessor'; -import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; +} from '../modules/core/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../modules/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; -import { EntityProvider } from '../providers/types'; import { CatalogProcessingEngine } from '../processing/types'; -import { ConfigLocationEntityProvider } from '../providers/ConfigLocationEntityProvider'; import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase'; import { applyDatabaseMigrations } from '../database/migrations'; import { DefaultCatalogProcessingEngine } from '../processing/DefaultCatalogProcessingEngine'; import { DefaultLocationService } from './DefaultLocationService'; -import { DefaultLocationStore } from '../providers/DefaultLocationStore'; import { DefaultEntitiesCatalog } from './DefaultEntitiesCatalog'; import { DefaultCatalogProcessingOrchestrator } from '../processing/DefaultCatalogProcessingOrchestrator'; import { Stitcher } from '../stitching/Stitcher'; import { - createRandomRefreshInterval, + createRandomProcessingInterval, RefreshIntervalFunction, + ProcessingIntervalFunction, } from '../processing/refresh'; import { createRouter } from './createRouter'; import { DefaultRefreshService } from './DefaultRefreshService'; @@ -137,8 +140,8 @@ export class CatalogBuilder { private processors: CatalogProcessor[]; private processorsReplace: boolean; private parser: CatalogProcessorParser | undefined; - private refreshInterval: RefreshIntervalFunction = - createRandomRefreshInterval({ + private processingInterval: ProcessingIntervalFunction = + createRandomProcessingInterval({ minSeconds: 100, maxSeconds: 150, }); @@ -190,9 +193,28 @@ export class CatalogBuilder { * Seconds provided will be multiplied by 1.5 * The default refresh duration is 100-150 seconds. * setting this too low will potentially deplete request quotas to upstream services. + * + * @deprecated use {@link CatalogBuilder#setProcessingIntervalSeconds} instead */ setRefreshIntervalSeconds(seconds: number): CatalogBuilder { - this.refreshInterval = createRandomRefreshInterval({ + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshIntervalSeconds is deprecated. Use CatalogBuilder.setProcessingIntervalSeconds instead.', + ); + this.processingInterval = createRandomProcessingInterval({ + minSeconds: seconds, + maxSeconds: seconds * 1.5, + }); + return this; + } + + /** + * Processing interval determines how often entities should be processed. + * Seconds provided will be multiplied by 1.5 + * The default processing interval is 100-150 seconds. + * setting this too low will potentially deplete request quotas to upstream services. + */ + setProcessingIntervalSeconds(seconds: number): CatalogBuilder { + this.processingInterval = createRandomProcessingInterval({ minSeconds: seconds, maxSeconds: seconds * 1.5, }); @@ -202,9 +224,25 @@ export class CatalogBuilder { /** * Overwrites the default refresh interval function used to spread * entity updates in the catalog. + * + * @deprecated use {@link CatalogBuilder#setProcessingInterval} instead */ setRefreshInterval(refreshInterval: RefreshIntervalFunction): CatalogBuilder { - this.refreshInterval = refreshInterval; + this.env.logger.warn( + '[DEPRECATION] - CatalogBuilder.setRefreshInterval is deprecated. Use CatalogBuilder.setProcessingInterval instead.', + ); + this.processingInterval = refreshInterval; + return this; + } + + /** + * Overwrites the default processing interval function used to spread + * entity updates in the catalog. + */ + setProcessingInterval( + processingInterval: ProcessingIntervalFunction, + ): CatalogBuilder { + this.processingInterval = processingInterval; return this; } @@ -394,7 +432,7 @@ export class CatalogBuilder { const processingDatabase = new DefaultProcessingDatabase({ database: dbClient, logger, - refreshInterval: this.refreshInterval, + refreshInterval: this.processingInterval, }); const integrations = ScmIntegrations.fromConfig(config); const rulesEnforcer = DefaultCatalogRulesEnforcer.fromConfig(config); diff --git a/plugins/catalog-backend/src/service/DefaultLocationService.ts b/plugins/catalog-backend/src/service/DefaultLocationService.ts index c6b5eaf162..7a1b923068 100644 --- a/plugins/catalog-backend/src/service/DefaultLocationService.ts +++ b/plugins/catalog-backend/src/service/DefaultLocationService.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, ANNOTATION_LOCATION, diff --git a/plugins/catalog-backend/src/util/conversion.ts b/plugins/catalog-backend/src/util/conversion.ts index 75ed4f5e96..63771d1028 100644 --- a/plugins/catalog-backend/src/util/conversion.ts +++ b/plugins/catalog-backend/src/util/conversion.ts @@ -23,7 +23,7 @@ import { stringifyLocationRef, } from '@backstage/catalog-model'; import { createHash } from 'crypto'; -import { LocationSpec } from '../ingestion'; +import { LocationSpec } from '../api'; export function locationSpecToMetadataName(location: LocationSpec) { const hash = createHash('sha1') diff --git a/plugins/catalog-common/api-report.md b/plugins/catalog-common/api-report.md index 9826a8ddef..a73cae9994 100644 --- a/plugins/catalog-common/api-report.md +++ b/plugins/catalog-common/api-report.md @@ -3,29 +3,46 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { IndexableDocument } from '@backstage/search-common'; import { Permission } from '@backstage/plugin-permission-common'; -// @public +// @alpha export const catalogEntityCreatePermission: Permission; -// @public +// @alpha export const catalogEntityDeletePermission: Permission; // @public +export interface CatalogEntityDocument extends IndexableDocument { + // @deprecated (undocumented) + componentType: string; + // (undocumented) + kind: string; + // (undocumented) + lifecycle: string; + // (undocumented) + namespace: string; + // (undocumented) + owner: string; + // (undocumented) + type: string; +} + +// @alpha export const catalogEntityReadPermission: Permission; -// @public +// @alpha export const catalogEntityRefreshPermission: Permission; -// @public +// @alpha export const catalogLocationCreatePermission: Permission; -// @public +// @alpha export const catalogLocationDeletePermission: Permission; -// @public +// @alpha export const catalogLocationReadPermission: Permission; -// @public (undocumented) +// @alpha export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; ``` diff --git a/plugins/catalog-common/package.json b/plugins/catalog-common/package.json index 9a52d22798..bc0d3750ba 100644 --- a/plugins/catalog-common/package.json +++ b/plugins/catalog-common/package.json @@ -10,7 +10,8 @@ "access": "public", "main": "dist/index.cjs.js", "module": "dist/index.esm.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "common-library" @@ -25,7 +26,7 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -33,12 +34,14 @@ "clean": "backstage-cli package clean" }, "dependencies": { - "@backstage/plugin-permission-common": "^0.5.1" + "@backstage/plugin-permission-common": "^0.5.1", + "@backstage/search-common": "^0.2.4" }, "devDependencies": { "@backstage/cli": "^0.14.0" }, "files": [ - "dist" + "dist", + "alpha" ] } diff --git a/plugins/catalog-common/src/index.ts b/plugins/catalog-common/src/index.ts index 9eddf37452..08184eb871 100644 --- a/plugins/catalog-common/src/index.ts +++ b/plugins/catalog-common/src/index.ts @@ -31,3 +31,5 @@ export { catalogLocationCreatePermission, catalogLocationDeletePermission, } from './permissions'; + +export * from './search'; diff --git a/plugins/catalog-common/src/permissions.ts b/plugins/catalog-common/src/permissions.ts index 52c225e441..31a8f32118 100644 --- a/plugins/catalog-common/src/permissions.ts +++ b/plugins/catalog-common/src/permissions.ts @@ -17,8 +17,10 @@ import { Permission } from '@backstage/plugin-permission-common'; /** + * Permission resource type which corresponds to catalog entities. + * * {@link https://backstage.io/docs/features/software-catalog/software-catalog-overview} - * @public + * @alpha */ export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; @@ -28,7 +30,7 @@ export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity'; * * If this permission is not authorized, it will appear that the entity does not * exist in the catalog — both in the frontend and in API responses. - * @public + * @alpha */ export const catalogEntityReadPermission: Permission = { name: 'catalog.entity.read', @@ -42,20 +44,19 @@ export const catalogEntityReadPermission: Permission = { * This permission is used to authorize actions that involve creating a new * catalog entity. This includes registering an existing component into the * catalog. - * @public + * @alpha */ export const catalogEntityCreatePermission: Permission = { name: 'catalog.entity.create', attributes: { action: 'create', }, - resourceType: RESOURCE_TYPE_CATALOG_ENTITY, }; /** * This permission is used to designate actions that involve removing one or * more entities from the catalog. - * @public + * @alpha */ export const catalogEntityDeletePermission: Permission = { name: 'catalog.entity.delete', @@ -68,7 +69,7 @@ export const catalogEntityDeletePermission: Permission = { /** * This permission is used to designate refreshing one or more entities from the * catalog. - * @public + * @alpha */ export const catalogEntityRefreshPermission: Permission = { name: 'catalog.entity.refresh', @@ -84,7 +85,7 @@ export const catalogEntityRefreshPermission: Permission = { * * If this permission is not authorized, it will appear that the location does * not exist in the catalog — both in the frontend and in API responses. - * @public + * @alpha */ export const catalogLocationReadPermission: Permission = { name: 'catalog.location.read', @@ -96,7 +97,7 @@ export const catalogLocationReadPermission: Permission = { /** * This permission is used to designate actions that involve creating catalog * locations. - * @public + * @alpha */ export const catalogLocationCreatePermission: Permission = { name: 'catalog.location.create', @@ -108,7 +109,7 @@ export const catalogLocationCreatePermission: Permission = { /** * This permission is used to designate actions that involve deleting locations * from the catalog. - * @public + * @alpha */ export const catalogLocationDeletePermission: Permission = { name: 'catalog.location.delete', diff --git a/plugins/catalog-common/src/search/CatalogEntityDocument.ts b/plugins/catalog-common/src/search/CatalogEntityDocument.ts new file mode 100644 index 0000000000..834dde2568 --- /dev/null +++ b/plugins/catalog-common/src/search/CatalogEntityDocument.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { IndexableDocument } from '@backstage/search-common'; + +/** + * The Document format for an Entity in the Catalog for search + * + * @public + */ +export interface CatalogEntityDocument extends IndexableDocument { + /** @deprecated `componentType` is being renamed to `type`. During the transition both of these fields should be set to the same value, in order to avoid issues with indexing. */ + componentType: string; + type: string; + namespace: string; + kind: string; + lifecycle: string; + owner: string; +} diff --git a/plugins/catalog-common/src/search/index.ts b/plugins/catalog-common/src/search/index.ts new file mode 100644 index 0000000000..82e713aba0 --- /dev/null +++ b/plugins/catalog-common/src/search/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export type { CatalogEntityDocument } from './CatalogEntityDocument'; diff --git a/plugins/catalog-graph/api-report.md b/plugins/catalog-graph/api-report.md index 515a578d3c..82db1dac3b 100644 --- a/plugins/catalog-graph/api-report.md +++ b/plugins/catalog-graph/api-report.md @@ -6,8 +6,8 @@ /// import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { DependencyGraphTypes } from '@backstage/core-components'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { MouseEvent as MouseEvent_2 } from 'react'; @@ -129,7 +129,7 @@ export const EntityRelationsGraph: ({ className, zoom, }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number | undefined; unidirectional?: boolean | undefined; mergeRelations?: boolean | undefined; diff --git a/plugins/catalog-graph/dev/index.tsx b/plugins/catalog-graph/dev/index.tsx index 62d5eccef3..7d772702a1 100644 --- a/plugins/catalog-graph/dev/index.tsx +++ b/plugins/catalog-graph/dev/index.tsx @@ -16,7 +16,7 @@ import { GetEntitiesResponse } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, RELATION_API_CONSUMED_BY, RELATION_API_PROVIDED_BY, @@ -139,8 +139,12 @@ createDevApp() deps: {}, factory() { return { - async getEntityByName(name: EntityName): Promise { - return entities[stringifyEntityRef(name)]; + async getEntityByRef( + ref: string | CompoundEntityRef, + ): Promise { + return entities[ + typeof ref === 'string' ? ref : stringifyEntityRef(ref) + ]; }, async getEntities(): Promise { return { items: Object.values(entities) }; diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx index 7cebdee692..858fca1a83 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.test.tsx @@ -57,7 +57,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByRef: jest.fn(async _ => ({ ...entity, relations: [] })), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -88,7 +89,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders with custom title', async () => { diff --git a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx index b2dfd9f9b2..4a21418107 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphCard/CatalogGraphCard.tsx @@ -14,14 +14,14 @@ * limitations under the License. */ import { - getEntityName, + getCompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { InfoCard, InfoCardVariants } from '@backstage/core-components'; import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { - formatEntityRefTitle, + humanizeEntityRef, useEntity, entityRouteRef, } from '@backstage/plugin-catalog-react'; @@ -77,7 +77,7 @@ export const CatalogGraphCard = ({ zoom?: 'enabled' | 'disabled' | 'enable-on-click'; }) => { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const catalogEntityRoute = useRouteRef(entityRouteRef); const catalogGraphRoute = useRouteRef(catalogGraphRouteRef); const navigate = useNavigate(); @@ -94,7 +94,7 @@ export const CatalogGraphCard = ({ }); analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx index 2f686f8f62..0bf6469123 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.test.tsx @@ -88,7 +88,10 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => (n.name === 'e' ? entityE : entityC)), + getEntityByRef: jest.fn(async (n: any) => + n === 'b:d/e' ? entityE : entityC, + ), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -128,7 +131,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findByText('b:d/e')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('should toggle filters', async () => { diff --git a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx index 2ae7a99dd8..15d0cd139b 100644 --- a/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx +++ b/plugins/catalog-graph/src/components/CatalogGraphPage/CatalogGraphPage.tsx @@ -24,7 +24,7 @@ import { import { useAnalytics, useRouteRef } from '@backstage/core-plugin-api'; import { entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Grid, makeStyles, Paper, Typography } from '@material-ui/core'; import FilterListIcon from '@material-ui/icons/FilterList'; @@ -149,14 +149,14 @@ export const CatalogGraphPage = ({ analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), { attributes: { to: path } }, ); navigate(path); } else { analytics.captureEvent( 'click', - node.title ?? formatEntityRefTitle(nodeEntityName), + node.title ?? humanizeEntityRef(nodeEntityName), ); setRootEntityNames([nodeEntityName]); } @@ -168,7 +168,7 @@ export const CatalogGraphPage = ({

formatEntityRefTitle(e)).join(', ')} + subtitle={rootEntityNames.map(e => humanizeEntityRef(e)).join(', ')} /> >; + rootEntityNames: CompoundEntityRef[]; + setRootEntityNames: Dispatch>; maxDepth: number; setMaxDepth: Dispatch>; selectedRelations: string[] | undefined; @@ -82,11 +82,12 @@ export function useCatalogGraphPage({ ); // Initial state - const [rootEntityNames, setRootEntityNames] = useState(() => - (Array.isArray(query.rootEntityRefs) - ? query.rootEntityRefs - : initialState?.rootEntityRefs ?? [] - ).map(r => parseEntityRef(r)), + const [rootEntityNames, setRootEntityNames] = useState( + () => + (Array.isArray(query.rootEntityRefs) + ? query.rootEntityRefs + : initialState?.rootEntityRefs ?? [] + ).map(r => parseEntityRef(r)), ); const [maxDepth, setMaxDepth] = useState(() => typeof query.maxDepth === 'string' diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx index 9cd752a5d7..ae541fa997 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/CustomNode.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { DependencyGraphTypes } from '@backstage/core-components'; -import { formatEntityRefTitle } from '@backstage/plugin-catalog-react'; +import { humanizeEntityRef } from '@backstage/plugin-catalog-react'; import { BackstageTheme } from '@backstage/theme'; import { makeStyles } from '@material-ui/core/styles'; import classNames from 'classnames'; @@ -95,7 +95,7 @@ export function CustomNode({ const displayTitle = title ?? (kind && name && namespace - ? formatEntityRefTitle({ kind, name, namespace }) + ? humanizeEntityRef({ kind, name, namespace }) : id); return ( diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx index 36dc816444..58ff81a91f 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.test.tsx @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Entity, RELATION_HAS_PART, RELATION_OWNED_BY, RELATION_OWNER_OF, RELATION_PART_OF, - stringifyEntityRef, } from '@backstage/catalog-model'; import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; @@ -155,7 +155,8 @@ describe('', () => { }; catalog = { getEntities: jest.fn(), - getEntityByName: jest.fn(async n => entities[stringifyEntityRef(n)]), + getEntityByRef: jest.fn(async n => entities[n as string]), + getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), getLocationByRef: jest.fn(), @@ -178,7 +179,7 @@ describe('', () => { }); test('renders a single node without exploding', async () => { - catalog.getEntityByName.mockResolvedValue({ + catalog.getEntityByRef.mockResolvedValue({ apiVersion: 'a', kind: 'b', metadata: { @@ -198,11 +199,11 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('renders a progress indicator while loading', async () => { - catalog.getEntityByName.mockImplementation(() => new Promise(() => {})); + catalog.getEntityByRef.mockImplementation(() => new Promise(() => {})); const { findByRole } = await renderInTestApp( @@ -213,12 +214,12 @@ describe('', () => { ); expect(await findByRole('progressbar')).toBeInTheDocument(); - expect(catalog.getEntityByName).toBeCalledTimes(1); + expect(catalog.getEntityByRef).toBeCalledTimes(1); }); test('does not explode if an entity is missing', async () => { - catalog.getEntityByName.mockImplementation(async n => { - if (n.name === 'c') { + catalog.getEntityByRef.mockImplementation(async (n: any) => { + if (n === 'b:d/c') { return { apiVersion: 'a', kind: 'b', @@ -253,7 +254,7 @@ describe('', () => { expect(await findByText('b:d/c')).toBeInTheDocument(); expect(await findAllByTestId('node')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('renders at max depth of one', async () => { @@ -276,7 +277,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(2); - expect(catalog.getEntityByName).toBeCalledTimes(3); + expect(catalog.getEntityByRef).toBeCalledTimes(3); }); test('renders simplied graph at full depth', async () => { @@ -301,7 +302,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth', async () => { @@ -328,7 +329,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(8); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders full graph at full depth with merged relations', async () => { @@ -353,7 +354,7 @@ describe('', () => { expect(await findAllByText('hasPart')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(4); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with multiple root nodes', async () => { @@ -379,7 +380,7 @@ describe('', () => { expect(await findAllByText('partOf')).toHaveLength(2); expect(await findAllByTestId('label')).toHaveLength(3); - expect(catalog.getEntityByName).toBeCalledTimes(4); + expect(catalog.getEntityByRef).toBeCalledTimes(4); }); test('renders a graph with filtered kinds and relations', async () => { @@ -401,7 +402,7 @@ describe('', () => { expect(await findAllByText('ownerOf')).toHaveLength(1); expect(await findAllByTestId('label')).toHaveLength(1); - expect(catalog.getEntityByName).toBeCalledTimes(2); + expect(catalog.getEntityByRef).toBeCalledTimes(2); }); test('handle clicks on a node', async () => { diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx index 76cd44394a..c5c83a54cd 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/EntityRelationsGraph.tsx @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { DependencyGraph, DependencyGraphTypes, @@ -77,7 +80,7 @@ export const EntityRelationsGraph = ({ className, zoom = 'enabled', }: { - rootEntityNames: EntityName | EntityName[]; + rootEntityNames: CompoundEntityRef | CompoundEntityRef[]; maxDepth?: number; unidirectional?: boolean; mergeRelations?: boolean; diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts index a5dc82eab9..5349b64a72 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.test.ts @@ -29,6 +29,7 @@ describe('useEntityStore', () => { beforeEach(() => { catalogApi = { getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), removeEntityByUid: jest.fn(), getLocationById: jest.fn(), @@ -64,7 +65,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity); + catalogApi.getEntityByRef.mockResolvedValue(entity); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -84,7 +85,7 @@ describe('useEntityStore', () => { test('handles request failures', async () => { const err = new Error('Hello World'); - catalogApi.getEntityByName.mockRejectedValue(err); + catalogApi.getEntityByRef.mockRejectedValue(err); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -101,7 +102,7 @@ describe('useEntityStore', () => { }); test('handles loading', async () => { - catalogApi.getEntityByName.mockReturnValue(new Promise(() => {})); + catalogApi.getEntityByRef.mockReturnValue(new Promise(() => {})); const { result } = renderHook(() => useEntityStore()); @@ -133,7 +134,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -150,7 +151,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities([ @@ -188,7 +189,7 @@ describe('useEntityStore', () => { }, }; - catalogApi.getEntityByName.mockResolvedValue(entity1); + catalogApi.getEntityByRef.mockResolvedValue(entity1); const { result, waitFor } = renderHook(() => useEntityStore()); @@ -205,7 +206,7 @@ describe('useEntityStore', () => { }); }); - catalogApi.getEntityByName.mockResolvedValue(entity2); + catalogApi.getEntityByRef.mockResolvedValue(entity2); act(() => { result.current.requestEntities(['kind:namespace/name2']); @@ -233,6 +234,6 @@ describe('useEntityStore', () => { }); }); - expect(catalogApi.getEntityByName).toBeCalledTimes(2); + expect(catalogApi.getEntityByRef).toBeCalledTimes(2); }); }); diff --git a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts index b5ec3bef19..843b308281 100644 --- a/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts +++ b/plugins/catalog-graph/src/components/EntityRelationsGraph/useEntityStore.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, parseEntityRef } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import limiterFactory from 'p-limit'; @@ -73,9 +73,7 @@ export function useEntityStore(): { return; } - const promise = catalogClient.getEntityByName( - parseEntityRef(entityRef), - ); + const promise = catalogClient.getEntityByRef(entityRef); outstandingEntities.set(entityRef, promise); diff --git a/plugins/catalog-import/api-report.md b/plugins/catalog-import/api-report.md index 164d831c96..5a9e8f03fd 100644 --- a/plugins/catalog-import/api-report.md +++ b/plugins/catalog-import/api-report.md @@ -8,11 +8,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi } from '@backstage/core-plugin-api'; import { Controller } from 'react-hook-form'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FieldErrors } from 'react-hook-form'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; @@ -33,7 +33,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -165,7 +165,7 @@ export interface EntityListComponentProps { // (undocumented) locations: Array<{ target: string; - entities: (Entity | EntityName)[]; + entities: (Entity | CompoundEntityRef)[]; }>; // (undocumented) onItemClick?: (target: string) => void; @@ -246,7 +246,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -258,7 +258,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-import/dev/index.tsx b/plugins/catalog-import/dev/index.tsx index a0cf034782..e2e5e1d22c 100644 --- a/plugins/catalog-import/dev/index.tsx +++ b/plugins/catalog-import/dev/index.tsx @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { createDevApp } from '@backstage/dev-utils'; import { catalogApiRef } from '@backstage/plugin-catalog-react'; import { Grid, ListItem, ListItemIcon, ListItemText } from '@material-ui/core'; @@ -32,7 +32,7 @@ import { import { ImportPage } from '../src/components/ImportPage'; import { Content, Header, InfoCard, Page } from '@backstage/core-components'; -const getEntityNames = (url: string): EntityName[] => [ +const getEntityNames = (url: string): CompoundEntityRef[] => [ { kind: 'Component', namespace: url.replace(/^.*(folder-[^/]+).*|.*()$/, '$1') || 'default', diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 0923f4ac98..3cfd53470a 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { createApiRef } from '@backstage/core-plugin-api'; import { PartialEntity } from '../types'; @@ -38,7 +38,7 @@ export type AnalyzeResult = locations: Array<{ target: string; exists?: boolean; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts index f689c0a883..969ff25c00 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.test.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -93,6 +93,7 @@ describe('CatalogImportClient', () => { getEntities: jest.fn(), addLocation: jest.fn(), removeLocationById: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index b856a4e359..c12e391ec9 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigApi, DiscoveryApi, @@ -212,7 +212,7 @@ the component will become available.\n\nFor more information, read an \ }): Promise< Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }> > { const { url, owner, repo, githubIntegrationConfig } = options; diff --git a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx index 2c59eec18f..47f0139e5e 100644 --- a/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx +++ b/plugins/catalog-import/src/components/EntityListComponent/EntityListComponent.tsx @@ -14,11 +14,11 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useApp } from '@backstage/core-plugin-api'; import { EntityRefLink, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Collapse, @@ -41,9 +41,9 @@ const useStyles = makeStyles(theme => ({ }, })); -function sortEntities(entities: Array) { +function sortEntities(entities: Array) { return entities.sort((a, b) => - formatEntityRefTitle(a).localeCompare(formatEntityRefTitle(b)), + humanizeEntityRef(a).localeCompare(humanizeEntityRef(b)), ); } @@ -53,7 +53,10 @@ function sortEntities(entities: Array) { * @public */ export interface EntityListComponentProps { - locations: Array<{ target: string; entities: (Entity | EntityName)[] }>; + locations: Array<{ + target: string; + entities: (Entity | CompoundEntityRef)[]; + }>; locationListItemIcon: (target: string) => React.ReactElement; collapsed?: boolean; firstListItem?: React.ReactElement; @@ -130,7 +133,7 @@ export const EntityListComponent = (props: EntityListComponentProps) => { ) ?? WorkIcon; return ( { - + ); })} diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx index fc9b35136a..dd8087c51e 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.test.tsx @@ -38,6 +38,7 @@ describe('', () => { const catalogApi: jest.Mocked = { getEntities: jest.fn(), addLocation: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx index 36e8c33a5c..1cbdfce355 100644 --- a/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx +++ b/plugins/catalog-import/src/components/StepPrepareCreatePullRequest/StepPrepareCreatePullRequest.tsx @@ -19,7 +19,7 @@ import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { assertError } from '@backstage/errors'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { Box, FormHelperText, Grid, Typography } from '@material-ui/core'; import { makeStyles } from '@material-ui/core/styles'; @@ -139,7 +139,7 @@ export const StepPrepareCreatePullRequest = ( }); return groupEntities.items - .map(e => formatEntityRefTitle(e, { defaultKind: 'group' })) + .map(e => humanizeEntityRef(e, { defaultKind: 'group' })) .sort(); }); diff --git a/plugins/catalog-import/src/components/useImportState.test.tsx b/plugins/catalog-import/src/components/useImportState.test.tsx index 1a01c61449..155adb8290 100644 --- a/plugins/catalog-import/src/components/useImportState.test.tsx +++ b/plugins/catalog-import/src/components/useImportState.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { cleanup } from '@testing-library/react'; import { act, renderHook } from '@testing-library/react-hooks'; import { AnalyzeResult } from '../api'; @@ -37,7 +37,7 @@ describe('useImportState', () => { locations: [ { target: 'https://0', - entities: [] as EntityName[], + entities: [] as CompoundEntityRef[], }, ], }; diff --git a/plugins/catalog-import/src/components/useImportState.ts b/plugins/catalog-import/src/components/useImportState.ts index 0cb6b59917..965f855f30 100644 --- a/plugins/catalog-import/src/components/useImportState.ts +++ b/plugins/catalog-import/src/components/useImportState.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { useReducer } from 'react'; import { AnalyzeResult } from '../api'; @@ -43,7 +43,7 @@ export type PrepareResult = locations: Array<{ exists?: boolean; target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; } | { @@ -55,7 +55,7 @@ export type PrepareResult = }; locations: Array<{ target: string; - entities: EntityName[]; + entities: CompoundEntityRef[]; }>; }; diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index e78ec75daa..ca5de26048 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -11,8 +11,8 @@ import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { CatalogApi } from '@backstage/catalog-client'; import { ComponentEntity } from '@backstage/catalog-model'; import { ComponentProps } from 'react'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { GetEntitiesResponse } from '@backstage/catalog-client'; import { IconButton } from '@material-ui/core'; import { LinkProps } from '@backstage/core-components'; @@ -24,7 +24,6 @@ import { default as React_2 } from 'react'; import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; -import { StorageApi } from '@backstage/core-plugin-api'; import { StyleRules } from '@material-ui/core/styles/withStyles'; import { SystemEntity } from '@backstage/catalog-model'; import { TableColumn } from '@backstage/core-components'; @@ -96,9 +95,6 @@ export type CatalogReactUserListPickerClassKey = | 'menuItem' | 'groupWrapper'; -// @public @deprecated (undocumented) -export const catalogRouteRef: RouteRef; - // @public (undocumented) export const columnFactories: Readonly<{ createEntityRefColumn(options: { @@ -138,17 +134,6 @@ export type DefaultEntityFilters = { text?: EntityTextFilter; }; -// @public -export class DefaultStarredEntitiesApi implements StarredEntitiesApi { - constructor(opts: { storageApi: StorageApi }); - // (undocumented) - isStarred(entityRef: string): boolean; - // (undocumented) - starredEntitie$(): Observable>; - // (undocumented) - toggleStarred(entityRef: string): Promise; -} - // @public (undocumented) export type EntityFilter = { getCatalogFilters?: () => Record< @@ -225,8 +210,8 @@ export const EntityListProvider: ({ }: PropsWithChildren<{}>) => JSX.Element; // @public (undocumented) -export type EntityLoadingStatus = { - entity?: Entity; +export type EntityLoadingStatus = { + entity?: TEntity; loading: boolean; error?: Error; refresh?: VoidFunction; @@ -265,7 +250,7 @@ export const EntityRefLink: (props: EntityRefLinkProps) => JSX.Element; // @public export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React_2.ReactNode; @@ -280,17 +265,10 @@ export const EntityRefLinks: ({ // @public export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; -// @public @deprecated (undocumented) -export const entityRoute: RouteRef<{ - name: string; - kind: string; - namespace: string; -}>; - // @public export function entityRouteParams(entity: Entity): { readonly kind: string; @@ -422,7 +400,7 @@ export type EntityTypeReturn = { // @public export const FavoriteEntity: (props: FavoriteEntityProps) => JSX.Element; -// @public (undocumented) +// @public @deprecated (undocumented) export const favoriteEntityIcon: (isStarred: boolean) => JSX.Element; // @public (undocumented) @@ -430,18 +408,13 @@ export type FavoriteEntityProps = ComponentProps & { entity: Entity; }; -// @public (undocumented) +// @public @deprecated (undocumented) export const favoriteEntityTooltip: ( isStarred: boolean, ) => 'Remove from favorites' | 'Add to favorites'; -// @public (undocumented) -export function formatEntityRefTitle( - entityRef: Entity | EntityName, - opts?: { - defaultKind?: string; - }, -): string; +// @public @deprecated (undocumented) +export const formatEntityRefTitle: typeof humanizeEntityRef; // @public @deprecated (undocumented) export function getEntityMetadataEditUrl(entity: Entity): string | undefined; @@ -456,7 +429,7 @@ export function getEntityRelations( filter?: { kind: string; }, -): EntityName[]; +): CompoundEntityRef[]; // @public (undocumented) export function getEntitySourceLocation( @@ -464,6 +437,14 @@ export function getEntitySourceLocation( scmIntegrationsApi: ScmIntegrationRegistry, ): EntitySourceLocation | undefined; +// @public (undocumented) +export function humanizeEntityRef( + entityRef: Entity | CompoundEntityRef, + opts?: { + defaultKind?: string; + }, +): string; + // @public export function InspectEntityDialog(props: { open: boolean; @@ -471,8 +452,8 @@ export function InspectEntityDialog(props: { onClose: () => void; }): JSX.Element | null; -// @public -export function isOwnerOf(owner: Entity, owned: Entity): boolean; +// @alpha +export function isOwnerOf(owner: Entity, entity: Entity): boolean; // @public @deprecated export function loadCatalogOwnerRefs( @@ -488,6 +469,14 @@ export const MockEntityListContextProvider: ({ value?: Partial> | undefined; }>) => JSX.Element; +// @public +export class MockStarredEntitiesApi implements StarredEntitiesApi { + // (undocumented) + starredEntitie$(): Observable>; + // (undocumented) + toggleStarred(entityRef: string): Promise; +} + // @public @deprecated (undocumented) export function reduceCatalogFilters( filters: EntityFilter[], @@ -498,9 +487,6 @@ export function reduceEntityFilters( filters: EntityFilter[], ): (entity: Entity) => boolean; -// @public @deprecated (undocumented) -export const rootRoute: RouteRef; - // @public export interface StarredEntitiesApi { starredEntitie$(): Observable>; @@ -524,11 +510,16 @@ export type UnregisterEntityDialogProps = { }; // @public -export function useEntity(): { - entity: T; +export function useAsyncEntity< + TEntity extends Entity = Entity, +>(): EntityLoadingStatus; + +// @public +export function useEntity(): { + entity: TEntity; loading: boolean; - error: Error | undefined; - refresh: VoidFunction | undefined; + error?: Error; + refresh?: VoidFunction; }; // @public @deprecated @@ -561,10 +552,10 @@ export function useEntityListProvider< // @public export function useEntityOwnership(): { loading: boolean; - isOwnedEntity: (entity: Entity | EntityName) => boolean; + isOwnedEntity: (entity: Entity) => boolean; }; -// @public +// @alpha export function useEntityPermission(permission: Permission): { loading: boolean; allowed: boolean; @@ -636,12 +627,18 @@ export type UserListPickerProps = { // @public (undocumented) export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; }; // @public (undocumented) -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; }; diff --git a/plugins/catalog-react/package.json b/plugins/catalog-react/package.json index c7e255f973..119c3f19a4 100644 --- a/plugins/catalog-react/package.json +++ b/plugins/catalog-react/package.json @@ -9,7 +9,8 @@ "publishConfig": { "access": "public", "main": "dist/index.esm.js", - "types": "dist/index.d.ts" + "types": "dist/index.d.ts", + "alphaTypes": "dist/index.alpha.d.ts" }, "backstage": { "role": "web-library" @@ -24,7 +25,7 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "prepack": "backstage-cli package prepack", @@ -76,6 +77,7 @@ "react-test-renderer": "^16.13.1" }, "files": [ - "dist" + "dist", + "alpha" ] } diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts deleted file mode 100644 index 55b992fadf..0000000000 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { stringifyEntityRef } from '@backstage/catalog-model'; -import { StorageApi } from '@backstage/core-plugin-api'; -import { MockStorageApi } from '@backstage/test-utils'; -import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; -import { performMigrationToTheNewBucket } from './migration'; - -jest.mock('./migration'); - -describe('DefaultStarredEntitiesApi', () => { - let mockStorage: StorageApi; - let starredEntitiesApi: DefaultStarredEntitiesApi; - - const mockEntityRef = stringifyEntityRef({ - apiVersion: '1', - kind: 'Component', - metadata: { - name: 'mock', - }, - }); - - beforeEach(() => { - (performMigrationToTheNewBucket as jest.Mock).mockResolvedValue(undefined); - - mockStorage = MockStorageApi.create(); - starredEntitiesApi = new DefaultStarredEntitiesApi({ - storageApi: mockStorage, - }); - }); - - afterEach(() => { - jest.resetAllMocks(); - }); - - describe('constructor', () => { - it('should call migration', () => { - expect(performMigrationToTheNewBucket).toBeCalledTimes(1); - }); - }); - - describe('toggleStarred', () => { - it('should star unstarred entity', async () => { - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); - - await starredEntitiesApi.toggleStarred(mockEntityRef); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); - }); - - it('should unstar starred entity', async () => { - const bucket = mockStorage.forBucket('starredEntities'); - await bucket.set('entityRefs', ['component:default/mock']); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(true); - - await starredEntitiesApi.toggleStarred(mockEntityRef); - - expect(starredEntitiesApi.isStarred(mockEntityRef)).toBe(false); - }); - }); - - describe('starredEntities$', () => { - const handler = jest.fn(); - - beforeEach(async () => { - await new Promise(resolve => { - starredEntitiesApi.starredEntitie$().subscribe({ - next: (...args) => { - handler(...args); - - if (handler.mock.calls.length >= 2) { - resolve(); - } - }, - }); - - const bucket = mockStorage.forBucket('starredEntities'); - bucket.set('entityRefs', ['component:default/mock']).then(); - }); - }); - - it('should receive updates', async () => { - expect(handler).toBeCalledTimes(2); - expect(handler).toBeCalledWith(new Set()); - expect(handler).toBeCalledWith(new Set(['component:default/mock'])); - }); - }); -}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts new file mode 100644 index 0000000000..aadecefe98 --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MockStarredEntitiesApi } from './MockStarredEntitiesApi'; + +describe('MockStarredEntitiesApi', () => { + it('should toggle starred entities', async () => { + const api = new MockStarredEntitiesApi(); + + const updates1 = new Array>(); + const sub1 = api + .starredEntitie$() + .subscribe(entities => updates1.push(entities)); + + api.toggleStarred('k:ns/e1'); + api.toggleStarred('k:ns/e2'); + + await Promise.resolve(); + expect(updates1).toEqual([ + new Set(), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + ]); + + const updates2 = new Array>(); + const sub2 = api + .starredEntitie$() + .subscribe(entities => updates2.push(entities)); + + api.toggleStarred('k:ns/e2'); + sub1.unsubscribe(); + api.toggleStarred('k:ns/e2'); + + await Promise.resolve(); + expect(updates1).toEqual([ + new Set(), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + new Set(['k:ns/e1']), + ]); + expect(updates2).toEqual([ + new Set(['k:ns/e1', 'k:ns/e2']), + new Set(['k:ns/e1']), + new Set(['k:ns/e1', 'k:ns/e2']), + ]); + sub2.unsubscribe(); + }); +}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts new file mode 100644 index 0000000000..9467153dd9 --- /dev/null +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/MockStarredEntitiesApi.ts @@ -0,0 +1,54 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Observable } from '@backstage/types'; +import ObservableImpl from 'zen-observable'; +import { StarredEntitiesApi } from './StarredEntitiesApi'; + +/** + * An in-memory mock implementation of the StarredEntitiesApi. + * + * @public + */ +export class MockStarredEntitiesApi implements StarredEntitiesApi { + private readonly starredEntities = new Set(); + private readonly subscribers = new Set< + ZenObservable.SubscriptionObserver> + >(); + + private readonly observable = new ObservableImpl>(subscriber => { + subscriber.next(new Set(this.starredEntities)); + + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + }); + + async toggleStarred(entityRef: string): Promise { + if (!this.starredEntities.delete(entityRef)) { + this.starredEntities.add(entityRef); + } + + for (const subscription of this.subscribers) { + subscription.next(new Set(this.starredEntities)); + } + } + + starredEntitie$(): Observable> { + return this.observable; + } +} diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts index e9f9c8923a..ba44c38e1d 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts +++ b/plugins/catalog-react/src/apis/StarredEntitiesApi/index.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; export { starredEntitiesApiRef } from './StarredEntitiesApi'; export type { StarredEntitiesApi } from './StarredEntitiesApi'; +export { MockStarredEntitiesApi } from './MockStarredEntitiesApi'; diff --git a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx index d025d08012..ec2c8126e5 100644 --- a/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx +++ b/plugins/catalog-react/src/components/EntityOwnerPicker/EntityOwnerPicker.tsx @@ -31,7 +31,7 @@ import React, { useEffect, useMemo, useState } from 'react'; import { useEntityList } from '../../hooks/useEntityListProvider'; import { EntityOwnerFilter } from '../../filters'; import { getEntityRelations } from '../../utils'; -import { formatEntityRefTitle } from '../EntityRefLink'; +import { humanizeEntityRef } from '../EntityRefLink'; /** @public */ export type CatalogReactEntityOwnerPickerClassKey = 'input'; @@ -86,7 +86,7 @@ export const EntityOwnerPicker = () => { backendEntities .flatMap((e: Entity) => getEntityRelations(e, RELATION_OWNED_BY).map(o => - formatEntityRefTitle(o, { defaultKind: 'group' }), + humanizeEntityRef(o, { defaultKind: 'group' }), ), ) .filter(Boolean) as string[], diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx index 1e01402ddf..06a632232e 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLink.tsx @@ -16,13 +16,13 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, parseEntityRef, } from '@backstage/catalog-model'; import React, { forwardRef } from 'react'; import { entityRouteRef } from '../../routes'; -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './humanize'; import { Link, LinkProps } from '@backstage/core-components'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Tooltip } from '@material-ui/core'; @@ -33,7 +33,7 @@ import { Tooltip } from '@material-ui/core'; * @public */ export type EntityRefLinkProps = { - entityRef: Entity | EntityName | string; + entityRef: Entity | CompoundEntityRef | string; defaultKind?: string; title?: string; children?: React.ReactNode; @@ -72,7 +72,7 @@ export const EntityRefLink = forwardRef( namespace = namespace?.toLocaleLowerCase('en-US') ?? DEFAULT_NAMESPACE; const routeParams = { kind, namespace, name }; - const formattedEntityRefTitle = formatEntityRefTitle( + const formattedEntityRefTitle = humanizeEntityRef( { kind, namespace, name }, { defaultKind }, ); diff --git a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx index 9be970aa56..d92cd2ec87 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx +++ b/plugins/catalog-react/src/components/EntityRefLink/EntityRefLinks.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import React from 'react'; import { EntityRefLink } from './EntityRefLink'; import { LinkProps } from '@backstage/core-components'; @@ -25,7 +25,7 @@ import { LinkProps } from '@backstage/core-components'; * @public */ export type EntityRefLinksProps = { - entityRefs: (Entity | EntityName)[]; + entityRefs: (Entity | CompoundEntityRef)[]; defaultKind?: string; } & Omit; diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts similarity index 86% rename from plugins/catalog-react/src/components/EntityRefLink/format.test.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts index b489f088fd..b541946ece 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.test.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { formatEntityRefTitle } from './format'; +import { humanizeEntityRef } from './humanize'; describe('formatEntityRefTitle', () => { it('formats entity in default namespace', () => { @@ -30,7 +30,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:software'); }); @@ -48,7 +48,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity); + const title = humanizeEntityRef(entity); expect(title).toEqual('component:test/software'); }); @@ -66,7 +66,7 @@ describe('formatEntityRefTitle', () => { lifecycle: 'production', }, }; - const title = formatEntityRefTitle(entity, { defaultKind: 'Component' }); + const title = humanizeEntityRef(entity, { defaultKind: 'Component' }); expect(title).toEqual('test/software'); }); @@ -76,7 +76,7 @@ describe('formatEntityRefTitle', () => { namespace: 'default', name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:software'); }); @@ -87,7 +87,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName); + const title = humanizeEntityRef(entityName); expect(title).toEqual('component:test/software'); }); @@ -98,7 +98,7 @@ describe('formatEntityRefTitle', () => { name: 'software', }; - const title = formatEntityRefTitle(entityName, { + const title = humanizeEntityRef(entityName, { defaultKind: 'component', }); expect(title).toEqual('test/software'); diff --git a/plugins/catalog-react/src/components/EntityRefLink/format.ts b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts similarity index 85% rename from plugins/catalog-react/src/components/EntityRefLink/format.ts rename to plugins/catalog-react/src/components/EntityRefLink/humanize.ts index 90100245a1..0ade1da0b9 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/format.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/humanize.ts @@ -16,13 +16,16 @@ import { Entity, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, } from '@backstage/catalog-model'; +/** @public @deprecated please use {@link humanizeEntityRef} instead */ +export const formatEntityRefTitle = humanizeEntityRef; + /** @public */ -export function formatEntityRefTitle( - entityRef: Entity | EntityName, +export function humanizeEntityRef( + entityRef: Entity | CompoundEntityRef, opts?: { defaultKind?: string }, ) { const defaultKind = opts?.defaultKind; diff --git a/plugins/catalog-react/src/components/EntityRefLink/index.ts b/plugins/catalog-react/src/components/EntityRefLink/index.ts index d49993feb6..50394547a0 100644 --- a/plugins/catalog-react/src/components/EntityRefLink/index.ts +++ b/plugins/catalog-react/src/components/EntityRefLink/index.ts @@ -18,4 +18,4 @@ export { EntityRefLink } from './EntityRefLink'; export type { EntityRefLinkProps } from './EntityRefLink'; export { EntityRefLinks } from './EntityRefLinks'; export type { EntityRefLinksProps } from './EntityRefLinks'; -export { formatEntityRefTitle } from './format'; +export { humanizeEntityRef, formatEntityRefTitle } from './humanize'; diff --git a/plugins/catalog-react/src/components/EntityTable/columns.tsx b/plugins/catalog-react/src/components/EntityTable/columns.tsx index d6c3582af6..08eb31bea3 100644 --- a/plugins/catalog-react/src/components/EntityTable/columns.tsx +++ b/plugins/catalog-react/src/components/EntityTable/columns.tsx @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, RELATION_PART_OF, } from '@backstage/catalog-model'; @@ -26,7 +26,7 @@ import { getEntityRelations } from '../../utils'; import { EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, } from '../EntityRefLink'; /** @public */ @@ -38,7 +38,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: T): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind, }) ); @@ -81,13 +81,13 @@ export const columnFactories = Object.freeze({ defaultKind?: string; filter?: { kind: string }; }): TableColumn { - function getRelations(entity: T): EntityName[] { + function getRelations(entity: T): CompoundEntityRef[] { return getEntityRelations(entity, relation, entityFilter); } function formatContent(entity: T): string { return getRelations(entity) - .map(r => formatEntityRefTitle(r, { defaultKind })) + .map(r => humanizeEntityRef(r, { defaultKind })) .join(', '); } diff --git a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx index 3832a82877..eaa2821448 100644 --- a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx +++ b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx @@ -32,11 +32,17 @@ const YellowStar = withStyles({ }, })(Star); -/** @public */ +/** + * @public + * @deprecated due to low utility value. + */ export const favoriteEntityTooltip = (isStarred: boolean) => isStarred ? 'Remove from favorites' : 'Add to favorites'; -/** @public */ +/** + * @public + * @deprecated due to low utility value. + */ export const favoriteEntityIcon = (isStarred: boolean) => isStarred ? : ; @@ -55,8 +61,10 @@ export const FavoriteEntity = (props: FavoriteEntityProps) => { {...props} onClick={() => toggleStarredEntity()} > - - {favoriteEntityIcon(isStarredEntity)} + + {isStarredEntity ? : } ); diff --git a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx index bf6c8971b0..0423c0da44 100644 --- a/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx +++ b/plugins/catalog-react/src/components/InspectEntityDialog/components/AncestryPage.tsx @@ -33,7 +33,7 @@ import React, { useLayoutEffect, useRef, useState } from 'react'; import { useNavigate } from 'react-router'; import useAsync from 'react-use/lib/useAsync'; import { catalogApiRef } from '../../../api'; -import { formatEntityRefTitle } from '../../../components/EntityRefLink/format'; +import { humanizeEntityRef } from '../../EntityRefLink'; import { entityRouteRef } from '../../../routes'; import { EntityKindIcon } from './EntityKindIcon'; @@ -132,7 +132,7 @@ function CustomNode({ node }: DependencyGraphTypes.RenderNodeProps) { const displayTitle = node.metadata.title || (node.kind && node.metadata.name && node.metadata.namespace - ? formatEntityRefTitle({ + ? humanizeEntityRef({ kind: node.kind, name: node.metadata.name, namespace: node.metadata.namespace || '', diff --git a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts index 6420ebd5ae..c14b6523c0 100644 --- a/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts @@ -16,8 +16,8 @@ import { Entity, - EntityName, - getEntityName, + CompoundEntityRef, + getCompoundEntityRef, ANNOTATION_ORIGIN_LOCATION, } from '@backstage/catalog-model'; import { catalogApiRef } from '../../api'; @@ -44,7 +44,7 @@ export type UseUnregisterEntityDialogState = | { type: 'unregister'; location: string; - colocatedEntities: EntityName[]; + colocatedEntities: CompoundEntityRef[]; unregisterLocation: () => Promise; deleteEntity: () => Promise; } @@ -141,7 +141,7 @@ export function useUnregisterEntityDialogState( return { type: 'unregister', location: locationRef!, - colocatedEntities: colocatedEntities.map(getEntityName), + colocatedEntities: colocatedEntities.map(getCompoundEntityRef), unregisterLocation, deleteEntity, }; diff --git a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx index 316a0f2c06..310deabbb7 100644 --- a/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx +++ b/plugins/catalog-react/src/components/UserListPicker/UserListPicker.test.tsx @@ -55,7 +55,7 @@ const mockConfigApi = { } as Partial; const mockCatalogApi = { - getEntityByName: () => Promise.resolve(mockUser), + getEntityByRef: () => Promise.resolve(mockUser), } as Partial; const mockIdentityApi = { diff --git a/plugins/catalog-react/src/filters.ts b/plugins/catalog-react/src/filters.ts index 5d2c2e531b..9fb057a01c 100644 --- a/plugins/catalog-react/src/filters.ts +++ b/plugins/catalog-react/src/filters.ts @@ -15,7 +15,7 @@ */ import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; -import { formatEntityRefTitle } from './components/EntityRefLink'; +import { humanizeEntityRef } from './components/EntityRefLink'; import { EntityFilter, UserListFilterKind } from './types'; import { getEntityRelations } from './utils'; @@ -107,7 +107,7 @@ export class EntityOwnerFilter implements EntityFilter { filterEntity(entity: Entity): boolean { return this.values.some(v => getEntityRelations(entity, RELATION_OWNED_BY).some( - o => formatEntityRefTitle(o, { defaultKind: 'group' }) === v, + o => humanizeEntityRef(o, { defaultKind: 'group' }) === v, ), ); } diff --git a/plugins/catalog-react/src/hooks/index.ts b/plugins/catalog-react/src/hooks/index.ts index 3e6ef7c945..da23d54ad7 100644 --- a/plugins/catalog-react/src/hooks/index.ts +++ b/plugins/catalog-react/src/hooks/index.ts @@ -18,6 +18,7 @@ export { useEntityFromUrl, EntityProvider, AsyncEntityProvider, + useAsyncEntity, } from './useEntity'; export type { EntityLoadingStatus, diff --git a/plugins/catalog-react/src/hooks/useEntity.tsx b/plugins/catalog-react/src/hooks/useEntity.tsx index 5cb67cb79a..f80f858dec 100644 --- a/plugins/catalog-react/src/hooks/useEntity.tsx +++ b/plugins/catalog-react/src/hooks/useEntity.tsx @@ -27,8 +27,8 @@ import { catalogApiRef } from '../api'; import { useEntityCompoundName } from './useEntityCompoundName'; /** @public */ -export type EntityLoadingStatus = { - entity?: Entity; +export type EntityLoadingStatus = { + entity?: TEntity; loading: boolean; error?: Error; refresh?: VoidFunction; @@ -115,7 +115,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); @@ -130,24 +130,25 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { }; /** - * Grab the current entity from the context and its current loading state. + * Grab the current entity from the context, throws if the entity has not yet been loaded + * or is not available. * * @public */ -export function useEntity() { +export function useEntity(): { + entity: TEntity; + /** @deprecated use {@link useAsyncEntity} instead */ + loading: boolean; + /** @deprecated use {@link useAsyncEntity} instead */ + error?: Error; + /** @deprecated use {@link useAsyncEntity} instead */ + refresh?: VoidFunction; +} { const versionedHolder = useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); if (!versionedHolder) { - // TODO(Rugvip): Throw this once we fully migrate to the new context - // throw new Error('Entity context is not available'); - - return { - entity: undefined as unknown as T, - loading: true, - error: undefined, - refresh: () => {}, - }; + throw new Error('Entity context is not available'); } const value = versionedHolder.atVersion(1); @@ -155,6 +156,40 @@ export function useEntity() { throw new Error('EntityContext v1 not available'); } + if (!value.entity) { + // Once we have removed the additional fields from being returned we can drop this deprecation + // and move to the error instead. + // throw new Error('useEntity hook is being called outside of an EntityLayout where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead.'); + + // eslint-disable-next-line no-console + console.warn( + 'DEPRECATION: useEntity hook is being called outside of an EntityLayout where the entity has not been loaded. If this is intentional, please use useAsyncEntity instead. This warning will be replaced with an error in future releases.', + ); + } + const { entity, loading, error, refresh } = value; - return { entity: entity as T, loading, error, refresh }; + return { entity: entity as TEntity, loading, error, refresh }; +} + +/** + * Grab the current entity from the context, provides loading state and errors, and the ability to refresh. + * + * @public + */ +export function useAsyncEntity< + TEntity extends Entity = Entity, +>(): EntityLoadingStatus { + const versionedHolder = + useVersionedContext<{ 1: EntityLoadingStatus }>('entity-context'); + + if (!versionedHolder) { + throw new Error('Entity context is not available'); + } + const value = versionedHolder.atVersion(1); + if (!value) { + throw new Error('EntityContext v1 not available'); + } + + const { entity, loading, error, refresh } = value; + return { entity: entity as TEntity, loading, error, refresh }; } diff --git a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx index c8ca29b07f..e3ab09f081 100644 --- a/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityListProvider.test.tsx @@ -29,7 +29,7 @@ import qs from 'qs'; import React, { PropsWithChildren } from 'react'; import { MemoryRouter } from 'react-router'; import { catalogApiRef } from '../api'; -import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { starredEntitiesApiRef, MockStarredEntitiesApi } from '../apis'; import { EntityKindPicker, UserListPicker } from '../components'; import { EntityKindFilter, EntityTypeFilter, UserListFilter } from '../filters'; import { UserListFilterKind } from '../types'; @@ -76,7 +76,7 @@ const mockIdentityApi: Partial = { }; const mockCatalogApi: Partial = { getEntities: jest.fn().mockImplementation(async () => ({ items: entities })), - getEntityByName: async () => undefined, + getEntityByRef: async () => undefined, }; const wrapper = ({ @@ -95,12 +95,7 @@ const wrapper = ({ [catalogApiRef, mockCatalogApi], [identityApiRef, mockIdentityApi], [storageApiRef, MockStorageApi.create()], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ - storageApi: MockStorageApi.create(), - }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ]} > diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx index 1bb9a649ad..4730f71a7d 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.test.tsx @@ -30,13 +30,13 @@ import { loadCatalogOwnerRefs, useEntityOwnership } from './useEntityOwnership'; describe('useEntityOwnership', () => { type MockIdentityApi = jest.Mocked>; - type MockCatalogApi = jest.Mocked>; + type MockCatalogApi = jest.Mocked>; const mockIdentityApi: MockIdentityApi = { getBackstageIdentity: jest.fn(), }; const mockCatalogApi: MockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), }; const identityApi = mockIdentityApi as unknown as IdentityApi; @@ -102,11 +102,11 @@ describe('useEntityOwnership', () => { describe('loadCatalogOwnerRefs', () => { it('loads the first user from the catalog', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(user2Entity); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(user2Entity); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual(['group:default/group1']); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -114,11 +114,11 @@ describe('useEntityOwnership', () => { }); it('gracefully handles missing user', async () => { - mockCatalogApi.getEntityByName.mockResolvedValueOnce(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValueOnce(undefined); await expect( loadCatalogOwnerRefs(catalogApi, ['user:default/user2']), ).resolves.toEqual([]); - expect(mockCatalogApi.getEntityByName).toBeCalledWith({ + expect(mockCatalogApi.getEntityByRef).toBeCalledWith({ kind: 'user', namespace: 'default', name: 'user2', @@ -133,7 +133,7 @@ describe('useEntityOwnership', () => { userEntityRef: 'user:default/user1', ownershipEntityRefs: ['user:default/user1', 'group:default/group1'], }); - mockCatalogApi.getEntityByName.mockResolvedValue(undefined); + mockCatalogApi.getEntityByRef.mockResolvedValue(undefined); const { result, waitForValueToChange } = renderHook( () => useEntityOwnership(), diff --git a/plugins/catalog-react/src/hooks/useEntityOwnership.ts b/plugins/catalog-react/src/hooks/useEntityOwnership.ts index 77f6c4bf24..55b4e75ace 100644 --- a/plugins/catalog-react/src/hooks/useEntityOwnership.ts +++ b/plugins/catalog-react/src/hooks/useEntityOwnership.ts @@ -17,7 +17,6 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, - EntityName, parseEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, @@ -49,7 +48,7 @@ export async function loadCatalogOwnerRefs( const primaryUserRef = identityOwnerRefs.find(ref => ref.startsWith('user:')); if (primaryUserRef) { - const entity = await catalogApi.getEntityByName( + const entity = await catalogApi.getEntityByRef( parseEntityRef(primaryUserRef), ); if (entity) { @@ -77,7 +76,7 @@ export async function loadCatalogOwnerRefs( */ export function useEntityOwnership(): { loading: boolean; - isOwnedEntity: (entity: Entity | EntityName) => boolean; + isOwnedEntity: (entity: Entity) => boolean; } { const identityApi = useApi(identityApiRef); const catalogApi = useApi(catalogApiRef); @@ -94,12 +93,10 @@ export function useEntityOwnership(): { const isOwnedEntity = useMemo(() => { const myOwnerRefs = new Set(refs ?? []); - return (entity: Entity | EntityName) => { - const entityOwnerRefs = ( - 'metadata' in entity - ? getEntityRelations(entity, RELATION_OWNED_BY) - : [entity] - ).map(stringifyEntityRef); + return (entity: Entity) => { + const entityOwnerRefs = getEntityRelations(entity, RELATION_OWNED_BY).map( + stringifyEntityRef, + ); for (const ref of entityOwnerRefs) { if (myOwnerRefs.has(ref)) { return true; diff --git a/plugins/catalog-react/src/hooks/useEntityPermission.ts b/plugins/catalog-react/src/hooks/useEntityPermission.ts index c1f62cfc39..092114936a 100644 --- a/plugins/catalog-react/src/hooks/useEntityPermission.ts +++ b/plugins/catalog-react/src/hooks/useEntityPermission.ts @@ -28,7 +28,7 @@ import { useEntity } from './useEntity'; * Note: this hook blocks the permission request until the entity has loaded in * context. If you have the entityRef and need concurrent requests, use the * `usePermission` hook directly. - * @public + * @alpha */ export function useEntityPermission(permission: Permission): { loading: boolean; diff --git a/plugins/catalog-react/src/hooks/useOwnUser.ts b/plugins/catalog-react/src/hooks/useOwnUser.ts index 74c44bc638..211321afdf 100644 --- a/plugins/catalog-react/src/hooks/useOwnUser.ts +++ b/plugins/catalog-react/src/hooks/useOwnUser.ts @@ -34,7 +34,12 @@ export function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx index aef876a1b2..16fa19a1cd 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntities.test.tsx @@ -15,15 +15,18 @@ */ import { Entity } from '@backstage/catalog-model'; -import { StorageApi } from '@backstage/core-plugin-api'; -import { MockStorageApi, TestApiProvider } from '@backstage/test-utils'; +import { TestApiProvider } from '@backstage/test-utils'; import { act, renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; -import { DefaultStarredEntitiesApi, starredEntitiesApiRef } from '../apis'; +import { + starredEntitiesApiRef, + StarredEntitiesApi, + MockStarredEntitiesApi, +} from '../apis'; import { useStarredEntities } from './useStarredEntities'; describe('useStarredEntities', () => { - let mockStorage: StorageApi; + let mockApi: StarredEntitiesApi; let wrapper: React.ComponentType; const mockEntity: Entity = { @@ -44,22 +47,15 @@ describe('useStarredEntities', () => { }; beforeEach(() => { - mockStorage = MockStorageApi.create(); + mockApi = new MockStarredEntitiesApi(); wrapper = ({ children }: PropsWithChildren<{}>) => ( - + {children} ); }); - it('should return an empty set for when there is no items in storage', async () => { + it('should return an empty set', async () => { const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), { wrapper }, @@ -70,10 +66,11 @@ describe('useStarredEntities', () => { expect(result.current.starredEntities.size).toBe(0); }); - it('should return a set with the current items when there are items in storage', async () => { + it('should return a set with the current items', async () => { const expectedIds = ['i', 'am', 'some', 'test', 'ids']; - const store = mockStorage?.forBucket('starredEntities'); - await store?.set('entityRefs', expectedIds); + for (const id of expectedIds) { + mockApi.toggleStarred(id); + } const { result, waitForNextUpdate } = renderHook( () => useStarredEntities(), diff --git a/plugins/catalog-react/src/hooks/useStarredEntities.ts b/plugins/catalog-react/src/hooks/useStarredEntities.ts index 5ddabfdd2e..328a26e0f8 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntities.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntities.ts @@ -16,7 +16,7 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; @@ -24,7 +24,9 @@ import { useCallback } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); @@ -33,8 +35,12 @@ function getEntityRef(entityOrRef: Entity | EntityName | string): string { /** @public */ export function useStarredEntities(): { starredEntities: Set; - toggleStarredEntity: (entityOrRef: Entity | EntityName | string) => void; - isStarredEntity: (entityOrRef: Entity | EntityName | string) => boolean; + toggleStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => void; + isStarredEntity: ( + entityOrRef: Entity | CompoundEntityRef | string, + ) => boolean; } { const starredEntitiesApi = useApi(starredEntitiesApiRef); @@ -44,13 +50,13 @@ export function useStarredEntities(): { ); const isStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntities.has(getEntityRef(entityOrRef)), [starredEntities], ); const toggleStarredEntity = useCallback( - (entityOrRef: Entity | EntityName | string) => + (entityOrRef: Entity | CompoundEntityRef | string) => starredEntitiesApi.toggleStarred(getEntityRef(entityOrRef)).then(), [starredEntitiesApi], ); diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx index 8ffc891092..317a0e391c 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx +++ b/plugins/catalog-react/src/hooks/useStarredEntity.test.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { TestApiProvider } from '@backstage/test-utils'; import { renderHook } from '@testing-library/react-hooks'; import React, { PropsWithChildren } from 'react'; @@ -44,7 +44,7 @@ describe('useStarredEntity', () => { describe.each` title | entityOrRef ${'entity reference'} | ${'component:default/mock'} - ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as EntityName} + ${'entity name'} | ${{ kind: 'component', namespace: 'default', name: 'mock' } as CompoundEntityRef} ${'entity'} | ${{ apiVersion: '1', kind: 'Component', metadata: { name: 'mock' } } as Entity} `('with $title', ({ entityOrRef }) => { describe('toggleStarredEntity', () => { diff --git a/plugins/catalog-react/src/hooks/useStarredEntity.ts b/plugins/catalog-react/src/hooks/useStarredEntity.ts index 2c647163aa..23d5677512 100644 --- a/plugins/catalog-react/src/hooks/useStarredEntity.ts +++ b/plugins/catalog-react/src/hooks/useStarredEntity.ts @@ -16,21 +16,25 @@ import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { useApi } from '@backstage/core-plugin-api'; import { useCallback, useEffect, useState } from 'react'; import { starredEntitiesApiRef } from '../apis'; -function getEntityRef(entityOrRef: Entity | EntityName | string): string { +function getEntityRef( + entityOrRef: Entity | CompoundEntityRef | string, +): string { return typeof entityOrRef === 'string' ? entityOrRef : stringifyEntityRef(entityOrRef); } /** @public */ -export function useStarredEntity(entityOrRef: Entity | EntityName | string): { +export function useStarredEntity( + entityOrRef: Entity | CompoundEntityRef | string, +): { toggleStarredEntity: () => void; isStarredEntity: boolean; } { diff --git a/plugins/catalog-react/src/index.ts b/plugins/catalog-react/src/index.ts index 7bf1bd1221..2279d49a2c 100644 --- a/plugins/catalog-react/src/index.ts +++ b/plugins/catalog-react/src/index.ts @@ -27,13 +27,7 @@ export * from './apis'; export * from './components'; export * from './hooks'; export * from './filters'; -export { - catalogRouteRef, - entityRoute, - entityRouteParams, - entityRouteRef, - rootRoute, -} from './routes'; +export { entityRouteParams, entityRouteRef } from './routes'; export * from './testUtils'; export * from './types'; export * from './utils'; diff --git a/plugins/catalog-react/src/routes.ts b/plugins/catalog-react/src/routes.ts index b9aa0de531..013bf7b5be 100644 --- a/plugins/catalog-react/src/routes.ts +++ b/plugins/catalog-react/src/routes.ts @@ -18,21 +18,6 @@ import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { createRouteRef } from '@backstage/core-plugin-api'; import { getOrCreateGlobalSingleton } from '@backstage/version-bridge'; -// TODO(Rugvip): Move these route refs back to the catalog plugin once we're all ported to using external routes -/** - * @deprecated Use an `ExternalRouteRef` instead, which can point to `catalogPlugin.routes.catalogIndex`. - * @public - */ -export const rootRoute = createRouteRef({ - id: 'catalog', -}); - -/** - * @deprecated Use an `ExternalRouteRef` instead, which can point to `catalogPlugin.routes.catalogIndex`. - * @public - */ -export const catalogRouteRef = rootRoute; - /** * A stable route ref that points to the catalog page for an individual entity. * @@ -52,12 +37,6 @@ export const entityRouteRef = getOrCreateGlobalSingleton( }), ); -/** - * @deprecated use `entityRouteRef` instead. - * @public - */ -export const entityRoute = entityRouteRef; - /** * Utility function to get suitable route params for entityRoute, given an * @public diff --git a/plugins/catalog-react/src/utils/getEntityRelations.ts b/plugins/catalog-react/src/utils/getEntityRelations.ts index 45cbc6967f..13086db4ea 100644 --- a/plugins/catalog-react/src/utils/getEntityRelations.ts +++ b/plugins/catalog-react/src/utils/getEntityRelations.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { Entity, EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { + Entity, + CompoundEntityRef, + parseEntityRef, +} from '@backstage/catalog-model'; // TODO(freben): This should be returning entity refs instead /** @@ -26,7 +30,7 @@ export function getEntityRelations( entity: Entity | undefined, relationType: string, filter?: { kind: string }, -): EntityName[] { +): CompoundEntityRef[] { let entityNames = entity?.relations ?.filter(r => r.type === relationType) diff --git a/plugins/catalog-react/src/utils/isOwnerOf.ts b/plugins/catalog-react/src/utils/isOwnerOf.ts index 3ef6b4b2a0..2b38ace247 100644 --- a/plugins/catalog-react/src/utils/isOwnerOf.ts +++ b/plugins/catalog-react/src/utils/isOwnerOf.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, stringifyEntityRef, @@ -24,18 +24,23 @@ import { import { getEntityRelations } from './getEntityRelations'; /** - * Get the related entity references. - * @public + * Returns true if the `owner` argument is a direct owner on the `entity` argument. + * + * @alpha + * @remarks + * + * Note that this ownership is not the same as using the claims in the auth-resolver, it only will take into account ownership as expressed by direct entity relations. + * It doesn't know anything about the additional groups that a user might belong to which the claims contain. */ -export function isOwnerOf(owner: Entity, owned: Entity) { +export function isOwnerOf(owner: Entity, entity: Entity) { const possibleOwners = new Set( [ ...getEntityRelations(owner, RELATION_MEMBER_OF, { kind: 'group' }), - ...(owner ? [getEntityName(owner)] : []), + ...(owner ? [getCompoundEntityRef(owner)] : []), ].map(stringifyEntityRef), ); - const owners = getEntityRelations(owned, RELATION_OWNED_BY).map( + const owners = getEntityRelations(entity, RELATION_OWNED_BY).map( stringifyEntityRef, ); diff --git a/plugins/catalog/api-report.md b/plugins/catalog/api-report.md index bd5edcbfa6..97a36ad020 100644 --- a/plugins/catalog/api-report.md +++ b/plugins/catalog/api-report.md @@ -7,16 +7,19 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { IconComponent } from '@backstage/core-plugin-api'; import { IndexableDocument } from '@backstage/search-common'; import { InfoCardVariants } from '@backstage/core-components'; +import { Observable } from '@backstage/types'; import { Overrides } from '@material-ui/core/styles/overrides'; import { default as React_2 } from 'react'; import { ReactNode } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; +import { StarredEntitiesApi } from '@backstage/plugin-catalog-react'; +import { StorageApi } from '@backstage/core-plugin-api'; import { StyleRules } from '@material-ui/core/styles/withStyles'; import { TableColumn } from '@backstage/core-components'; import { TableProps } from '@backstage/core-components'; @@ -154,9 +157,9 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } @@ -170,6 +173,15 @@ export interface DefaultCatalogPageProps { initiallySelectedFilter?: UserListFilterKind; } +// @public +export class DefaultStarredEntitiesApi implements StarredEntitiesApi { + constructor(opts: { storageApi: StorageApi }); + // (undocumented) + starredEntitie$(): Observable>; + // (undocumented) + toggleStarred(entityRef: string): Promise; +} + // @public (undocumented) export interface DependencyOfComponentsCardProps { // (undocumented) diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index f7800af139..6179cf6c44 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -44,6 +44,7 @@ "@backstage/plugin-catalog-react": "^0.7.0", "@backstage/search-common": "^0.2.4", "@backstage/theme": "^0.2.15", + "@backstage/types": "^0.1.2", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", @@ -51,7 +52,8 @@ "lodash": "^4.17.21", "react-helmet": "6.1.0", "react-router": "6.0.0-beta.0", - "react-use": "^17.2.4" + "react-use": "^17.2.4", + "zen-observable": "^0.8.15" }, "peerDependencies": { "@types/react": "^16.13.1 || ^17.0.0", diff --git a/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts new file mode 100644 index 0000000000..2a3c4d83ed --- /dev/null +++ b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MockStorageApi } from '@backstage/test-utils'; +import { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; +import { performMigrationToTheNewBucket } from './migration'; + +jest.mock('./migration'); + +function getStarred(api: DefaultStarredEntitiesApi) { + return new Promise((resolve, reject) => { + const subscription = api.starredEntitie$().subscribe({ + next(starred) { + resolve(starred); + subscription.unsubscribe(); + }, + error: reject, + }); + }); +} + +describe('DefaultStarredEntitiesApi', () => { + beforeEach(() => { + (performMigrationToTheNewBucket as jest.Mock).mockResolvedValue(undefined); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('constructor', () => { + it('should call migration', () => { + const api = new DefaultStarredEntitiesApi({ + storageApi: MockStorageApi.create(), + }); + expect(performMigrationToTheNewBucket).toBeCalledTimes(1); + expect(api).toBeDefined(); + }); + }); + + it('should notify and toggle starred entities', async () => { + const entityRef = 'component:default/mock'; + + const storageApi = MockStorageApi.create(); + const storageBucket = storageApi.forBucket('starredEntities'); + const api = new DefaultStarredEntitiesApi({ storageApi }); + + const values = new Array>(); + api.starredEntitie$().subscribe({ + next: value => { + values.push(value); + }, + }); + + await expect(getStarred(api)).resolves.toEqual(new Set()); + + await api.toggleStarred(entityRef); + await expect(getStarred(api)).resolves.toEqual(new Set([entityRef])); + expect(storageBucket.snapshot('entityRefs')).toEqual( + expect.objectContaining({ presence: 'present', value: [entityRef] }), + ); + + await api.toggleStarred(entityRef); + await expect(getStarred(api)).resolves.toEqual(new Set()); + expect(storageBucket.snapshot('entityRefs')).toEqual( + expect.objectContaining({ presence: 'present', value: [] }), + ); + + expect(values).toEqual([new Set(), new Set([entityRef]), new Set()]); + }); + + it('should read starred entities from storage', async () => { + const entityRef = 'component:default/mock'; + + const storageApi = MockStorageApi.create(); + const storageBucket = storageApi.forBucket('starredEntities'); + storageBucket.set('entityRefs', [entityRef]); + const api = new DefaultStarredEntitiesApi({ storageApi }); + + await expect(getStarred(api)).resolves.toEqual(new Set([entityRef])); + }); +}); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts similarity index 86% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts index 46a8e63084..f4b7771d04 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/DefaultStarredEntitiesApi.ts @@ -15,10 +15,10 @@ */ import { StorageApi } from '@backstage/core-plugin-api'; +import { StarredEntitiesApi } from '@backstage/plugin-catalog-react'; import { Observable } from '@backstage/types'; import ObservableImpl from 'zen-observable'; import { performMigrationToTheNewBucket } from './migration'; -import { StarredEntitiesApi } from './StarredEntitiesApi'; /** * Default implementation of the StarredEntitiesApi that is backed by the StorageApi. @@ -36,12 +36,12 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { this.settingsStore = opts.storageApi.forBucket('starredEntities'); this.starredEntities = new Set( - this.settingsStore.get('entityRefs') ?? [], + this.settingsStore.snapshot('entityRefs').value ?? [], ); this.settingsStore.observe$('entityRefs').subscribe({ next: next => { - this.starredEntities = new Set(next.newValue ?? []); + this.starredEntities = new Set(next.value ?? []); this.notifyChanges(); }, }); @@ -64,17 +64,13 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { return this.observable; } - isStarred(entityRef: string): boolean { - return this.starredEntities.has(entityRef); - } - private readonly subscribers = new Set< ZenObservable.SubscriptionObserver> >(); private readonly observable = new ObservableImpl>(subscriber => { // forward the the latest value - subscriber.next(this.starredEntities); + subscriber.next(new Set(this.starredEntities)); this.subscribers.add(subscriber); return () => { @@ -84,7 +80,7 @@ export class DefaultStarredEntitiesApi implements StarredEntitiesApi { private notifyChanges() { for (const subscription of this.subscribers) { - subscription.next(this.starredEntities); + subscription.next(new Set(this.starredEntities)); } } } diff --git a/plugins/catalog-backend/src/providers/index.ts b/plugins/catalog/src/apis/StarredEntitiesApi/index.ts similarity index 85% rename from plugins/catalog-backend/src/providers/index.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/index.ts index 3b4a1c4b7e..42dc977fb6 100644 --- a/plugins/catalog-backend/src/providers/index.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/index.ts @@ -14,8 +14,4 @@ * limitations under the License. */ -export type { - EntityProvider, - EntityProviderConnection, - EntityProviderMutation, -} from './types'; +export { DefaultStarredEntitiesApi } from './DefaultStarredEntitiesApi'; diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts similarity index 81% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts index 7eb5648d13..b3bc5400d5 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.test.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/migration.test.ts @@ -41,19 +41,19 @@ describe('performMigrationToTheNewBucket', () => { 'entity:Component:default:a', 'entity:template:custom:b', ]); - expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).not.toBeUndefined(); await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(await newBucket.get('entityRefs')).toEqual([ + expect(await newBucket.snapshot('entityRefs').value).toEqual([ 'component:default/c', 'component:default/a', 'template:custom/b', ]); // OLD bucket should be removed - expect(oldBucket.get('starredEntities')).toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).toBeUndefined(); }); it('should ignore invalid entries', async () => { @@ -67,15 +67,17 @@ describe('performMigrationToTheNewBucket', () => { 'entity:Component:a', 'invalid', ]); - expect(oldBucket.get('starredEntities')).not.toBeUndefined(); + expect(oldBucket.snapshot('starredEntities')).not.toBeUndefined(); await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(await newBucket.get('entityRefs')).toEqual(['component:default/a']); + expect(await newBucket.snapshot('entityRefs').value).toEqual([ + 'component:default/a', + ]); // OLD bucket should be removed - expect(oldBucket.get('starredEntities')).toBeUndefined(); + expect(oldBucket.snapshot('starredEntities').value).toBeUndefined(); }); it('should skip migration without old starred entities', async () => { @@ -88,7 +90,7 @@ describe('performMigrationToTheNewBucket', () => { await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(newBucket.get('entityRefs')).toEqual(expectedEntries); + expect(newBucket.snapshot('entityRefs').value).toEqual(expectedEntries); }); it('should skip migration with non-array old starred entities', async () => { @@ -105,9 +107,9 @@ describe('performMigrationToTheNewBucket', () => { await performMigrationToTheNewBucket({ storageApi: mockStorage }); // read NEW bucket - expect(newBucket.get('entityRefs')).toEqual(expectedEntries); + expect(newBucket.snapshot('entityRefs').value).toEqual(expectedEntries); // OLD bucket should be unchanged - expect(oldBucket.get('starredEntities')).toBe('invalid'); + expect(oldBucket.snapshot('starredEntities').value).toBe('invalid'); }); }); diff --git a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts b/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts similarity index 92% rename from plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts rename to plugins/catalog/src/apis/StarredEntitiesApi/migration.ts index 8405d9e08d..51dcd6e7ce 100644 --- a/plugins/catalog-react/src/apis/StarredEntitiesApi/migration.ts +++ b/plugins/catalog/src/apis/StarredEntitiesApi/migration.ts @@ -35,13 +35,15 @@ export async function performMigrationToTheNewBucket({ const source = storageApi.forBucket('settings'); const target = storageApi.forBucket('starredEntities'); - const oldStarredEntities = source.get('starredEntities'); + const oldStarredEntities = source.snapshot('starredEntities').value; if (!isArray(oldStarredEntities)) { // nothing to do return; } - const targetEntities = new Set(target.get('entityRefs') ?? []); + const targetEntities = new Set( + target.snapshot('entityRefs').value ?? [], + ); oldStarredEntities .filter(isString) diff --git a/plugins/catalog-backend/src/ingestion/providers/index.ts b/plugins/catalog/src/apis/index.ts similarity index 89% rename from plugins/catalog-backend/src/ingestion/providers/index.ts rename to plugins/catalog/src/apis/index.ts index cd1bc5cb36..5c7e980890 100644 --- a/plugins/catalog-backend/src/ingestion/providers/index.ts +++ b/plugins/catalog/src/apis/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider'; +export * from './StarredEntitiesApi'; diff --git a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts index 605305882e..f4d06b0bb2 100644 --- a/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts +++ b/plugins/catalog/src/components/CatalogEntityPage/useEntityFromUrl.ts @@ -40,7 +40,7 @@ export const useEntityFromUrl = (): EntityLoadingStatus => { loading, retry: refresh, } = useAsyncRetry( - () => catalogApi.getEntityByName({ kind, namespace, name }), + () => catalogApi.getEntityByRef({ kind, namespace, name }), [catalogApi, kind, namespace, name], ); diff --git a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx index dcf819f976..07bd81b287 100644 --- a/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx +++ b/plugins/catalog/src/components/CatalogPage/DefaultCatalogPage.test.tsx @@ -17,6 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, + parseEntityRef, RELATION_MEMBER_OF, RELATION_OWNED_BY, } from '@backstage/catalog-model'; @@ -29,9 +30,9 @@ import { } from '@backstage/core-plugin-api'; import { catalogApiRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { mockBreakpoint, @@ -104,11 +105,11 @@ describe('DefaultCatalogPage', () => { }), getLocationByRef: () => Promise.resolve({ id: 'id', type: 'url', target: 'url' }), - getEntityByName: async entityName => { + getEntityByRef: async entityRef => { return { apiVersion: 'backstage.io/v1alpha1', kind: 'User', - metadata: { name: entityName.name }, + metadata: { name: parseEntityRef(entityRef).name }, relations: [ { type: RELATION_MEMBER_OF, @@ -141,10 +142,7 @@ describe('DefaultCatalogPage', () => { [catalogApiRef, catalogApi], [identityApiRef, identityApi], [storageApiRef, storageApi], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ]} > {children} diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx index db9835e3a3..cc05bbe2ae 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.test.tsx @@ -22,16 +22,12 @@ import { import { ApiProvider } from '@backstage/core-app-api'; import { entityRouteRef, - DefaultStarredEntitiesApi, MockEntityListContextProvider, starredEntitiesApiRef, UserListFilter, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; -import { - MockStorageApi, - renderInTestApp, - TestApiRegistry, -} from '@backstage/test-utils'; +import { renderInTestApp, TestApiRegistry } from '@backstage/test-utils'; import { act, fireEvent } from '@testing-library/react'; import * as React from 'react'; import { CatalogTable } from './CatalogTable'; @@ -57,7 +53,7 @@ const entities: Entity[] = [ describe('CatalogTable component', () => { const mockApis = TestApiRegistry.from([ starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), + new MockStarredEntitiesApi(), ]); beforeEach(() => { diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index a9a6f26228..9d59d57647 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -20,9 +20,7 @@ import { RELATION_PART_OF, } from '@backstage/catalog-model'; import { - favoriteEntityIcon, - favoriteEntityTooltip, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, useEntityList, useStarredEntities, @@ -40,6 +38,9 @@ import { TableProps, WarningPanel, } from '@backstage/core-components'; +import StarBorder from '@material-ui/icons/StarBorder'; +import { withStyles } from '@material-ui/core/styles'; +import Star from '@material-ui/icons/Star'; /** * Props for {@link CatalogTable}. @@ -51,6 +52,12 @@ export interface CatalogTableProps { actions?: TableProps['actions']; } +const YellowStar = withStyles({ + root: { + color: '#f3ba37', + }, +})(Star); + /** @public */ export const CatalogTable = (props: CatalogTableProps) => { const { columns, actions } = props; @@ -116,8 +123,8 @@ export const CatalogTable = (props: CatalogTableProps) => { const isStarred = isStarredEntity(entity); return { cellStyle: { paddingLeft: '1em' }, - icon: () => favoriteEntityIcon(isStarred), - tooltip: favoriteEntityTooltip(isStarred), + icon: () => (isStarred ? : ), + tooltip: isStarred ? 'Remove from favorites' : 'Add to favorites', onClick: () => toggleStarredEntity(entity), }; }, @@ -132,16 +139,16 @@ export const CatalogTable = (props: CatalogTableProps) => { return { entity, resolved: { - name: formatEntityRefTitle(entity, { + name: humanizeEntityRef(entity, { defaultKind: 'Component', }), ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), ownedByRelations, partOfSystemRelationTitle: partOfSystemRelations .map(r => - formatEntityRefTitle(r, { + humanizeEntityRef(r, { defaultKind: 'system', }), ) diff --git a/plugins/catalog/src/components/CatalogTable/columns.tsx b/plugins/catalog/src/components/CatalogTable/columns.tsx index d243be7c6f..0fb91d4f3f 100644 --- a/plugins/catalog/src/components/CatalogTable/columns.tsx +++ b/plugins/catalog/src/components/CatalogTable/columns.tsx @@ -15,7 +15,7 @@ */ import React from 'react'; import { - formatEntityRefTitle, + humanizeEntityRef, EntityRefLink, EntityRefLinks, } from '@backstage/plugin-catalog-react'; @@ -34,7 +34,7 @@ export const columnFactories = Object.freeze({ function formatContent(entity: Entity): string { return ( entity.metadata?.title || - formatEntityRefTitle(entity, { + humanizeEntityRef(entity, { defaultKind: options?.defaultKind, }) ); diff --git a/plugins/catalog/src/components/CatalogTable/types.ts b/plugins/catalog/src/components/CatalogTable/types.ts index 7eaff6faed..5bbb3af122 100644 --- a/plugins/catalog/src/components/CatalogTable/types.ts +++ b/plugins/catalog/src/components/CatalogTable/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** @public */ export interface CatalogTableRow { @@ -22,8 +22,8 @@ export interface CatalogTableRow { resolved: { name: string; partOfSystemRelationTitle?: string; - partOfSystemRelations: EntityName[]; + partOfSystemRelations: CompoundEntityRef[]; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; } diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx index 78dcf855e8..c1deaf4b74 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.test.tsx @@ -21,15 +21,14 @@ import { AlertApi, alertApiRef } from '@backstage/core-plugin-api'; import { AsyncEntityProvider, catalogApiRef, - DefaultStarredEntitiesApi, EntityProvider, entityRouteRef, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { permissionApiRef } from '@backstage/plugin-permission-react'; import { MockPermissionApi, - MockStorageApi, renderInTestApp, TestApiRegistry, } from '@backstage/test-utils'; @@ -48,10 +47,7 @@ const mockEntity = { const mockApis = TestApiRegistry.from( [catalogApiRef, {} as CatalogApi], [alertApiRef, {} as AlertApi], - [ - starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ storageApi: MockStorageApi.create() }), - ], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], [permissionApiRef, new MockPermissionApi()], ); diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx index 634662f386..40c4f9e873 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx @@ -40,7 +40,7 @@ import { getEntityRelations, InspectEntityDialog, UnregisterEntityDialog, - useEntity, + useAsyncEntity, useEntityCompoundName, } from '@backstage/plugin-catalog-react'; import { Box, TabProps } from '@material-ui/core'; @@ -177,7 +177,7 @@ export const EntityLayout = (props: EntityLayoutProps) => { children, } = props; const { kind, namespace, name } = useEntityCompoundName(); - const { entity, loading, error } = useEntity(); + const { entity, loading, error } = useAsyncEntity(); const location = useLocation(); const routes = useElementFilter( children, @@ -190,7 +190,9 @@ export const EntityLayout = (props: EntityLayoutProps) => { }) .getElements() // all nodes, element data, maintain structure or not? .flatMap(({ props: elementProps }) => { - if (elementProps.if && entity && !elementProps.if(entity)) { + if (!entity) { + return []; + } else if (elementProps.if && !elementProps.if(entity)) { return []; } diff --git a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx index a8aac405fe..f00474868f 100644 --- a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx +++ b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.test.tsx @@ -14,14 +14,11 @@ * limitations under the License. */ -import { - catalogApiRef, - catalogRouteRef, - EntityProvider, -} from '@backstage/plugin-catalog-react'; +import { catalogApiRef, EntityProvider } from '@backstage/plugin-catalog-react'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import React from 'react'; +import { rootRouteRef } from '../../routes'; import { EntityOrphanWarning } from './EntityOrphanWarning'; describe('', () => { @@ -59,7 +56,7 @@ describe('', () => { , { mountedRoutes: { - '/create': catalogRouteRef, + '/create': rootRouteRef, }, }, ); diff --git a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx index aba28c558f..26b0e5de45 100644 --- a/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx +++ b/plugins/catalog/src/components/EntityOrphanWarning/EntityOrphanWarning.tsx @@ -15,12 +15,13 @@ */ import { Entity } from '@backstage/catalog-model'; -import { catalogRouteRef, useEntity } from '@backstage/plugin-catalog-react'; +import { useEntity } from '@backstage/plugin-catalog-react'; import { Alert } from '@material-ui/lab'; import React, { useState } from 'react'; import { useNavigate } from 'react-router'; import { DeleteEntityDialog } from './DeleteEntityDialog'; import { useRouteRef } from '@backstage/core-plugin-api'; +import { rootRouteRef } from '../../routes'; /** * Returns true if the given entity has the orphan annotation given by the @@ -40,7 +41,7 @@ export function isOrphan(entity: Entity): boolean { */ export function EntityOrphanWarning() { const navigate = useNavigate(); - const catalogLink = useRouteRef(catalogRouteRef); + const catalogLink = useRouteRef(rootRouteRef); const [confirmationDialogOpen, setConfirmationDialogOpen] = useState(false); const { entity } = useEntity(); diff --git a/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx b/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx index 4433517247..1b8667ce2a 100644 --- a/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx +++ b/plugins/catalog/src/components/EntitySwitch/EntitySwitch.tsx @@ -15,7 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; -import { useEntity } from '@backstage/plugin-catalog-react'; +import { useAsyncEntity } from '@backstage/plugin-catalog-react'; import React, { ReactNode, ReactElement } from 'react'; import { attachComponentData, @@ -63,7 +63,7 @@ export interface EntitySwitchProps { /** @public */ export const EntitySwitch = (props: EntitySwitchProps) => { - const { entity } = useEntity(); + const { entity } = useAsyncEntity(); const apis = useApiHolder(); const results = useElementFilter( props.children, @@ -75,6 +75,9 @@ export const EntitySwitch = (props: EntitySwitchProps) => { }) .getElements() .flatMap((element: ReactElement) => { + if (!entity) { + return []; + } const { if: condition, children: elementsChildren } = element.props as EntitySwitchCase; return [ diff --git a/plugins/catalog/src/components/EntitySwitch/conditions.ts b/plugins/catalog/src/components/EntitySwitch/conditions.ts index a02170cc6f..907aa45afe 100644 --- a/plugins/catalog/src/components/EntitySwitch/conditions.ts +++ b/plugins/catalog/src/components/EntitySwitch/conditions.ts @@ -27,7 +27,7 @@ function strCmp(a: string | undefined, b: string | undefined): boolean { * @public */ export function isKind(kind: string) { - return (entity: Entity) => strCmp(entity?.kind, kind); + return (entity: Entity) => strCmp(entity.kind, kind); } /** @@ -36,7 +36,7 @@ export function isKind(kind: string) { */ export function isComponentType(type: string) { return (entity: Entity) => { - if (!strCmp(entity?.kind, 'component')) { + if (!strCmp(entity.kind, 'component')) { return false; } const componentEntity = entity as ComponentEntity; @@ -49,5 +49,5 @@ export function isComponentType(type: string) { * @public */ export function isNamespace(namespace: string) { - return (entity: Entity) => strCmp(entity?.metadata?.namespace, namespace); + return (entity: Entity) => strCmp(entity.metadata?.namespace, namespace); } diff --git a/plugins/catalog/src/index.ts b/plugins/catalog/src/index.ts index fef420d6d3..662a920f8b 100644 --- a/plugins/catalog/src/index.ts +++ b/plugins/catalog/src/index.ts @@ -20,6 +20,8 @@ * @packageDocumentation */ +export * from './apis'; + export * from './components/AboutCard'; export * from './components/CatalogKindHeader'; export * from './components/CatalogSearchResultListItem'; diff --git a/plugins/catalog/src/plugin.ts b/plugins/catalog/src/plugin.ts index b4d1a61bc1..1b9083e7ca 100644 --- a/plugins/catalog/src/plugin.ts +++ b/plugins/catalog/src/plugin.ts @@ -18,8 +18,6 @@ import { CatalogClient } from '@backstage/catalog-client'; import { Entity } from '@backstage/catalog-model'; import { catalogApiRef, - catalogRouteRef, - DefaultStarredEntitiesApi, entityRouteRef, starredEntitiesApiRef, } from '@backstage/plugin-catalog-react'; @@ -33,6 +31,7 @@ import { fetchApiRef, storageApiRef, } from '@backstage/core-plugin-api'; +import { DefaultStarredEntitiesApi } from './apis'; import { AboutCardProps } from './components/AboutCard'; import { DefaultCatalogPageProps } from './components/CatalogPage'; import { DependencyOfComponentsCardProps } from './components/DependencyOfComponentsCard'; @@ -43,6 +42,7 @@ import { HasResourcesCardProps } from './components/HasResourcesCard'; import { HasSubcomponentsCardProps } from './components/HasSubcomponentsCard'; import { HasSystemsCardProps } from './components/HasSystemsCard'; import { RelatedEntitiesCardProps } from './components/RelatedEntitiesCard'; +import { rootRouteRef } from './routes'; /** @public */ export const catalogPlugin = createPlugin({ @@ -65,7 +65,7 @@ export const catalogPlugin = createPlugin({ }), ], routes: { - catalogIndex: catalogRouteRef, + catalogIndex: rootRouteRef, catalogEntity: entityRouteRef, }, externalRoutes: { @@ -81,7 +81,7 @@ export const CatalogIndexPage: (props: DefaultCatalogPageProps) => JSX.Element = name: 'CatalogIndexPage', component: () => import('./components/CatalogPage').then(m => m.CatalogPage), - mountPoint: catalogRouteRef, + mountPoint: rootRouteRef, }), ); diff --git a/plugins/catalog/src/routes.ts b/plugins/catalog/src/routes.ts index 63d9ccfb87..5c0d195d74 100644 --- a/plugins/catalog/src/routes.ts +++ b/plugins/catalog/src/routes.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { createExternalRouteRef } from '@backstage/core-plugin-api'; +import { + createExternalRouteRef, + createRouteRef, +} from '@backstage/core-plugin-api'; export const createComponentRouteRef = createExternalRouteRef({ id: 'create-component', @@ -26,3 +29,7 @@ export const viewTechDocRouteRef = createExternalRouteRef({ optional: true, params: ['namespace', 'kind', 'name'], }); + +export const rootRouteRef = createRouteRef({ + id: 'catalog', +}); diff --git a/plugins/code-coverage-backend/src/service/router.ts b/plugins/code-coverage-backend/src/service/router.ts index 4922f5c887..488103c835 100644 --- a/plugins/code-coverage-backend/src/service/router.ts +++ b/plugins/code-coverage-backend/src/service/router.ts @@ -18,7 +18,7 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import xmlparser from 'express-xml-bodyparser'; -import { CatalogClient } from '@backstage/catalog-client'; +import { CatalogApi, CatalogClient } from '@backstage/catalog-client'; import { errorHandler, PluginDatabaseManager, @@ -33,10 +33,7 @@ import { aggregateCoverage, CoverageUtils } from './CoverageUtils'; import { Cobertura } from './converter/cobertura'; import { Jacoco } from './converter/jacoco'; import { Converter } from './converter'; -import { - getEntitySourceLocation, - parseEntityRef, -} from '@backstage/catalog-model'; +import { getEntitySourceLocation } from '@backstage/catalog-model'; export interface RouterOptions { config: Config; @@ -59,7 +56,7 @@ export const makeRouter = async ( await database.getClient(), ); const codecovUrl = await discovery.getExternalBaseUrl('code-coverage'); - const catalogApi = new CatalogClient({ discoveryApi: discovery }); + const catalogApi: CatalogApi = new CatalogClient({ discoveryApi: discovery }); const scm = ScmIntegrations.fromConfig(config); const router = Router(); @@ -77,8 +74,7 @@ export const makeRouter = async ( */ router.get('/report', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -100,8 +96,7 @@ export const makeRouter = async ( */ router.get('/history', async (req, res) => { const { entity } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -119,8 +114,7 @@ export const makeRouter = async ( */ router.get('/file-content', async (req, res) => { const { entity, path } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } @@ -171,8 +165,7 @@ export const makeRouter = async ( */ router.post('/report', async (req, res) => { const { entity, coverageType } = req.query; - const entityName = parseEntityRef(entity as string); - const entityLookup = await catalogApi.getEntityByName(entityName); + const entityLookup = await catalogApi.getEntityByRef(entity as string); if (!entityLookup) { throw new NotFoundError(`No entity found matching ${entity}`); } diff --git a/plugins/code-coverage-backend/src/service/types.ts b/plugins/code-coverage-backend/src/service/types.ts index c4c622bfaf..31edad88d1 100644 --- a/plugins/code-coverage-backend/src/service/types.ts +++ b/plugins/code-coverage-backend/src/service/types.ts @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/code-coverage/src/api.ts b/plugins/code-coverage/src/api.ts index 61f920106b..081f1c7999 100644 --- a/plugins/code-coverage/src/api.ts +++ b/plugins/code-coverage/src/api.ts @@ -14,20 +14,25 @@ * limitations under the License. */ -import { EntityName, stringifyEntityRef } from '@backstage/catalog-model'; +import { + CompoundEntityRef, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; import { JsonCodeCoverage, JsonCoverageHistory } from './types'; import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api'; export type CodeCoverageApi = { discovery: DiscoveryApi; - getCoverageForEntity: (entity: EntityName) => Promise; + getCoverageForEntity: ( + entity: CompoundEntityRef, + ) => Promise; getFileContentFromEntity: ( - entity: EntityName, + entity: CompoundEntityRef, filePath: string, ) => Promise; getCoverageHistoryForEntity: ( - entity: EntityName, + entity: CompoundEntityRef, limit?: number, ) => Promise; }; @@ -59,7 +64,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); return (await this.fetch( @@ -68,7 +73,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getFileContentFromEntity( - entityName: EntityName, + entityName: CompoundEntityRef, filePath: string, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); @@ -78,7 +83,7 @@ export class CodeCoverageRestApi implements CodeCoverageApi { } async getCoverageHistoryForEntity( - entityName: EntityName, + entityName: CompoundEntityRef, limit?: number, ): Promise { const entity = encodeURIComponent(stringifyEntityRef(entityName)); diff --git a/plugins/code-coverage/src/types.ts b/plugins/code-coverage/src/types.ts index c4c622bfaf..b03025b7ad 100644 --- a/plugins/code-coverage/src/types.ts +++ b/plugins/code-coverage/src/types.ts @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; + +import { CompoundEntityRef } from '@backstage/catalog-model'; export type JsonCodeCoverage = { metadata: CoverageMetadata; - entity: EntityName; + entity: CompoundEntityRef; files: Array; }; export type JsonCoverageHistory = { - entity: EntityName; + entity: CompoundEntityRef; history: Array; }; diff --git a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx index 53b225daaf..c490dd00da 100644 --- a/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx +++ b/plugins/explore/src/components/DefaultExplorePage/DefaultExplorePage.test.tsx @@ -28,6 +28,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx index 1310d9f31e..d103541333 100644 --- a/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx +++ b/plugins/explore/src/components/DomainExplorerContent/DomainExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx index 36788ed1b6..23553b8b76 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsDiagram.tsx @@ -31,7 +31,7 @@ import { configApiRef, useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, entityRouteRef, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { BackstageTheme } from '@backstage/theme'; @@ -193,7 +193,7 @@ export function GroupsDiagram() { kind: catalogItem.kind, name: (catalogItem as GroupEntity).spec?.profile?.displayName || - formatEntityRefTitle(catalogItem, { defaultKind: 'Group' }), + humanizeEntityRef(catalogItem, { defaultKind: 'Group' }), }); // Edge to parent diff --git a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx index 255db061aa..99500336cf 100644 --- a/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx +++ b/plugins/explore/src/components/GroupsExplorerContent/GroupsExplorerContent.test.tsx @@ -29,6 +29,7 @@ describe('', () => { getLocationById: jest.fn(), removeLocationById: jest.fn(), removeEntityByUid: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), refreshEntity: jest.fn(), getEntityAncestors: jest.fn(), diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx index 2759afd585..f178b7de7f 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.test.tsx @@ -29,6 +29,7 @@ describe('', () => { const catalogApi: jest.Mocked = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), diff --git a/plugins/fossa/src/components/FossaPage/FossaPage.tsx b/plugins/fossa/src/components/FossaPage/FossaPage.tsx index 1b53546ccd..731852d4a7 100644 --- a/plugins/fossa/src/components/FossaPage/FossaPage.tsx +++ b/plugins/fossa/src/components/FossaPage/FossaPage.tsx @@ -16,14 +16,14 @@ import { Entity, - EntityName, + CompoundEntityRef, RELATION_OWNED_BY, } from '@backstage/catalog-model'; import { catalogApiRef, EntityRefLink, EntityRefLinks, - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { Tooltip } from '@material-ui/core'; @@ -56,7 +56,7 @@ type FossaRow = { resolved: { name: string; ownedByRelationsTitle?: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; loading: boolean; details?: FindingSummary; }; @@ -222,10 +222,10 @@ export const FossaPage = ({ return { entity, resolved: { - name: formatEntityRefTitle(entity), + name: humanizeEntityRef(entity), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), loading: summariesLoading, details: summary, diff --git a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx index ccc36f6964..9087ed3605 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/Content.test.tsx @@ -13,40 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - renderInTestApp, - TestApiProvider, - MockStorageApi, -} from '@backstage/test-utils'; +import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, - DefaultStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import React from 'react'; import { Content } from './Content'; describe('StarredEntitiesContent', () => { it('should render list of tools', async () => { - const mockStorageApi = MockStorageApi.create(); - await mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/mock-starred-entity', - 'component:default/mock-starred-entity-2', - ]); + const mockedApi = new MockStarredEntitiesApi(); + mockedApi.toggleStarred('component:default/mock-starred-entity'); + mockedApi.toggleStarred('component:default/mock-starred-entity-2'); const { getByText } = await renderInTestApp( - + , { diff --git a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx index 2e63762369..1b01820482 100644 --- a/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx +++ b/plugins/home/src/homePageComponents/StarredEntities/StarredEntities.stories.tsx @@ -15,44 +15,27 @@ */ import { HomePageStarredEntities } from '../../plugin'; -import { - wrapInTestApp, - TestApiProvider, - MockStorageApi, -} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider } from '@backstage/test-utils'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, - DefaultStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { Grid } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4', - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Components/StarredEntities', decorators: [ (Story: ComponentType<{}>) => wrapInTestApp( - + , { diff --git a/plugins/home/src/templates/DefaultTemplate.stories.tsx b/plugins/home/src/templates/DefaultTemplate.stories.tsx index ba5e1f33fd..c726294ba6 100644 --- a/plugins/home/src/templates/DefaultTemplate.stories.tsx +++ b/plugins/home/src/templates/DefaultTemplate.stories.tsx @@ -21,12 +21,12 @@ import { HomePageCompanyLogo, HomePageStarredEntities, } from '../plugin'; -import { wrapInTestApp, TestApiProvider, MockStorageApi} from '@backstage/test-utils'; +import { wrapInTestApp, TestApiProvider} from '@backstage/test-utils'; import { Content, Page, InfoCard } from '@backstage/core-components'; import { starredEntitiesApiRef, + MockStarredEntitiesApi, entityRouteRef, - DefaultStarredEntitiesApi } from '@backstage/plugin-catalog-react'; import { HomePageSearchBar, @@ -37,15 +37,11 @@ import { import { Grid, makeStyles } from '@material-ui/core'; import React, { ComponentType } from 'react'; -const mockStorageApi = MockStorageApi.create(); -mockStorageApi - .forBucket('starredEntities') - .set('entityRefs', [ - 'component:default/example-starred-entity', - 'component:default/example-starred-entity-2', - 'component:default/example-starred-entity-3', - 'component:default/example-starred-entity-4' - ]); +const starredEntitiesApi = new MockStarredEntitiesApi(); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-2'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-3'); +starredEntitiesApi.toggleStarred('component:default/example-starred-entity-4'); export default { title: 'Plugins/Home/Templates', @@ -57,9 +53,7 @@ export default { apis={[ [ starredEntitiesApiRef, - new DefaultStarredEntitiesApi({ - storageApi: mockStorageApi, - }), + starredEntitiesApi, ], [searchApiRef, { query: () => Promise.resolve({ results: [] }) }], ]} @@ -153,4 +147,3 @@ export const DefaultTemplate = () => { ); }; - diff --git a/plugins/jenkins-backend/README.md b/plugins/jenkins-backend/README.md index 1413a45136..443cb33c05 100644 --- a/plugins/jenkins-backend/README.md +++ b/plugins/jenkins-backend/README.md @@ -166,7 +166,7 @@ class AcmeJenkinsInfoProvider implements JenkinsInfoProvider { const PAAS_ANNOTATION = 'acme.example.com/paas-project-name'; // lookup pass-project-name from entity annotation - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/jenkins-backend/api-report.md b/plugins/jenkins-backend/api-report.md index d437a5dd75..cbc3ef4325 100644 --- a/plugins/jenkins-backend/api-report.md +++ b/plugins/jenkins-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; @@ -25,7 +25,7 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { }): DefaultJenkinsInfoProvider; // (undocumented) getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; // (undocumented) @@ -65,7 +65,7 @@ export interface JenkinsInfo { export interface JenkinsInfoProvider { // (undocumented) getInstance(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise; } diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts index 62299926df..a3a5ba069d 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.test.ts @@ -15,7 +15,7 @@ */ import { CatalogApi } from '@backstage/catalog-client'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import { DefaultJenkinsInfoProvider, @@ -160,10 +160,10 @@ describe('JenkinsConfig', () => { describe('DefaultJenkinsInfoProvider', () => { const mockCatalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any as jest.Mocked; - const entityRef: EntityName = { + const entityRef: CompoundEntityRef = { kind: 'Component', namespace: 'foo', name: 'bar', @@ -171,7 +171,7 @@ describe('DefaultJenkinsInfoProvider', () => { function configureProvider(configData: any, entityData: any) { const config = new ConfigReader(configData); - mockCatalog.getEntityByName.mockReturnValueOnce( + mockCatalog.getEntityByRef.mockReturnValueOnce( Promise.resolve(entityData as Entity), ); @@ -185,7 +185,7 @@ describe('DefaultJenkinsInfoProvider', () => { const provider = configureProvider({ jenkins: {} }, undefined); await expect(provider.getInstance({ entityRef })).rejects.toThrowError(); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); }); it('Reads simple config and annotation', async () => { @@ -207,7 +207,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toStrictEqual({ baseUrl: 'https://jenkins.example.com', crumbIssuer: undefined, @@ -243,7 +243,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -280,7 +280,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -317,7 +317,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', @@ -343,7 +343,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -369,7 +369,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins.example.com', jobFullName: 'teamA/artistLookup-build', @@ -400,7 +400,7 @@ describe('DefaultJenkinsInfoProvider', () => { ); const info: JenkinsInfo = await provider.getInstance({ entityRef }); - expect(mockCatalog.getEntityByName).toBeCalledWith(entityRef); + expect(mockCatalog.getEntityByRef).toBeCalledWith(entityRef); expect(info).toMatchObject({ baseUrl: 'https://jenkins-other.example.com', jobFullName: 'teamA/artistLookup-build', diff --git a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts index f822f475f1..a88dbf965e 100644 --- a/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts +++ b/plugins/jenkins-backend/src/service/jenkinsInfoProvider.ts @@ -17,7 +17,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; @@ -27,7 +27,7 @@ export interface JenkinsInfoProvider { /** * The entity to get the info about. */ - entityRef: EntityName; + entityRef: CompoundEntityRef; /** * A specific job to get. This is only passed in when we know about a job name we are interested in. */ @@ -182,11 +182,11 @@ export class DefaultJenkinsInfoProvider implements JenkinsInfoProvider { } async getInstance(opt: { - entityRef: EntityName; + entityRef: CompoundEntityRef; jobFullName?: string; }): Promise { // load entity - const entity = await this.catalog.getEntityByName(opt.entityRef); + const entity = await this.catalog.getEntityByRef(opt.entityRef); if (!entity) { throw new Error( `Couldn't find entity with name: ${stringifyEntityRef(opt.entityRef)}`, diff --git a/plugins/jenkins-backend/src/service/standaloneServer.ts b/plugins/jenkins-backend/src/service/standaloneServer.ts index 7f47d57ac3..f89f235667 100644 --- a/plugins/jenkins-backend/src/service/standaloneServer.ts +++ b/plugins/jenkins-backend/src/service/standaloneServer.ts @@ -18,7 +18,7 @@ import { createServiceBuilder } from '@backstage/backend-common'; import { Server } from 'http'; import { Logger } from 'winston'; import { createRouter } from './router'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { JenkinsInfo } from './jenkinsInfoProvider'; export interface ServerOptions { @@ -35,7 +35,9 @@ export async function startStandaloneServer( const router = await createRouter({ logger, jenkinsInfoProvider: { - async getInstance(_: { entityRef: EntityName }): Promise { + async getInstance(_: { + entityRef: CompoundEntityRef; + }): Promise { return { baseUrl: 'https://example.com/', jobFullName: 'build-foo' }; }, }, diff --git a/plugins/jenkins/api-report.md b/plugins/jenkins/api-report.md index 5db5fd0dfe..fc9de04a7a 100644 --- a/plugins/jenkins/api-report.md +++ b/plugins/jenkins/api-report.md @@ -7,9 +7,9 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import type { EntityName } from '@backstage/catalog-model'; import { IdentityApi } from '@backstage/core-plugin-api'; import { InfoCardVariants } from '@backstage/core-components'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -48,20 +48,20 @@ export const JENKINS_ANNOTATION = 'jenkins.io/job-full-name'; export interface JenkinsApi { // Warning: (ae-forgotten-export) The symbol "Build" needs to be exported by the entry point index.d.ts getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // Warning: (ae-forgotten-export) The symbol "Project" needs to be exported by the entry point index.d.ts getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -82,20 +82,20 @@ export class JenkinsClient implements JenkinsApi { }); // (undocumented) getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; // (undocumented) getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string; }; }): Promise; // (undocumented) retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; diff --git a/plugins/jenkins/src/api/JenkinsApi.ts b/plugins/jenkins/src/api/JenkinsApi.ts index dd169de78b..04bd313227 100644 --- a/plugins/jenkins/src/api/JenkinsApi.ts +++ b/plugins/jenkins/src/api/JenkinsApi.ts @@ -19,7 +19,7 @@ import { DiscoveryApi, IdentityApi, } from '@backstage/core-plugin-api'; -import type { EntityName } from '@backstage/catalog-model'; +import type { CompoundEntityRef } from '@backstage/catalog-model'; import { ResponseError } from '@backstage/errors'; export const jenkinsApiRef = createApiRef({ @@ -80,7 +80,7 @@ export interface JenkinsApi { */ getProjects(options: { /** the entity whose jobs should be retrieved. */ - entity: EntityName; + entity: CompoundEntityRef; /** a filter on jobs. Currently this just takes a branch (and assumes certain structures in jenkins) */ filter: { branch?: string }; }): Promise; @@ -93,13 +93,13 @@ export interface JenkinsApi { * TODO: abstract jobFullName (so we could support differentiating between the same named job on multiple instances). */ getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise; @@ -118,7 +118,7 @@ export class JenkinsClient implements JenkinsApi { } async getProjects(options: { - entity: EntityName; + entity: CompoundEntityRef; filter: { branch?: string }; }): Promise { const { entity, filter } = options; @@ -157,7 +157,7 @@ export class JenkinsClient implements JenkinsApi { } async getBuild(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { @@ -182,7 +182,7 @@ export class JenkinsClient implements JenkinsApi { } async retry(options: { - entity: EntityName; + entity: CompoundEntityRef; jobFullName: string; buildNumber: string; }): Promise { diff --git a/plugins/jenkins/src/components/useBuildWithSteps.ts b/plugins/jenkins/src/components/useBuildWithSteps.ts index 2166b2a1a1..0910564384 100644 --- a/plugins/jenkins/src/components/useBuildWithSteps.ts +++ b/plugins/jenkins/src/components/useBuildWithSteps.ts @@ -19,7 +19,7 @@ import { jenkinsApiRef } from '../api'; import { useAsyncPolling } from './useAsyncPolling'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; const INTERVAL_AMOUNT = 1500; @@ -41,7 +41,7 @@ export function useBuildWithSteps({ const getBuildWithSteps = useCallback(async () => { try { - const entityName = await getEntityName(entity); + const entityName = await getCompoundEntityRef(entity); return api.getBuild({ entity: entityName, jobFullName, buildNumber }); } catch (e) { errorApi.post(e); diff --git a/plugins/jenkins/src/components/useBuilds.ts b/plugins/jenkins/src/components/useBuilds.ts index a4afeb351f..07d23cce5a 100644 --- a/plugins/jenkins/src/components/useBuilds.ts +++ b/plugins/jenkins/src/components/useBuilds.ts @@ -18,7 +18,7 @@ import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { jenkinsApiRef } from '../api'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { useEntity } from '@backstage/plugin-catalog-react'; -import { getEntityName } from '@backstage/catalog-model'; +import { getCompoundEntityRef } from '@backstage/catalog-model'; export enum ErrorType { CONNECTION_ERROR, @@ -33,7 +33,7 @@ export enum ErrorType { */ export function useBuilds({ branch }: { branch?: string } = {}) { const { entity } = useEntity(); - const entityName = getEntityName(entity); + const entityName = getCompoundEntityRef(entity); const api = useApi(jenkinsApiRef); const errorApi = useApi(errorApiRef); @@ -60,7 +60,7 @@ export function useBuilds({ branch }: { branch?: string } = {}) { } = useAsyncRetry(async () => { try { const build = await api.getProjects({ - entity: getEntityName(entity), + entity: getCompoundEntityRef(entity), filter: { branch }, }); diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 2b292cbe8b..cbf517bdca 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -25,7 +25,7 @@ import { import { useApi, useRouteRef } from '@backstage/core-plugin-api'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, isOwnerOf, useEntity, } from '@backstage/plugin-catalog-react'; @@ -108,7 +108,7 @@ const getQueryParams = ( owner: Entity, selectedEntity: EntityTypeProps, ): string => { - const ownerName = formatEntityRefTitle(owner, { defaultKind: 'group' }); + const ownerName = humanizeEntityRef(owner, { defaultKind: 'group' }); const { kind, type } = selectedEntity; const filters = { kind, diff --git a/plugins/permission-common/src/types/permission.ts b/plugins/permission-common/src/types/permission.ts index 805aefa3de..16934d8771 100644 --- a/plugins/permission-common/src/types/permission.ts +++ b/plugins/permission-common/src/types/permission.ts @@ -37,8 +37,22 @@ export type PermissionAttributes = { * @public */ export type Permission = { + /** + * The name of the permission. + */ name: string; + /** + * {@link PermissionAttributes} which describe characteristics of the permission, to help + * policy authors make consistent decisions for similar permissions without referring to them + * all by name. + */ attributes: PermissionAttributes; + /** + * Some permissions can be authorized based on characteristics of a resource + * such a catalog entity. For these permissions, the resourceType field + * denotes the type of the resource whose resourceRef should be passed when + * authorizing. + */ resourceType?: string; }; diff --git a/plugins/permission-node/api-report.md b/plugins/permission-node/api-report.md index 534ad45110..f6de7f4909 100644 --- a/plugins/permission-node/api-report.md +++ b/plugins/permission-node/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AllOfCriteria } from '@backstage/plugin-permission-common'; +import { AnyOfCriteria } from '@backstage/plugin-permission-common'; import { AuthorizeDecision } from '@backstage/plugin-permission-common'; import { AuthorizeQuery } from '@backstage/plugin-permission-common'; import { AuthorizeRequestOptions } from '@backstage/plugin-permission-common'; @@ -11,6 +13,7 @@ import { BackstageIdentityResponse } from '@backstage/plugin-auth-node'; import { Config } from '@backstage/config'; import express from 'express'; import { Identified } from '@backstage/plugin-permission-common'; +import { NotCriteria } from '@backstage/plugin-permission-common'; import { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import { PermissionCondition } from '@backstage/plugin-permission-common'; import { PermissionCriteria } from '@backstage/plugin-permission-common'; @@ -118,6 +121,21 @@ export type DefinitivePolicyDecision = { result: AuthorizeResult.ALLOW | AuthorizeResult.DENY; }; +// @alpha +export const isAndCriteria: ( + criteria: PermissionCriteria, +) => criteria is AllOfCriteria; + +// @alpha +export const isNotCriteria: ( + criteria: PermissionCriteria, +) => criteria is NotCriteria; + +// @alpha +export const isOrCriteria: ( + criteria: PermissionCriteria, +) => criteria is AnyOfCriteria; + // @public export const makeCreatePermissionRule: () => < TParams extends unknown[], diff --git a/plugins/permission-node/src/integration/index.ts b/plugins/permission-node/src/integration/index.ts index 978342e4ed..7702fea95b 100644 --- a/plugins/permission-node/src/integration/index.ts +++ b/plugins/permission-node/src/integration/index.ts @@ -19,3 +19,4 @@ export * from './createConditionExports'; export * from './createConditionTransformer'; export * from './createPermissionIntegrationRouter'; export * from './createPermissionRule'; +export { isAndCriteria, isOrCriteria, isNotCriteria } from './util'; diff --git a/plugins/permission-node/src/integration/util.ts b/plugins/permission-node/src/integration/util.ts index 3878e18895..9102f092c4 100644 --- a/plugins/permission-node/src/integration/util.ts +++ b/plugins/permission-node/src/integration/util.ts @@ -22,20 +22,44 @@ import { } from '@backstage/plugin-permission-common'; import { PermissionRule } from '../types'; -export const isAndCriteria = ( - filter: PermissionCriteria, -): filter is AllOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'allOf'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type allOf, + * narrowing down `criteria` to the specific type. + */ +export const isAndCriteria = ( + criteria: PermissionCriteria, +): criteria is AllOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'allOf'); -export const isOrCriteria = ( - filter: PermissionCriteria, -): filter is AnyOfCriteria => - Object.prototype.hasOwnProperty.call(filter, 'anyOf'); +/** + * Utility function used to parse a PermissionCriteria of type + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type anyOf, + * narrowing down `criteria` to the specific type. + */ +export const isOrCriteria = ( + criteria: PermissionCriteria, +): criteria is AnyOfCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'anyOf'); -export const isNotCriteria = ( - filter: PermissionCriteria, -): filter is NotCriteria => - Object.prototype.hasOwnProperty.call(filter, 'not'); +/** + * Utility function used to parse a PermissionCriteria + * @param criteria - a PermissionCriteria + * @alpha + * + * @returns `true` if the permission criteria is of type not, + * narrowing down `criteria` to the specific type. + */ +export const isNotCriteria = ( + criteria: PermissionCriteria, +): criteria is NotCriteria => + Object.prototype.hasOwnProperty.call(criteria, 'not'); export const createGetRule = ( rules: PermissionRule[], diff --git a/plugins/rollbar/src/hooks/useCatalogEntity.ts b/plugins/rollbar/src/hooks/useCatalogEntity.ts index 468d0e8c50..21d22dd13c 100644 --- a/plugins/rollbar/src/hooks/useCatalogEntity.ts +++ b/plugins/rollbar/src/hooks/useCatalogEntity.ts @@ -30,7 +30,7 @@ export function useCatalogEntity() { error, loading, } = useAsync( - () => catalogApi.getEntityByName({ kind: 'Component', namespace, name }), + () => catalogApi.getEntityByRef({ kind: 'Component', namespace, name }), [catalogApi, namespace, name], ); diff --git a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts index 6b0cb4695d..7575627506 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts +++ b/plugins/scaffolder-backend-module-cookiecutter/src/actions/fetch/cookiecutter.ts @@ -57,23 +57,29 @@ export class CookiecutterRunner { workspacePath, values, logStream, + imageName, + templateDir, + templateContentsDir, }: { workspacePath: string; values: JsonObject; logStream: Writable; + imageName?: string; + templateDir: string; + templateContentsDir: string; }): Promise { - const templateDir = path.join(workspacePath, 'template'); const intermediateDir = path.join(workspacePath, 'intermediate'); await fs.ensureDir(intermediateDir); const resultDir = path.join(workspacePath, 'result'); // First lets grab the default cookiecutter.json file - const cookieCutterJson = await this.fetchTemplateCookieCutter(templateDir); + const cookieCutterJson = await this.fetchTemplateCookieCutter( + templateContentsDir, + ); - const { imageName, ...valuesForCookieCutterJson } = values; const cookieInfo = { ...cookieCutterJson, - ...valuesForCookieCutterJson, + ...values, }; await fs.writeJSON(path.join(templateDir, 'cookiecutter.json'), cookieInfo); @@ -96,7 +102,7 @@ export class CookiecutterRunner { }); } else { await this.containerRunner.runContainer({ - imageName: (imageName as string) ?? 'spotify/backstage-cookiecutter', + imageName: imageName ?? 'spotify/backstage-cookiecutter', command: 'cookiecutter', args: ['--no-input', '-o', '/output', '/input', '--verbose'], mountDirs, @@ -232,14 +238,16 @@ export function createFetchCookiecutterAction(options: { ...ctx.input.values, _copy_without_render: ctx.input.copyWithoutRender, _extensions: ctx.input.extensions, - imageName: ctx.input.imageName, }; // Will execute the template in ./template and put the result in ./result await cookiecutter.run({ workspacePath: workDir, logStream: ctx.logStream, - values, + values: values, + imageName: ctx.input.imageName, + templateDir: templateDir, + templateContentsDir: templateContentsDir, }); // Finally move the template result into the task workspace diff --git a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts index 43559fcd5f..d095b77b4c 100644 --- a/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts +++ b/plugins/scaffolder-backend/src/processor/ScaffolderEntitiesProcessor.ts @@ -16,7 +16,7 @@ import { Entity, - getEntityName, + getCompoundEntityRef, parseEntityRef, RELATION_OWNED_BY, RELATION_OWNER_OF, @@ -25,7 +25,7 @@ import { CatalogProcessor, CatalogProcessorEmit, LocationSpec, - results, + processingResult, } from '@backstage/plugin-catalog-backend'; import { TemplateEntityV1beta3, @@ -55,7 +55,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { _location: LocationSpec, emit: CatalogProcessorEmit, ): Promise { - const selfRef = getEntityName(entity); + const selfRef = getCompoundEntityRef(entity); if ( entity.apiVersion === 'scaffolder.backstage.io/v1beta3' && @@ -70,7 +70,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { defaultNamespace: selfRef.namespace, }); emit( - results.relation({ + processingResult.relation({ source: selfRef, type: RELATION_OWNED_BY, target: { @@ -81,7 +81,7 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor { }), ); emit( - results.relation({ + processingResult.relation({ source: { kind: targetRef.kind, namespace: targetRef.namespace, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts index 445a494234..665c54fb3e 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/bitbucket.ts @@ -33,6 +33,7 @@ const createBitbucketCloudRepository = async (opts: { repoVisibility: 'private' | 'public'; mainBranch: string; authorization: string; + apiBaseUrl: string; }) => { const { workspace, @@ -42,6 +43,7 @@ const createBitbucketCloudRepository = async (opts: { repoVisibility, mainBranch, authorization, + apiBaseUrl, } = opts; const options: RequestInit = { @@ -61,7 +63,7 @@ const createBitbucketCloudRepository = async (opts: { let response: Response; try { response = await fetch( - `https://api.bitbucket.org/2.0/repositories/${workspace}/${repo}`, + `${apiBaseUrl}/repositories/${workspace}/${repo}`, options, ); } catch (e) { @@ -91,16 +93,14 @@ const createBitbucketCloudRepository = async (opts: { }; const createBitbucketServerRepository = async (opts: { - host: string; project: string; repo: string; description?: string; repoVisibility: 'private' | 'public'; authorization: string; - apiBaseUrl?: string; + apiBaseUrl: string; }) => { const { - host, project, repo, description, @@ -124,8 +124,7 @@ const createBitbucketServerRepository = async (opts: { }; try { - const baseUrl = apiBaseUrl ? apiBaseUrl : `https://${host}/rest/api/1.0`; - response = await fetch(`${baseUrl}/projects/${project}/repos`, options); + response = await fetch(`${apiBaseUrl}/projects/${project}/repos`, options); } catch (e) { throw new Error(`Unable to create repository, ${e}`); } @@ -314,7 +313,11 @@ export function createPublishBitbucketAction(options: { const authorization = getAuthorizationHeader( ctx.input.token - ? { host: integrationConfig.config.host, token: ctx.input.token } + ? { + host: integrationConfig.config.host, + apiBaseUrl: integrationConfig.config.apiBaseUrl, + token: ctx.input.token, + } : integrationConfig.config, ); @@ -327,7 +330,6 @@ export function createPublishBitbucketAction(options: { const { remoteUrl, repoContentsUrl } = await createMethod({ authorization, - host, workspace: workspace || '', project, repo, diff --git a/plugins/scaffolder-backend/src/service/helpers.ts b/plugins/scaffolder-backend/src/service/helpers.ts index e49c28cb74..a7148dac40 100644 --- a/plugins/scaffolder-backend/src/service/helpers.ts +++ b/plugins/scaffolder-backend/src/service/helpers.ts @@ -20,8 +20,9 @@ import { ANNOTATION_LOCATION, parseLocationRef, ANNOTATION_SOURCE_LOCATION, - EntityName, + CompoundEntityRef, DEFAULT_NAMESPACE, + stringifyEntityRef, } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { assertError, InputError, NotFoundError } from '@backstage/errors'; @@ -91,7 +92,7 @@ export function getEntityBaseUrl(entity: Entity): string | undefined { * Returns the matching template, or throws a NotFoundError if no such template existed. */ export async function findTemplate(options: { - entityRef: EntityName; + entityRef: CompoundEntityRef; token?: string; catalogApi: CatalogApi; }): Promise { @@ -106,9 +107,11 @@ export async function findTemplate(options: { throw new InputError(`Invalid kind, only 'Template' kind is supported`); } - const template = await catalogApi.getEntityByName(entityRef, { token }); + const template = await catalogApi.getEntityByRef(entityRef, { token }); if (!template) { - throw new NotFoundError(`Template ${entityRef} not found`); + throw new NotFoundError( + `Template ${stringifyEntityRef(entityRef)} not found`, + ); } return template as TemplateEntityV1beta3 | TemplateEntityV1beta2; diff --git a/plugins/scaffolder-backend/src/service/router.test.ts b/plugins/scaffolder-backend/src/service/router.test.ts index fa1b649231..4041e41cf8 100644 --- a/plugins/scaffolder-backend/src/service/router.test.ts +++ b/plugins/scaffolder-backend/src/service/router.test.ts @@ -53,7 +53,7 @@ import { stringifyEntityRef } from '@backstage/catalog-model'; const createCatalogClient = (template: any) => ({ - getEntityByName: async () => template, + getEntityByRef: async () => template, } as unknown as CatalogApi); function createDatabase(): PluginDatabaseManager { diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 82dfecfaa5..b51f18fcc9 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -8,7 +8,6 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; -import { ComponentProps } from 'react'; import { ComponentType } from 'react'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; @@ -17,7 +16,6 @@ import { ExternalRouteRef } from '@backstage/core-plugin-api'; import { FetchApi } from '@backstage/core-plugin-api'; import { FieldProps } from '@rjsf/core'; import { FieldValidation } from '@rjsf/core'; -import { IconButton } from '@material-ui/core'; import { JsonObject } from '@backstage/types'; import { JSONSchema7 } from 'json-schema'; import { JsonValue } from '@backstage/types'; @@ -28,15 +26,13 @@ import { ScmIntegrationRegistry } from '@backstage/integration'; import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { TemplateEntityV1beta2 } from '@backstage/plugin-scaffolder-common'; -// Warning: (ae-missing-release-tag) "createScaffolderFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, >( options: FieldExtensionOptions, -): Extension<() => null>; +): Extension>; // @public export type CustomFieldValidator = ( @@ -50,19 +46,18 @@ export type CustomFieldValidator = ( // Warning: (ae-missing-release-tag) "EntityNamePickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const EntityNamePickerFieldExtension: () => null; - -// Warning: (ae-missing-release-tag) "EntityPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const EntityPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; +export const EntityNamePickerFieldExtension: FieldExtensionComponent< + string, + {} +>; // Warning: (ae-missing-release-tag) "EntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const EntityPickerFieldExtension: () => null; +export const EntityPickerFieldExtension: FieldExtensionComponent< + string, + EntityPickerUiOptions +>; // Warning: (ae-missing-release-tag) "EntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -76,15 +71,11 @@ export interface EntityPickerUiOptions { defaultKind?: string; } -// Warning: (ae-missing-release-tag) "EntityTagsPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public -export const EntityTagsPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - -// @public -export const EntityTagsPickerFieldExtension: () => null; +export const EntityTagsPickerFieldExtension: FieldExtensionComponent< + string[], + EntityTagsPickerUiOptions +>; // Warning: (ae-missing-release-tag) "EntityTagsPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -94,11 +85,8 @@ export interface EntityTagsPickerUiOptions { kinds?: string[]; } -// Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts -// Warning: (ae-missing-release-tag) "FavouriteTemplate" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public -export const FavouriteTemplate: (props: Props) => JSX.Element; +export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; // @public export interface FieldExtensionComponentProps< @@ -114,10 +102,12 @@ export interface FieldExtensionComponentProps< // @public export type FieldExtensionOptions< TFieldReturnValue = unknown, - TProps = FieldProps, + TInputProps = unknown, > = { name: string; - component: (props: TProps) => JSX.Element | null; + component: ( + props: FieldExtensionComponentProps, + ) => JSX.Element | null; validation?: CustomFieldValidator; }; @@ -153,17 +143,13 @@ export type LogEvent = { taskId: string; }; -// Warning: (ae-missing-release-tag) "OwnedEntityPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const OwnedEntityPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "OwnedEntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const OwnedEntityPickerFieldExtension: () => null; +export const OwnedEntityPickerFieldExtension: FieldExtensionComponent< + string, + OwnedEntityPickerUiOptions +>; // Warning: (ae-missing-release-tag) "OwnedEntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -175,17 +161,13 @@ export interface OwnedEntityPickerUiOptions { defaultKind?: string; } -// Warning: (ae-missing-release-tag) "OwnerPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const OwnerPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "OwnerPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const OwnerPickerFieldExtension: () => null; +export const OwnerPickerFieldExtension: FieldExtensionComponent< + string, + OwnerPickerUiOptions +>; // Warning: (ae-missing-release-tag) "OwnerPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -206,17 +188,13 @@ export const repoPickerValidation: ( }, ) => void; -// Warning: (ae-missing-release-tag) "RepoUrlPicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public -export const RepoUrlPicker: ( - props: FieldExtensionComponentProps, -) => JSX.Element; - // Warning: (ae-missing-release-tag) "RepoUrlPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const RepoUrlPickerFieldExtension: () => null; +export const RepoUrlPickerFieldExtension: FieldExtensionComponent< + string, + RepoUrlPickerUiOptions +>; // Warning: (ae-missing-release-tag) "RepoUrlPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -313,9 +291,7 @@ export class ScaffolderClient implements ScaffolderApi { streamLogs(options: ScaffolderStreamLogsOptions): Observable; } -// Warning: (ae-missing-release-tag) "ScaffolderFieldExtensions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const ScaffolderFieldExtensions: React_2.ComponentType; // Warning: (ae-missing-release-tag) "ScaffolderGetIntegrationsListOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/plugins/scaffolder/dev/index.tsx b/plugins/scaffolder/dev/index.tsx index 78e6254e09..03632f434d 100644 --- a/plugins/scaffolder/dev/index.tsx +++ b/plugins/scaffolder/dev/index.tsx @@ -20,16 +20,12 @@ import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { catalogApiRef, starredEntitiesApiRef, - DefaultStarredEntitiesApi, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import React from 'react'; import { scaffolderApiRef, ScaffolderClient } from '../src'; import { ScaffolderPage } from '../src/plugin'; -import { - discoveryApiRef, - fetchApiRef, - storageApiRef, -} from '@backstage/core-plugin-api'; +import { discoveryApiRef, fetchApiRef } from '@backstage/core-plugin-api'; import { CatalogEntityPage } from '@backstage/plugin-catalog'; createDevApp() @@ -44,8 +40,8 @@ createDevApp() }) .registerApi({ api: starredEntitiesApiRef, - deps: { storageApi: storageApiRef }, - factory: ({ storageApi }) => new DefaultStarredEntitiesApi({ storageApi }), + deps: {}, + factory: () => new MockStarredEntitiesApi(), }) .registerApi({ api: scaffolderApiRef, diff --git a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx b/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx deleted file mode 100644 index 6e8fe45b2f..0000000000 --- a/plugins/scaffolder/src/components/FavouriteTemplate/FavouriteTemplate.tsx +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Entity } from '@backstage/catalog-model'; -import { useStarredEntity } from '@backstage/plugin-catalog-react'; -import { IconButton, makeStyles, Tooltip, withStyles } from '@material-ui/core'; -import Star from '@material-ui/icons/Star'; -import StarBorder from '@material-ui/icons/StarBorder'; -import React, { ComponentProps } from 'react'; - -type Props = ComponentProps & { entity: Entity }; - -const YellowStar = withStyles({ - root: { - color: '#f3ba37', - }, -})(Star); - -const WhiteBorderStar = withStyles({ - root: { - color: '#ffffff', - }, -})(StarBorder); - -const useStyles = makeStyles(theme => ({ - starButton: { - position: 'absolute', - top: theme.spacing(0.5), - right: theme.spacing(0.5), - padding: '0.25rem', - }, -})); - -export const favouriteTemplateTooltip = (isStarred: boolean) => - isStarred ? 'Remove from favorites' : 'Add to favorites'; - -export const favouriteTemplateIcon = (isStarred: boolean) => - isStarred ? : ; - -/** - * IconButton for showing if a current entity is starred and adding/removing it from the favourite entities - * @param props - MaterialUI IconButton props extended by required `entity` prop - */ -export const FavouriteTemplate = (props: Props) => { - const classes = useStyles(); - const { toggleStarredEntity, isStarredEntity } = useStarredEntity( - props.entity, - ); - return ( - toggleStarredEntity()} - > - - {favouriteTemplateIcon(isStarredEntity)} - - - ); -}; diff --git a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx index f4710427c2..3a7691dd5a 100644 --- a/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx +++ b/plugins/scaffolder/src/components/TemplateCard/TemplateCard.tsx @@ -21,6 +21,7 @@ import { } from '@backstage/integration-react'; import { EntityRefLinks, + FavoriteEntity, getEntityRelations, getEntitySourceLocation, } from '@backstage/plugin-catalog-react'; @@ -42,7 +43,6 @@ import { import WarningIcon from '@material-ui/icons/Warning'; import React from 'react'; import { selectedTemplateRouteRef } from '../../routes'; -import { FavouriteTemplate } from '../FavouriteTemplate/FavouriteTemplate'; import { Button, ItemCardHeader } from '@backstage/core-components'; import { useApi, useRouteRef } from '@backstage/core-plugin-api'; @@ -74,6 +74,12 @@ const useStyles = makeStyles(theme => ({ leftButton: { marginRight: 'auto', }, + starButton: { + position: 'absolute', + top: theme.spacing(0.5), + right: theme.spacing(0.5), + padding: '0.25rem', + }, })); const useDeprecationStyles = makeStyles(theme => ({ @@ -159,7 +165,7 @@ export const TemplateCard = ({ template, deprecated }: TemplateCardProps) => { return ( - + {deprecated && } []; }) => { const apiHolder = useApiHolder(); const secretsContext = useContext(SecretsContext); diff --git a/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts b/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts index 0ba88d1277..7076221480 100644 --- a/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityNamePicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityNamePicker } from './EntityNamePicker'; export { entityNamePickerValidation } from './validation'; diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index 31c44bfe97..4a2f212fee 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -16,7 +16,7 @@ import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef, - formatEntityRefTitle, + humanizeEntityRef, } from '@backstage/plugin-catalog-react'; import { TextField } from '@material-ui/core'; import FormControl from '@material-ui/core/FormControl'; @@ -58,7 +58,7 @@ export const EntityPicker = ( ); const entityRefs = entities?.items.map(e => - formatEntityRefTitle(e, { defaultKind }), + humanizeEntityRef(e, { defaultKind }), ); const onSelect = useCallback( diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/index.ts b/plugins/scaffolder/src/components/fields/EntityPicker/index.ts index eb7511bbef..891b5bef16 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityPicker } from './EntityPicker'; export type { EntityPickerUiOptions } from './EntityPicker'; diff --git a/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts b/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts index 3ba36a2409..9ff6e553a6 100644 --- a/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/EntityTagsPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { EntityTagsPicker } from './EntityTagsPicker'; export type { EntityTagsPickerUiOptions } from './EntityTagsPicker'; diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx index 80f63efaf9..aded9acffa 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import { - formatEntityRefTitle, + humanizeEntityRef, useOwnedEntities, } from '@backstage/plugin-catalog-react'; import { TextField } from '@material-ui/core'; @@ -50,7 +50,7 @@ export const OwnedEntityPicker = ( const { ownedEntities, loading } = useOwnedEntities(allowedKinds); const entityRefs = ownedEntities?.items - .map(e => formatEntityRefTitle(e, { defaultKind })) + .map(e => humanizeEntityRef(e, { defaultKind })) .filter(n => n); const onSelect = (_: any, value: string | null) => { diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts index 125f6fdef0..2988ba8cdc 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { OwnedEntityPicker } from './OwnedEntityPicker'; export type { OwnedEntityPickerUiOptions } from './OwnedEntityPicker'; diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx index 9012ed149e..1824275380 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ import React from 'react'; -import { EntityPicker } from '../EntityPicker'; +import { EntityPicker } from '../EntityPicker/EntityPicker'; import { FieldExtensionComponentProps } from '../../../extensions'; export interface OwnerPickerUiOptions { diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts b/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts index b8cb97eb97..aa26024b5b 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { OwnerPicker } from './OwnerPicker'; export type { OwnerPickerUiOptions } from './OwnerPicker'; diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts index 2fdbf0aac9..c5f596a786 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/index.ts @@ -13,6 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { RepoUrlPicker } from './RepoUrlPicker'; export type { RepoUrlPickerUiOptions } from './RepoUrlPicker'; export { repoPickerValidation } from './validation'; diff --git a/plugins/scaffolder/src/components/index.ts b/plugins/scaffolder/src/components/index.ts index 030c485fd7..47540ff492 100644 --- a/plugins/scaffolder/src/components/index.ts +++ b/plugins/scaffolder/src/components/index.ts @@ -15,7 +15,6 @@ */ export * from './fields'; export type { RepoUrlPickerUiOptions } from './fields'; -export { FavouriteTemplate } from './FavouriteTemplate'; export { TemplateList } from './TemplateList'; export type { TemplateListProps } from './TemplateList'; export { TemplateTypePicker } from './TemplateTypePicker'; diff --git a/plugins/scaffolder/src/extensions/default.ts b/plugins/scaffolder/src/extensions/default.ts index 10c0746e41..8081dcb0ca 100644 --- a/plugins/scaffolder/src/extensions/default.ts +++ b/plugins/scaffolder/src/extensions/default.ts @@ -13,46 +13,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityPicker } from '../components/fields/EntityPicker'; -import { - EntityNamePicker, - entityNamePickerValidation, -} from '../components/fields/EntityNamePicker'; -import { EntityTagsPicker } from '../components/fields/EntityTagsPicker'; -import { OwnerPicker } from '../components/fields/OwnerPicker'; -import { - repoPickerValidation, - RepoUrlPicker, -} from '../components/fields/RepoUrlPicker'; -import { FieldExtensionOptions } from './types'; -import { OwnedEntityPicker } from '../components/fields/OwnedEntityPicker'; +import { EntityPicker } from '../components/fields/EntityPicker/EntityPicker'; +import { EntityNamePicker } from '../components/fields/EntityNamePicker/EntityNamePicker'; +import { entityNamePickerValidation } from '../components/fields/EntityNamePicker/validation'; +import { EntityTagsPicker } from '../components/fields/EntityTagsPicker/EntityTagsPicker'; +import { OwnerPicker } from '../components/fields/OwnerPicker/OwnerPicker'; +import { RepoUrlPicker } from '../components/fields/RepoUrlPicker/RepoUrlPicker'; +import { repoPickerValidation } from '../components/fields/RepoUrlPicker/validation'; +import { OwnedEntityPicker } from '../components/fields/OwnedEntityPicker/OwnedEntityPicker'; -export const DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS: FieldExtensionOptions[] = - [ - { - component: EntityPicker, - name: 'EntityPicker', - }, - { - component: EntityNamePicker, - name: 'EntityNamePicker', - validation: entityNamePickerValidation, - }, - { - component: EntityTagsPicker, - name: 'EntityTagsPicker', - }, - { - component: RepoUrlPicker, - name: 'RepoUrlPicker', - validation: repoPickerValidation, - }, - { - component: OwnerPicker, - name: 'OwnerPicker', - }, - { - component: OwnedEntityPicker, - name: 'OwnedEntityPicker', - }, - ]; +export const DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS = [ + { + component: EntityPicker, + name: 'EntityPicker', + }, + { + component: EntityNamePicker, + name: 'EntityNamePicker', + validation: entityNamePickerValidation, + }, + { + component: EntityTagsPicker, + name: 'EntityTagsPicker', + }, + { + component: RepoUrlPicker, + name: 'RepoUrlPicker', + validation: repoPickerValidation, + }, + { + component: OwnerPicker, + name: 'OwnerPicker', + }, + { + component: OwnedEntityPicker, + name: 'OwnedEntityPicker', + }, +]; diff --git a/plugins/scaffolder/src/extensions/index.tsx b/plugins/scaffolder/src/extensions/index.tsx index 53db4cd927..60b9022053 100644 --- a/plugins/scaffolder/src/extensions/index.tsx +++ b/plugins/scaffolder/src/extensions/index.tsx @@ -25,14 +25,24 @@ import { Extension, attachComponentData } from '@backstage/core-plugin-api'; export const FIELD_EXTENSION_WRAPPER_KEY = 'scaffolder.extensions.wrapper.v1'; export const FIELD_EXTENSION_KEY = 'scaffolder.extensions.field.v1'; +/** + * A type used to wrap up the FieldExtension to embed the ReturnValue and the InputProps + * + * @public + */ +export type FieldExtensionComponent<_TReturnValue, _TInputProps> = () => null; + +/** + * Method for creating field extensions that can be used in the scaffolder + * frontend form. + * @public + */ export function createScaffolderFieldExtension< TReturnValue = unknown, TInputProps = unknown, >( options: FieldExtensionOptions, - // TODO: need know how to embed these types nicely so the api report looks nice. - // then we can remove the export of the components -): Extension<() => null> { +): Extension> { return { expose() { const FieldExtensionDataHolder: any = () => null; @@ -48,6 +58,11 @@ export function createScaffolderFieldExtension< }; } +/** + * The Wrapping component for defining fields extensions inside + * + * @public + */ export const ScaffolderFieldExtensions: React.ComponentType = (): JSX.Element | null => null; diff --git a/plugins/scaffolder/src/extensions/types.ts b/plugins/scaffolder/src/extensions/types.ts index 574ce7ea64..be8dc14b0a 100644 --- a/plugins/scaffolder/src/extensions/types.ts +++ b/plugins/scaffolder/src/extensions/types.ts @@ -35,10 +35,12 @@ export type CustomFieldValidator = ( */ export type FieldExtensionOptions< TFieldReturnValue = unknown, - TProps = FieldProps, + TInputProps = unknown, > = { name: string; - component: (props: TProps) => JSX.Element | null; + component: ( + props: FieldExtensionComponentProps, + ) => JSX.Element | null; validation?: CustomFieldValidator; }; diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 71cd1d8f84..f33cf289f4 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -44,6 +44,7 @@ export type { CustomFieldValidator, FieldExtensionOptions, FieldExtensionComponentProps, + FieldExtensionComponent, } from './extensions'; export { EntityPickerFieldExtension, diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index ccf8d2ee34..0d44a40c80 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -16,16 +16,12 @@ import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { scaffolderApiRef, ScaffolderClient } from './api'; -import { EntityPicker } from './components/fields/EntityPicker'; -import { - entityNamePickerValidation, - EntityNamePicker, -} from './components/fields/EntityNamePicker'; -import { OwnerPicker } from './components/fields/OwnerPicker'; -import { - repoPickerValidation, - RepoUrlPicker, -} from './components/fields/RepoUrlPicker'; +import { EntityPicker } from './components/fields/EntityPicker/EntityPicker'; +import { entityNamePickerValidation } from './components/fields/EntityNamePicker'; +import { EntityNamePicker } from './components/fields/EntityNamePicker/EntityNamePicker'; +import { OwnerPicker } from './components/fields/OwnerPicker/OwnerPicker'; +import { repoPickerValidation } from './components/fields/RepoUrlPicker'; +import { RepoUrlPicker } from './components/fields/RepoUrlPicker/RepoUrlPicker'; import { createScaffolderFieldExtension } from './extensions'; import { registerComponentRouteRef, rootRouteRef } from './routes'; import { @@ -35,8 +31,8 @@ import { discoveryApiRef, fetchApiRef, } from '@backstage/core-plugin-api'; -import { OwnedEntityPicker } from './components/fields/OwnedEntityPicker'; -import { EntityTagsPicker } from './components/fields/EntityTagsPicker'; +import { OwnedEntityPicker } from './components/fields/OwnedEntityPicker/OwnedEntityPicker'; +import { EntityTagsPicker } from './components/fields/EntityTagsPicker/EntityTagsPicker'; export const scaffolderPlugin = createPlugin({ id: 'scaffolder', diff --git a/plugins/search-backend-module-elasticsearch/api-report.md b/plugins/search-backend-module-elasticsearch/api-report.md index 6dc0256586..d4edcc6b25 100644 --- a/plugins/search-backend-module-elasticsearch/api-report.md +++ b/plugins/search-backend-module-elasticsearch/api-report.md @@ -5,6 +5,8 @@ ```ts /// +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; import { Config } from '@backstage/config'; import type { ConnectionOptions } from 'tls'; import { IndexableDocument } from '@backstage/search-common'; @@ -114,7 +116,7 @@ export class ElasticSearchSearchEngine implements SearchEngine { indexPrefix, }: ElasticSearchOptions): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; newClient(create: (options: ElasticSearchClientOptions) => T): T; // (undocumented) query(query: SearchQuery): Promise; @@ -127,4 +129,31 @@ export class ElasticSearchSearchEngine implements SearchEngine { // (undocumented) protected translator(query: SearchQuery): ConcreteElasticSearchQuery; } + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: ElasticSearchSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + readonly indexName: string; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "ElasticSearchSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger_2; + elasticSearchClient: Client; +}; ``` diff --git a/plugins/search-backend-module-elasticsearch/package.json b/plugins/search-backend-module-elasticsearch/package.json index 2a2dbd6f85..0a279d6b4c 100644 --- a/plugins/search-backend-module-elasticsearch/package.json +++ b/plugins/search-backend-module-elasticsearch/package.json @@ -25,6 +25,7 @@ "dependencies": { "@backstage/config": "^0.1.15", "@backstage/search-common": "^0.2.4", + "@backstage/plugin-search-backend-node": "^0.4.7", "@elastic/elasticsearch": "7.13.0", "@acuris/aws-es-connection": "^2.2.0", "aws-sdk": "^2.948.0", diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts index 1566a35b57..467e1b4e77 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.test.ts @@ -15,7 +15,8 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { Client } from '@elastic/elasticsearch'; +import { ConfigReader } from '@backstage/config'; +import { Client, errors } from '@elastic/elasticsearch'; import Mock from '@elastic/elasticsearch-mock'; import { ConcreteElasticSearchQuery, @@ -23,7 +24,7 @@ import { ElasticSearchSearchEngine, encodePageCursor, } from './ElasticSearchSearchEngine'; -import { ConfigReader } from '@backstage/config'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; class ElasticSearchSearchEngineForTranslatorTests extends ElasticSearchSearchEngine { getTranslator() { @@ -37,6 +38,16 @@ const options = { Connection: mock.getConnection(), }; +const indexerMock = { + on: jest.fn(), + indexName: 'expected-index-name', +}; +jest.mock('./ElasticSearchSearchEngineIndexer', () => ({ + ElasticSearchSearchEngineIndexer: jest + .fn() + .mockImplementation(() => indexerMock), +})); + describe('ElasticSearchSearchEngine', () => { let testSearchEngine: ElasticSearchSearchEngine; let inspectableSearchEngine: ElasticSearchSearchEngineForTranslatorTests; @@ -542,23 +553,67 @@ describe('ElasticSearchSearchEngine', () => { }); }); - describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + describe('indexer', () => { + it('should get indexer', async () => { + const indexer = await testSearchEngine.getIndexer('test-index'); - // call index func and ensure the index func was invoked. - await testSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + expect(indexer).toStrictEqual(indexerMock); + expect(ElasticSearchSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + alias: 'test-index__search', + type: 'test-index', + indexPrefix: '', + indexSeparator: '-index__', + elasticSearchClient: client, + }), + ); + expect(indexerMock.on).toHaveBeenCalledWith( + 'error', + expect.any(Function), + ); + }); + + describe('onError', () => { + let errorHandler: Function; + const error = new Error('some error'); + + beforeEach(async () => { + mock.clearAll(); + await testSearchEngine.getIndexer('test-index'); + errorHandler = indexerMock.on.mock.calls[0][1]; + }); + + it('should check for and delete expected index', async () => { + const existsSpy = jest.fn().mockReturnValue('truthy value'); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check and delete HTTP requests were made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('should not delete index if none exists', async () => { + // Exists call returns 404 on no index. + const existsSpy = jest.fn().mockReturnValue( + new errors.ResponseError({ + statusCode: 404, + body: { status: 404 }, + } as unknown as any), + ); + const deleteSpy = jest.fn().mockReturnValue({}); + mock.add({ method: 'HEAD', path: '/expected-index-name' }, existsSpy); + mock.add({ method: 'DELETE', path: '/expected-index-name' }, deleteSpy); + + await errorHandler(error); + + // Check request was made, but no delete request was made. + expect(existsSpy).toHaveBeenCalled(); + expect(deleteSpy).not.toHaveBeenCalled(); + }); }); }); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts index 48caf0d4f1..c16567fee0 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngine.ts @@ -29,8 +29,8 @@ import { Client } from '@elastic/elasticsearch'; import esb from 'elastic-builder'; import { isEmpty, isNaN as nan, isNumber } from 'lodash'; import { Logger } from 'winston'; - import type { ElasticSearchClientOptions } from './ElasticSearchClientOptions'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; export type { ElasticSearchClientOptions }; @@ -58,12 +58,6 @@ type ElasticSearchResult = { _source: IndexableDocument; }; -function duration(startTimestamp: [number, number]): string { - const delta = process.hrtime(startTimestamp); - const seconds = delta[0] + delta[1] / 1e9; - return `${seconds.toFixed(1)}s`; -} - function isBlank(str: string) { return (isEmpty(str) && !isNumber(str)) || nan(str); } @@ -165,67 +159,37 @@ export class ElasticSearchSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - this.logger.info( - `Started indexing ${documents.length} documents for index ${type}`, - ); - const startTimestamp = process.hrtime(); + async getIndexer(type: string) { const alias = this.constructSearchAlias(type); - const index = this.constructIndexName(type, `${Date.now()}`); - try { - const aliases = await this.elasticSearchClient.cat.aliases({ - format: 'json', - name: alias, - }); - const removableIndices = aliases.body.map( - (r: Record) => r.index, - ); + const indexer = new ElasticSearchSearchEngineIndexer({ + type, + indexPrefix: this.indexPrefix, + indexSeparator: this.indexSeparator, + alias, + elasticSearchClient: this.elasticSearchClient, + logger: this.logger, + }); - await this.elasticSearchClient.indices.create({ - index, - }); - const result = await this.elasticSearchClient.helpers.bulk({ - datasource: documents, - onDocument() { - return { - index: { _index: index }, - }; - }, - refreshOnCompletion: index, - }); - - this.logger.info( - `Indexing completed for index ${type} in ${duration(startTimestamp)}`, - result, - ); - await this.elasticSearchClient.indices.updateAliases({ - body: { - actions: [ - { remove: { index: this.constructIndexName(type, '*'), alias } }, - { add: { index, alias } }, - ], - }, - }); - - this.logger.info('Removing stale search indices', removableIndices); - if (removableIndices.length) { - await this.elasticSearchClient.indices.delete({ - index: removableIndices, - }); - } - } catch (e) { + // Attempt cleanup upon failure. + indexer.on('error', async e => { this.logger.error(`Failed to index documents for type ${type}`, e); - const response = await this.elasticSearchClient.indices.exists({ - index, - }); - const indexCreated = response.body; - if (indexCreated) { - this.logger.info(`Removing created index ${index}`); - await this.elasticSearchClient.indices.delete({ - index, + try { + const response = await this.elasticSearchClient.indices.exists({ + index: indexer.indexName, }); + const indexCreated = response.body; + if (indexCreated) { + this.logger.info(`Removing created index ${indexer.indexName}`); + await this.elasticSearchClient.indices.delete({ + index: indexer.indexName, + }); + } + } catch (error) { + this.logger.error(`Unable to clean up elastic index: ${error}`); } - } + }); + + return indexer; } async query(query: SearchQuery): Promise { @@ -268,10 +232,6 @@ export class ElasticSearchSearchEngine implements SearchEngine { private readonly indexSeparator = '-index__'; - private constructIndexName(type: string, postFix: string) { - return `${this.indexPrefix}${type}${this.indexSeparator}${postFix}`; - } - private getTypeFromIndex(index: string) { return index .substring(this.indexPrefix.length) diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..0867887d9a --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.test.ts @@ -0,0 +1,211 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { Client } from '@elastic/elasticsearch'; +import Mock from '@elastic/elasticsearch-mock'; +import { range } from 'lodash'; +import { ElasticSearchSearchEngineIndexer } from './ElasticSearchSearchEngineIndexer'; + +const mock = new Mock(); +const client = new Client({ + node: 'http://localhost:9200', + Connection: mock.getConnection(), +}); + +describe('ElasticSearchSearchEngineIndexer', () => { + let indexer: ElasticSearchSearchEngineIndexer; + let bulkSpy: jest.Mock; + let catSpy: jest.Mock; + let createSpy: jest.Mock; + let aliasesSpy: jest.Mock; + let deleteSpy: jest.Mock; + + beforeEach(() => { + // Instantiate the indexer to be tested. + indexer = new ElasticSearchSearchEngineIndexer({ + type: 'some-type', + indexPrefix: '', + indexSeparator: '-index__', + alias: 'some-type-index__search', + logger: getVoidLogger(), + elasticSearchClient: client, + }); + + // Set up all requisite Elastic mocks. + mock.clearAll(); + bulkSpy = jest.fn().mockReturnValue({ took: 9, errors: false, items: [] }); + mock.add( + { + method: 'POST', + path: '/_bulk', + }, + bulkSpy, + ); + mock.add( + { + method: 'GET', + path: '/:index/_refresh', + }, + jest.fn().mockReturnValue({}), + ); + + catSpy = jest.fn().mockReturnValue([ + { + alias: 'some-type-index__search', + index: 'some-type-index__123tobedeleted', + filter: '-', + 'routing.index': '-', + 'routing.search': '-', + is_write_index: '-', + }, + ]); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + createSpy = jest.fn().mockReturnValue({ + acknowledged: true, + shards_acknowledged: true, + index: 'single_index', + }); + mock.add( + { + method: 'PUT', + path: '/:index', + }, + createSpy, + ); + + aliasesSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'POST', + path: '*', + }, + aliasesSpy, + ); + + deleteSpy = jest.fn().mockReturnValue({}); + mock.add( + { + method: 'DELETE', + path: '/some-type-index__123tobedeleted', + }, + deleteSpy, + ); + }); + + it('indexes documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + { + title: 'Another test', + text: 'Some more text', + location: 'test/location/2', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Older indices should have been queried for. + expect(catSpy).toHaveBeenCalled(); + + // A new index should have been created. + const createdIndex = createSpy.mock.calls[0][0].path.slice(1); + expect(createdIndex).toContain('some-type-index__'); + + // Bulk helper should have been called with documents. + const bulkBody = bulkSpy.mock.calls[0][0].body; + expect(bulkBody[0]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[1]).toStrictEqual(documents[0]); + expect(bulkBody[2]).toStrictEqual({ index: { _index: createdIndex } }); + expect(bulkBody[3]).toStrictEqual(documents[1]); + + // Alias should have been rotated. + expect(aliasesSpy).toHaveBeenCalled(); + const aliasActions = aliasesSpy.mock.calls[0][0].body.actions; + expect(aliasActions[0]).toStrictEqual({ + remove: { index: 'some-type-index__*', alias: 'some-type-index__search' }, + }); + expect(aliasActions[1]).toStrictEqual({ + add: { index: createdIndex, alias: 'some-type-index__search' }, + }); + + // Old index should be cleaned up. + expect(deleteSpy).toHaveBeenCalled(); + }); + + it('handles bulk and batching during indexing', async () => { + const documents = range(550).map(i => ({ + title: `Hello World ${i}`, + location: `location-${i}`, + // Generate large document sizes to trigger ES bulk flushing. + text: range(2000).join(', '), + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Ensure multiple bulk requests were made. + expect(bulkSpy).toHaveBeenCalledTimes(2); + + // Ensure the first and last documents were included in the payloads. + const docLocations: string[] = [ + ...bulkSpy.mock.calls[0][0].body.map((l: any) => l.location), + ...bulkSpy.mock.calls[1][0].body.map((l: any) => l.location), + ]; + expect(docLocations).toContain('location-0'); + expect(docLocations).toContain('location-549'); + }); + + it('ignores cleanup when no existing indices exist', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + // Update initial alias cat to return nothing. + catSpy = jest.fn().mockReturnValue([]); + mock.clear({ + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }); + mock.add( + { + method: 'GET', + path: '/_cat/aliases/some-type-index__search', + }, + catSpy, + ); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Final deletion shouldn't be called. + expect(deleteSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts new file mode 100644 index 0000000000..2e4996cd2f --- /dev/null +++ b/plugins/search-backend-module-elasticsearch/src/engines/ElasticSearchSearchEngineIndexer.ts @@ -0,0 +1,176 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Client } from '@elastic/elasticsearch'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +export type ElasticSearchSearchEngineIndexerOptions = { + type: string; + indexPrefix: string; + indexSeparator: string; + alias: string; + logger: Logger; + elasticSearchClient: Client; +}; + +function duration(startTimestamp: [number, number]): string { + const delta = process.hrtime(startTimestamp); + const seconds = delta[0] + delta[1] / 1e9; + return `${seconds.toFixed(1)}s`; +} + +export class ElasticSearchSearchEngineIndexer extends BatchSearchEngineIndexer { + private received: number = 0; + private processed: number = 0; + private removableIndices: string[] = []; + + private readonly startTimestamp: [number, number]; + private readonly type: string; + public readonly indexName: string; + private readonly indexPrefix: string; + private readonly indexSeparator: string; + private readonly alias: string; + private readonly logger: Logger; + private readonly sourceStream: Readable; + private readonly elasticSearchClient: Client; + private bulkResult: Promise; + + constructor(options: ElasticSearchSearchEngineIndexerOptions) { + super({ batchSize: 100 }); + this.logger = options.logger; + this.startTimestamp = process.hrtime(); + this.type = options.type; + this.indexPrefix = options.indexPrefix; + this.indexSeparator = options.indexSeparator; + this.indexName = this.constructIndexName(`${Date.now()}`); + this.alias = options.alias; + this.elasticSearchClient = options.elasticSearchClient; + + // The ES client bulk helper supports stream-based indexing, but we have to + // supply the stream directly to it at instantiation-time. We can't supply + // this class itself, so instead, we create this inline stream instead. + this.sourceStream = new Readable({ objectMode: true }); + this.sourceStream._read = () => {}; + + // eslint-disable-next-line consistent-this + const that = this; + + // Keep a reference to the ES Bulk helper so that we can know when all + // documents have been successfully written to ES. + this.bulkResult = this.elasticSearchClient.helpers.bulk({ + datasource: this.sourceStream, + onDocument() { + that.processed++; + return { + index: { _index: that.indexName }, + }; + }, + refreshOnCompletion: that.indexName, + }); + } + + async initialize(): Promise { + this.logger.info(`Started indexing documents for index ${this.type}`); + + const aliases = await this.elasticSearchClient.cat.aliases({ + format: 'json', + name: this.alias, + }); + + this.removableIndices = aliases.body.map( + (r: Record) => r.index, + ); + + await this.elasticSearchClient.indices.create({ + index: this.indexName, + }); + } + + async index(documents: IndexableDocument[]): Promise { + await this.isReady(); + documents.forEach(document => { + this.received++; + this.sourceStream.push(document); + }); + } + + async finalize(): Promise { + // Wait for all documents to be processed. + await this.isReady(); + + // Close off the underlying stream connected to ES, indicating that no more + // documents will be written. + this.sourceStream.push(null); + + // Wait for the bulk helper to finish processing. + const result = await this.bulkResult; + + // Rotate aliases upon completion. Allow errors to bubble up so that we can + // clean up the create index. + this.logger.info( + `Indexing completed for index ${this.type} in ${duration( + this.startTimestamp, + )}`, + result, + ); + await this.elasticSearchClient.indices.updateAliases({ + body: { + actions: [ + { + remove: { index: this.constructIndexName('*'), alias: this.alias }, + }, + { add: { index: this.indexName, alias: this.alias } }, + ], + }, + }); + + // If any indices are removable, remove them. Do not bubble up this error, + // as doing so would delete the now aliased index. Log instead. + if (this.removableIndices.length) { + this.logger.info('Removing stale search indices', this.removableIndices); + try { + await this.elasticSearchClient.indices.delete({ + index: this.removableIndices, + }); + } catch (e) { + this.logger.warn(`Failed to remove stale search indices: ${e}`); + } + } + } + + /** + * Ensures that the number of documents sent over the wire to ES matches the + * number of documents this stream has received so far. This helps manage + * backpressure in other parts of the indexing pipeline. + */ + private isReady(): Promise { + return new Promise(resolve => { + const interval = setInterval(() => { + if (this.received === this.processed) { + clearInterval(interval); + resolve(); + } + }, 50); + }); + } + + private constructIndexName(postFix: string) { + return `${this.indexPrefix}${this.type}${this.indexSeparator}${postFix}`; + } +} diff --git a/plugins/search-backend-module-elasticsearch/src/engines/index.ts b/plugins/search-backend-module-elasticsearch/src/engines/index.ts index d5eee37803..19c24b37ff 100644 --- a/plugins/search-backend-module-elasticsearch/src/engines/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/engines/index.ts @@ -19,3 +19,7 @@ export type { ConcreteElasticSearchQuery, ElasticSearchClientOptions, } from './ElasticSearchSearchEngine'; +export type { + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './ElasticSearchSearchEngineIndexer'; diff --git a/plugins/search-backend-module-elasticsearch/src/index.ts b/plugins/search-backend-module-elasticsearch/src/index.ts index 8cf96de858..223141be6b 100644 --- a/plugins/search-backend-module-elasticsearch/src/index.ts +++ b/plugins/search-backend-module-elasticsearch/src/index.ts @@ -21,4 +21,8 @@ */ export { ElasticSearchSearchEngine } from './engines'; -export type { ElasticSearchClientOptions } from './engines'; +export type { + ElasticSearchClientOptions, + ElasticSearchSearchEngineIndexer, + ElasticSearchSearchEngineIndexerOptions, +} from './engines'; diff --git a/plugins/search-backend-module-pg/api-report.md b/plugins/search-backend-module-pg/api-report.md index e2edfc291a..ba8fde5f17 100644 --- a/plugins/search-backend-module-pg/api-report.md +++ b/plugins/search-backend-module-pg/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; import { IndexableDocument } from '@backstage/search-common'; import { Knex } from 'knex'; import { PluginDatabaseManager } from '@backstage/backend-common'; @@ -28,6 +29,8 @@ export class DatabaseDocumentStore implements DatabaseStore { // (undocumented) static create(knex: Knex): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -55,6 +58,8 @@ export interface DatabaseStore { // (undocumented) completeInsert(tx: Knex.Transaction, type: string): Promise; // (undocumented) + getTransaction(): Promise; + // (undocumented) insertDocuments( tx: Knex.Transaction, type: string, @@ -81,7 +86,7 @@ export class PgSearchEngine implements SearchEngine { database: PluginDatabaseManager; }): Promise; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) query(query: SearchQuery): Promise; // (undocumented) @@ -94,6 +99,28 @@ export class PgSearchEngine implements SearchEngine { translator(query: SearchQuery): ConcretePgSearchQuery; } +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(options: PgSearchEngineIndexerOptions); + // (undocumented) + finalize(): Promise; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + +// Warning: (ae-missing-release-tag) "PgSearchEngineIndexerOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + // Warning: (ae-missing-release-tag) "PgSearchQuery" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts index 56a3a6e643..8618bed4c8 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.test.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { range } from 'lodash'; import { DatabaseStore } from '../database'; import { ConcretePgSearchQuery, @@ -21,6 +20,13 @@ import { encodePageCursor, PgSearchEngine, } from './PgSearchEngine'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; + +jest.mock('./PgSearchEngineIndexer', () => ({ + PgSearchEngineIndexer: jest + .fn() + .mockImplementation(async () => 'the-expected-indexer'), +})); describe('PgSearchEngine', () => { const tx: any = {} as any; @@ -30,6 +36,7 @@ describe('PgSearchEngine', () => { beforeEach(() => { database = { transaction: jest.fn(), + getTransaction: jest.fn(), insertDocuments: jest.fn(), query: jest.fn(), completeInsert: jest.fn(), @@ -122,46 +129,21 @@ describe('PgSearchEngine', () => { }); }); - describe('insert', () => { - it('should insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); + describe('index', () => { + it('should instantiate indexer', async () => { + const indexer = await searchEngine.getIndexer('my-type'); - const documents = [ - { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, - { - location: 'location-2', - text: 'Hello World', - title: 'Dolor sit amet', - }, - ]; - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toHaveBeenCalledWith( - tx, - 'my-type', - documents, + // Indexer instantiated with expected args. + expect(PgSearchEngineIndexer).toHaveBeenCalledWith( + expect.objectContaining({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }), ); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); - }); - it('should batch insert documents', async () => { - database.transaction.mockImplementation(fn => fn(tx)); - - const documents = range(350).map(i => ({ - title: `Hello World ${i}`, - text: 'Lorem Ipsum', - location: `location-${i}`, - })); - - await searchEngine.index('my-type', documents); - - expect(database.transaction).toHaveBeenCalledTimes(1); - expect(database.prepareInsert).toHaveBeenCalledTimes(1); - expect(database.insertDocuments).toBeCalledTimes(4); - expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + // Indexer is as expected. + expect(indexer).toBe('the-expected-indexer'); }); }); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts index 6fd6c571a8..ac42007401 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngine.ts @@ -15,12 +15,8 @@ */ import { PluginDatabaseManager } from '@backstage/backend-common'; import { SearchEngine } from '@backstage/plugin-search-backend-node'; -import { - IndexableDocument, - SearchQuery, - SearchResultSet, -} from '@backstage/search-common'; -import { chunk } from 'lodash'; +import { SearchQuery, SearchResultSet } from '@backstage/search-common'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; import { DatabaseDocumentStore, DatabaseStore, @@ -77,16 +73,11 @@ export class PgSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - await this.databaseStore.transaction(async tx => { - await this.databaseStore.prepareInsert(tx); - - const batchSize = 100; - for (const documentBatch of chunk(documents, batchSize)) { - await this.databaseStore.insertDocuments(tx, type, documentBatch); - } - - await this.databaseStore.completeInsert(tx, type); + async getIndexer(type: string) { + return new PgSearchEngineIndexer({ + batchSize: 100, + type, + databaseStore: this.databaseStore, }); } diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..1fc7e74fc4 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { range } from 'lodash'; +import { PgSearchEngineIndexer } from './PgSearchEngineIndexer'; +import { DatabaseStore } from '../database'; + +describe('PgSearchEngineIndexer', () => { + const tx = { + rollback: jest.fn(), + commit: jest.fn(), + } as any; + let database: jest.Mocked; + let indexer: PgSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + database = { + transaction: jest.fn().mockImplementation(fn => fn(tx)), + getTransaction: jest.fn().mockReturnValue(tx), + insertDocuments: jest.fn(), + query: jest.fn(), + completeInsert: jest.fn(), + prepareInsert: jest.fn(), + }; + indexer = new PgSearchEngineIndexer({ + batchSize: 100, + type: 'my-type', + databaseStore: database, + }); + }); + + it('should insert documents', async () => { + const documents = [ + { title: 'Hello World', text: 'Lorem Ipsum', location: 'location-1' }, + { + location: 'location-2', + text: 'Hello World', + title: 'Dolor sit amet', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledWith( + tx, + 'my-type', + documents, + ); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + expect(tx.commit).toHaveBeenCalled(); + }); + + it('should batch insert documents', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toBeCalledTimes(4); + expect(database.completeInsert).toHaveBeenCalledWith(tx, 'my-type'); + }); + + it('should close out stream and bubble up error on prepare', async () => { + const expectedError = new Error('Prepare error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.prepareInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).not.toHaveBeenCalled(); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on insert', async () => { + const expectedError = new Error('Index error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.insertDocuments.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.completeInsert).not.toHaveBeenCalled(); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); + + it('should close tx and bubble up error on completion', async () => { + const expectedError = new Error('Completion error'); + const documents = [ + { + title: `Hello World`, + text: 'Lorem Ipsum', + location: `location`, + }, + ]; + + database.completeInsert.mockRejectedValueOnce(expectedError); + const result = await TestPipeline.withSubject(indexer) + .withDocuments(documents) + .execute(); + + expect(database.getTransaction).toHaveBeenCalledTimes(1); + expect(database.prepareInsert).toHaveBeenCalledTimes(1); + expect(database.insertDocuments).toHaveBeenCalledTimes(1); + expect(database.completeInsert).toHaveBeenCalledTimes(1); + expect(result.error).toBe(expectedError); + expect(tx.rollback).toHaveBeenCalledWith(expectedError); + }); +}); diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts new file mode 100644 index 0000000000..53d040cb87 --- /dev/null +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/PgSearchEngineIndexer.ts @@ -0,0 +1,74 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BatchSearchEngineIndexer } from '@backstage/plugin-search-backend-node'; +import { IndexableDocument } from '@backstage/search-common'; +import { Knex } from 'knex'; +import { DatabaseStore } from '../database'; + +export type PgSearchEngineIndexerOptions = { + batchSize: number; + type: string; + databaseStore: DatabaseStore; +}; + +export class PgSearchEngineIndexer extends BatchSearchEngineIndexer { + private store: DatabaseStore; + private type: string; + private tx: Knex.Transaction | undefined; + + constructor(options: PgSearchEngineIndexerOptions) { + super({ batchSize: options.batchSize }); + this.store = options.databaseStore; + this.type = options.type; + } + + async initialize(): Promise { + this.tx = await this.store.getTransaction(); + try { + await this.store.prepareInsert(this.tx); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx.rollback(e); + throw e; + } + } + + async index(documents: IndexableDocument[]): Promise { + try { + await this.store.insertDocuments(this.tx!, this.type, documents); + } catch (e) { + // In case of error, rollback the transaction and re-throw the error so + // that the stream can be closed and destroyed properly. + this.tx!.rollback(e); + throw e; + } + } + + async finalize(): Promise { + // Attempt to complete and commit the transaction. + try { + await this.store.completeInsert(this.tx!, this.type); + this.tx!.commit(); + } catch (e) { + // Otherwise, rollback the transaction and re-throw the error so that the + // stream can be closed and destroyed properly. + this.tx!.rollback!(e); + throw e; + } + } +} diff --git a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts index 7994998baf..7f8e297648 100644 --- a/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts +++ b/plugins/search-backend-module-pg/src/PgSearchEngine/index.ts @@ -15,3 +15,7 @@ */ export { PgSearchEngine } from './PgSearchEngine'; export type { ConcretePgSearchQuery } from './PgSearchEngine'; +export type { + PgSearchEngineIndexer, + PgSearchEngineIndexerOptions, +} from './PgSearchEngineIndexer'; diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index 8c180ea019..0d3ee63ca4 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -71,6 +71,10 @@ export class DatabaseDocumentStore implements DatabaseStore { return await this.db.transaction(fn); } + async getTransaction(): Promise { + return this.db.transaction(); + } + async prepareInsert(tx: Knex.Transaction): Promise { // We create a temporary table to collect the hashes of the documents that // we expect to be in the documents table at the end. The table is deleted diff --git a/plugins/search-backend-module-pg/src/database/types.ts b/plugins/search-backend-module-pg/src/database/types.ts index 0c0596160e..0a0dc35682 100644 --- a/plugins/search-backend-module-pg/src/database/types.ts +++ b/plugins/search-backend-module-pg/src/database/types.ts @@ -26,6 +26,7 @@ export interface PgSearchQuery { export interface DatabaseStore { transaction(fn: (tx: Knex.Transaction) => Promise): Promise; + getTransaction(): Promise; prepareInsert(tx: Knex.Transaction): Promise; insertDocuments( tx: Knex.Transaction, diff --git a/plugins/search-backend-node/api-report.md b/plugins/search-backend-node/api-report.md index 1929009760..37945c8902 100644 --- a/plugins/search-backend-node/api-report.md +++ b/plugins/search-backend-node/api-report.md @@ -3,30 +3,60 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { DocumentCollator } from '@backstage/search-common'; -import { DocumentDecorator } from '@backstage/search-common'; +/// + +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { DocumentDecoratorFactory } from '@backstage/search-common'; import { DocumentTypeInfo } from '@backstage/search-common'; import { IndexableDocument } from '@backstage/search-common'; import { Logger as Logger_2 } from 'winston'; import { default as lunr_2 } from 'lunr'; import { QueryTranslator } from '@backstage/search-common'; +import { Readable } from 'stream'; import { SearchEngine } from '@backstage/search-common'; import { SearchQuery } from '@backstage/search-common'; import { SearchResultSet } from '@backstage/search-common'; +import { Transform } from 'stream'; +import { Writable } from 'stream'; -// Warning: (ae-missing-release-tag) "IndexBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta +export abstract class BatchSearchEngineIndexer extends Writable { + constructor(options: BatchSearchEngineOptions); + abstract finalize(): Promise; + abstract index(documents: IndexableDocument[]): Promise; + abstract initialize(): Promise; +} + +// @beta (undocumented) +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +// @beta (undocumented) +export type ConcreteLunrQuery = { + lunrQueryBuilder: lunr_2.Index.QueryBuilder; + documentTypes?: string[]; + pageSize: number; +}; + +// @beta +export abstract class DecoratorBase extends Transform { + constructor(); + abstract decorate( + document: IndexableDocument, + ): Promise; + abstract finalize(): Promise; + abstract initialize(): Promise; +} + +// @beta (undocumented) export class IndexBuilder { - // Warning: (ae-forgotten-export) The symbol "IndexBuilderOptions" needs to be exported by the entry point index.d.ts constructor({ logger, searchEngine }: IndexBuilderOptions); - // Warning: (ae-forgotten-export) The symbol "RegisterCollatorParameters" needs to be exported by the entry point index.d.ts addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void; - // Warning: (ae-forgotten-export) The symbol "RegisterDecoratorParameters" needs to be exported by the entry point index.d.ts - addDecorator({ decorator }: RegisterDecoratorParameters): void; + addDecorator({ factory }: RegisterDecoratorParameters): void; build(): Promise<{ scheduler: Scheduler; }>; @@ -36,32 +66,61 @@ export class IndexBuilder { getSearchEngine(): SearchEngine; } -// Warning: (ae-missing-release-tag) "LunrSearchEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @beta (undocumented) +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger_2; +}; + +// @beta (undocumented) +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; + +// @beta (undocumented) export class LunrSearchEngine implements SearchEngine { constructor({ logger }: { logger: Logger_2 }); // (undocumented) protected docStore: Record; // (undocumented) - index(type: string, documents: IndexableDocument[]): Promise; + getIndexer(type: string): Promise; // (undocumented) protected logger: Logger_2; // (undocumented) protected lunrIndices: Record; // (undocumented) query(query: SearchQuery): Promise; - // Warning: (ae-forgotten-export) The symbol "LunrQueryTranslator" needs to be exported by the entry point index.d.ts - // // (undocumented) setTranslator(translator: LunrQueryTranslator): void; // (undocumented) protected translator: QueryTranslator; } -// Warning: (ae-missing-release-tag) "Scheduler" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public +// @beta (undocumented) +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + constructor(); + // (undocumented) + buildIndex(): lunr_2.Index; + // (undocumented) + finalize(): Promise; + // (undocumented) + getDocumentStore(): Record; + // (undocumented) + index(documents: IndexableDocument[]): Promise; + // (undocumented) + initialize(): Promise; +} + +// @beta +export interface RegisterCollatorParameters { + defaultRefreshIntervalSeconds: number; + factory: DocumentCollatorFactory; +} + +// @beta +export interface RegisterDecoratorParameters { + factory: DocumentDecoratorFactory; +} + +// @beta (undocumented) export class Scheduler { constructor({ logger }: { logger: Logger_2 }); addToSchedule(task: Function, interval: number): void; @@ -70,4 +129,17 @@ export class Scheduler { } export { SearchEngine }; + +// @beta +export class TestPipeline { + execute(): Promise; + withDocuments(documents: IndexableDocument[]): TestPipeline; + static withSubject(subject: Readable | Transform | Writable): TestPipeline; +} + +// @beta +export type TestPipelineResult = { + error: unknown; + documents: IndexableDocument[]; +}; ``` diff --git a/plugins/search-backend-node/package.json b/plugins/search-backend-node/package.json index dd8ef5599a..2c644e357d 100644 --- a/plugins/search-backend-node/package.json +++ b/plugins/search-backend-node/package.json @@ -23,10 +23,12 @@ "clean": "backstage-cli package clean" }, "dependencies": { + "@backstage/errors": "^0.2.2", "@backstage/search-common": "^0.2.4", - "winston": "^3.2.1", + "@types/lunr": "^2.3.3", + "lodash": "^4.17.21", "lunr": "^2.3.9", - "@types/lunr": "^2.3.3" + "winston": "^3.2.1" }, "devDependencies": { "@backstage/backend-common": "^0.11.0", diff --git a/plugins/search-backend-node/src/IndexBuilder.test.ts b/plugins/search-backend-node/src/IndexBuilder.test.ts index 0465f2701c..95a2e1d439 100644 --- a/plugins/search-backend-node/src/IndexBuilder.test.ts +++ b/plugins/search-backend-node/src/IndexBuilder.test.ts @@ -16,35 +16,37 @@ import { getVoidLogger } from '@backstage/backend-common'; import { - DocumentCollator, - DocumentDecorator, - IndexableDocument, + DocumentCollatorFactory, + DocumentDecoratorFactory, } from '@backstage/search-common'; +import { Readable, Transform } from 'stream'; import { IndexBuilder } from './IndexBuilder'; import { LunrSearchEngine, SearchEngine } from './index'; -class TestDocumentCollator implements DocumentCollator { +class TestDocumentCollatorFactory implements DocumentCollatorFactory { readonly type: string = 'anything'; - async execute(): Promise { - return []; + async getCollator(): Promise { + const collator = new Readable({ objectMode: true }); + collator._read = () => {}; + return collator; } } -class TypedDocumentCollator extends TestDocumentCollator { +class TypedDocumentCollatorFactory extends TestDocumentCollatorFactory { readonly type = 'an-expected-type'; } -class TestDocumentDecorator implements DocumentDecorator { - async execute(documents: IndexableDocument[]) { - return documents; +class TestDocumentDecoratorFactory implements DocumentDecoratorFactory { + async getDecorator(): Promise { + return new Transform(); } } -class TypedDocumentDecorator extends TestDocumentDecorator { +class TypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['an-expected-type']; } -class DifferentlyTypedDocumentDecorator extends TestDocumentDecorator { +class DifferentlyTypedDocumentDecoratorFactory extends TestDocumentDecoratorFactory { readonly types = ['not-the-expected-type']; } @@ -64,13 +66,13 @@ describe('IndexBuilder', () => { describe('addCollator', () => { it('adds a collator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const collatorSpy = jest.spyOn(testCollator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Build the index and ensure the collator was invoked. @@ -84,19 +86,19 @@ describe('IndexBuilder', () => { describe('addDecorator', () => { it('adds a decorator', async () => { jest.useFakeTimers(); - const testCollator = new TestDocumentCollator(); - const testDecorator = new TestDocumentDecorator(); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = new TestDocumentDecoratorFactory(); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -110,27 +112,20 @@ describe('IndexBuilder', () => { it('adds a type-specific decorator', async () => { jest.useFakeTimers(); - const testCollator = new TypedDocumentCollator(); - const testDecorator = new TypedDocumentDecorator(); - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TypedDocumentCollatorFactory(); + const testDecoratorFactory = new TypedDocumentDecoratorFactory(); + jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for the same type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was invoked. @@ -140,31 +135,24 @@ describe('IndexBuilder', () => { // wait for async decorator execution await Promise.resolve(); expect(decoratorSpy).toHaveBeenCalled(); - expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); it('adds a type-specific decorator that should not be called', async () => { - const docFixture = { - title: 'Test', - text: 'Test text.', - location: '/test/location', - }; - const testCollator = new TestDocumentCollator(); - const testDecorator = new DifferentlyTypedDocumentDecorator(); - const collatorSpy = jest - .spyOn(testCollator, 'execute') - .mockImplementation(async () => [docFixture]); - const decoratorSpy = jest.spyOn(testDecorator, 'execute'); + const testCollatorFactory = new TestDocumentCollatorFactory(); + const testDecoratorFactory = + new DifferentlyTypedDocumentDecoratorFactory(); + const collatorSpy = jest.spyOn(testCollatorFactory, 'getCollator'); + const decoratorSpy = jest.spyOn(testDecoratorFactory, 'getDecorator'); // Add a collator. testIndexBuilder.addCollator({ defaultRefreshIntervalSeconds: 6, - collator: testCollator, + factory: testCollatorFactory, }); // Add a decorator for a different type. testIndexBuilder.addDecorator({ - decorator: testDecorator, + factory: testDecoratorFactory, }); // Build the index and ensure the decorator was not invoked. diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 92adb03d7d..e0a059e303 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -15,32 +15,31 @@ */ import { - DocumentCollator, - DocumentDecorator, + DocumentCollatorFactory, + DocumentDecoratorFactory, DocumentTypeInfo, - IndexableDocument, SearchEngine, } from '@backstage/search-common'; +import { Transform, pipeline } from 'stream'; import { Logger } from 'winston'; import { Scheduler } from './index'; import { + IndexBuilderOptions, RegisterCollatorParameters, RegisterDecoratorParameters, } from './types'; interface CollatorEnvelope { - collate: DocumentCollator; + factory: DocumentCollatorFactory; refreshInterval: number; } -type IndexBuilderOptions = { - searchEngine: SearchEngine; - logger: Logger; -}; - +/** + * @beta + */ export class IndexBuilder { private collators: Record; - private decorators: Record; + private decorators: Record; private documentTypes: Record; private searchEngine: SearchEngine; private logger: Logger; @@ -66,18 +65,18 @@ export class IndexBuilder { * given refresh interval. */ addCollator({ - collator, + factory, defaultRefreshIntervalSeconds, }: RegisterCollatorParameters): void { this.logger.info( - `Added ${collator.constructor.name} collator for type ${collator.type}`, + `Added ${factory.constructor.name} collator factory for type ${factory.type}`, ); - this.collators[collator.type] = { + this.collators[factory.type] = { refreshInterval: defaultRefreshIntervalSeconds, - collate: collator, + factory, }; - this.documentTypes[collator.type] = { - visibilityPermission: collator.visibilityPermission, + this.documentTypes[factory.type] = { + visibilityPermission: factory.visibilityPermission, }; } @@ -86,18 +85,18 @@ export class IndexBuilder { * the decorator, it will be applied to documents from all known collators, * otherwise it will only be applied to documents of the given types. */ - addDecorator({ decorator }: RegisterDecoratorParameters): void { - const types = decorator.types || ['*']; + addDecorator({ factory }: RegisterDecoratorParameters): void { + const types = factory.types || ['*']; this.logger.info( - `Added decorator ${decorator.constructor.name} to types ${types.join( + `Added decorator ${factory.constructor.name} to types ${types.join( ', ', )}`, ); types.forEach(type => { if (this.decorators.hasOwnProperty(type)) { - this.decorators[type].push(decorator); + this.decorators[type].push(factory); } else { - this.decorators[type] = [decorator]; + this.decorators[type] = [factory]; } }); } @@ -111,46 +110,43 @@ export class IndexBuilder { Object.keys(this.collators).forEach(type => { scheduler.addToSchedule(async () => { - // Collate, Decorate, Index. - const decorators: DocumentDecorator[] = ( - this.decorators['*'] || [] - ).concat(this.decorators[type] || []); - - this.logger.debug( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, + // Instantiate the collator. + const collator = await this.collators[type].factory.getCollator(); + this.logger.info( + `Collating documents for ${type} via ${this.collators[type].factory.constructor.name}`, ); - let documents: IndexableDocument[]; - try { - documents = await this.collators[type].collate.execute(); - } catch (e) { - this.logger.error( - `Collating documents for ${type} via ${this.collators[type].collate.constructor.name} failed: ${e}`, - ); - return; - } + // Instantiate all relevant decorators. + const decorators: Transform[] = await Promise.all( + (this.decorators['*'] || []) + .concat(this.decorators[type] || []) + .map(async factory => { + const decorator = await factory.getDecorator(); + this.logger.info( + `Attached decorator via ${factory.constructor.name} to ${type} index pipeline.`, + ); + return decorator; + }), + ); - for (let i = 0; i < decorators.length; i++) { - this.logger.debug( - `Decorating ${type} documents via ${decorators[i].constructor.name}`, - ); - try { - documents = await decorators[i].execute(documents); - } catch (e) { - this.logger.error( - `Decorating ${type} documents via ${decorators[i].constructor.name} failed: ${e}`, - ); - return; - } - } + // Instantiate the indexer. + const indexer = await this.searchEngine.getIndexer(type); - if (!documents || documents.length === 0) { - this.logger.debug(`No documents for type "${type}" to index`); - return; - } + // Compose collator/decorators/indexer into a pipeline + return new Promise(done => { + pipeline([collator, ...decorators, indexer], error => { + if (error) { + this.logger.error( + `Collating documents for ${type} failed: ${error}`, + ); + } else { + this.logger.info(`Collating documents for ${type} succeeded`); + } - // pushing documents to index to a configured search engine. - await this.searchEngine.index(type, documents); + // Signal index pipeline completion! + done(); + }); + }); }, this.collators[type].refreshInterval * 1000); }); diff --git a/plugins/search-backend-node/src/Scheduler.ts b/plugins/search-backend-node/src/Scheduler.ts index 3e356aa6aa..6debaa3dcd 100644 --- a/plugins/search-backend-node/src/Scheduler.ts +++ b/plugins/search-backend-node/src/Scheduler.ts @@ -26,6 +26,9 @@ type TaskEnvelope = { * TODO: coordination, error handling */ +/** + * @beta + */ export class Scheduler { private logger: Logger; private schedule: TaskEnvelope[]; diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts index cbe48ef5a7..88a3ba9a24 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.test.ts @@ -16,28 +16,61 @@ import { getVoidLogger } from '@backstage/backend-common'; import lunr from 'lunr'; -import { SearchEngine } from '@backstage/search-common'; +import { IndexableDocument, SearchEngine } from '@backstage/search-common'; import { ConcreteLunrQuery, LunrSearchEngine, decodePageCursor, encodePageCursor, } from './LunrSearchEngine'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; /** * Just used to test the default translator shipped with LunrSearchEngine. */ -class LunrSearchEngineForTranslatorTests extends LunrSearchEngine { +class LunrSearchEngineForTests extends LunrSearchEngine { + getDocStore() { + return this.docStore; + } + setDocStore(docStore: Record) { + this.docStore = docStore; + } + getLunrIndices() { + return this.lunrIndices; + } getTranslator() { return this.translator; } } +const indexerMock = { + on: jest.fn(), + buildIndex: jest.fn(), + getDocumentStore: jest.fn(), +}; +jest.mock('./LunrSearchEngineIndexer', () => ({ + LunrSearchEngineIndexer: jest.fn().mockImplementation(() => indexerMock), +})); + +const getActualIndexer = (engine: SearchEngine, index: string) => { + (LunrSearchEngineIndexer as unknown as jest.Mock).mockImplementationOnce( + () => { + const ActualIndexer = jest.requireActual( + './LunrSearchEngineIndexer', + ).LunrSearchEngineIndexer; + return new ActualIndexer(); + }, + ); + return engine.getIndexer(index); +}; + describe('LunrSearchEngine', () => { let testLunrSearchEngine: SearchEngine; beforeEach(() => { testLunrSearchEngine = new LunrSearchEngine({ logger: getVoidLogger() }); + jest.clearAllMocks(); }); describe('translator', () => { @@ -65,7 +98,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -107,7 +140,7 @@ describe('LunrSearchEngine', () => { }); it('should have default offset and limit', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -148,7 +181,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with 1 filter', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -193,7 +226,7 @@ describe('LunrSearchEngine', () => { }); it('should handle single-item array filter as scalar value', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -224,7 +257,7 @@ describe('LunrSearchEngine', () => { }); it('should return translated query with multiple filters', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -273,7 +306,7 @@ describe('LunrSearchEngine', () => { }); it('should throw if translated query references missing field', async () => { - const inspectableSearchEngine = new LunrSearchEngineForTranslatorTests({ + const inspectableSearchEngine = new LunrSearchEngineForTests({ logger: getVoidLogger(), }); const translatorUnderTest = inspectableSearchEngine.getTranslator(); @@ -334,7 +367,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -359,7 +398,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -392,7 +437,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -424,7 +475,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -456,7 +513,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -489,7 +552,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -522,7 +591,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 1 document - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -560,7 +635,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -604,8 +685,21 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 1 document each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer1 = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer1) + .withDocuments(mockDocuments) + .execute(); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); + // Perform search query scoped to "test-index-2" with a filter on the field "extraField" const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -642,7 +736,13 @@ describe('LunrSearchEngine', () => { ]; // Mock indexing of 2 documents - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); // Perform search query const mockedSearchResult = await testLunrSearchEngine.query({ @@ -695,8 +795,20 @@ describe('LunrSearchEngine', () => { ]; // Mock 2 indices with 2 documents each - await testLunrSearchEngine.index('test-index', mockDocuments); - await testLunrSearchEngine.index('test-index-2', mockDocuments2); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); + const indexer2 = await getActualIndexer( + testLunrSearchEngine, + 'test-index-2', + ); + await TestPipeline.withSubject(indexer2) + .withDocuments(mockDocuments2) + .execute(); // Perform search query scoped to "test-index-2" const mockedSearchResult = await testLunrSearchEngine.query({ @@ -734,7 +846,13 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer( + testLunrSearchEngine, + 'test-index', + ); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -767,7 +885,10 @@ describe('LunrSearchEngine', () => { location: `test/location/${i}`, })); - await testLunrSearchEngine.index('test-index', mockDocuments); + const indexer = await getActualIndexer(testLunrSearchEngine, 'test-index'); + await TestPipeline.withSubject(indexer) + .withDocuments(mockDocuments) + .execute(); const mockedSearchResult = await testLunrSearchEngine.query({ term: 'testTitle', @@ -793,22 +914,46 @@ describe('LunrSearchEngine', () => { }); describe('index', () => { - it('should index document', async () => { - const indexSpy = jest.spyOn(testLunrSearchEngine, 'index'); - const mockDocuments = [ - { - title: 'testTerm', - text: 'testText', - location: 'test/location', - }, - ]; + it('should get indexer', async () => { + const indexer = await testLunrSearchEngine.getIndexer('test-index'); + expect(LunrSearchEngineIndexer).toHaveBeenCalled(); + expect(indexer.on).toHaveBeenCalledWith('close', expect.any(Function)); + }); - // call index func and ensure the index func was invoked. - await testLunrSearchEngine.index('test-index', mockDocuments); - expect(indexSpy).toHaveBeenCalled(); - expect(indexSpy).toHaveBeenCalledWith('test-index', [ - { title: 'testTerm', text: 'testText', location: 'test/location' }, - ]); + it('should manage indices and docs on close', async () => { + const doc = { title: 'A doc', text: 'test', location: 'some-location' }; + + // Set up an inspectable search engine to pre-set some data. + const inspectableSearchEngine = new LunrSearchEngineForTests({ + logger: getVoidLogger(), + }); + inspectableSearchEngine.setDocStore({ 'existing-location': doc }); + + // Mock methds called by close handler. + indexerMock.buildIndex.mockReturnValueOnce('expected-index'); + indexerMock.getDocumentStore.mockReturnValueOnce({ + 'new-location': doc, + }); + + // Get the indexer and invoke its close handler. + await inspectableSearchEngine.getIndexer('test-index'); + const onClose = indexerMock.on.mock.calls[0][1] as Function; + onClose(); + + // Ensure mocked methods were called. + expect(indexerMock.buildIndex).toHaveBeenCalled(); + expect(indexerMock.getDocumentStore).toHaveBeenCalled(); + + // Ensure the lunr index was written to the search engine. + expect(inspectableSearchEngine.getLunrIndices()).toStrictEqual({ + 'test-index': 'expected-index', + }); + + // Ensure documents are merged into the existing store. + expect(inspectableSearchEngine.getDocStore()).toStrictEqual({ + 'existing-location': doc, + 'new-location': doc, + }); }); }); }); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts index ea51ffe8e1..394cfffac8 100644 --- a/plugins/search-backend-node/src/engines/LunrSearchEngine.ts +++ b/plugins/search-backend-node/src/engines/LunrSearchEngine.ts @@ -23,7 +23,11 @@ import { } from '@backstage/search-common'; import lunr from 'lunr'; import { Logger } from 'winston'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; +/** + * @beta + */ export type ConcreteLunrQuery = { lunrQueryBuilder: lunr.Index.QueryBuilder; documentTypes?: string[]; @@ -35,8 +39,14 @@ type LunrResultEnvelope = { type: string; }; -type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ +export type LunrQueryTranslator = (query: SearchQuery) => ConcreteLunrQuery; +/** + * @beta + */ export class LunrSearchEngine implements SearchEngine { protected lunrIndices: Record = {}; protected docStore: Record; @@ -124,30 +134,17 @@ export class LunrSearchEngine implements SearchEngine { this.translator = translator; } - async index(type: string, documents: IndexableDocument[]): Promise { - const lunrBuilder = new lunr.Builder(); + async getIndexer(type: string) { + const indexer = new LunrSearchEngineIndexer(); - lunrBuilder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); - lunrBuilder.searchPipeline.add(lunr.stemmer); - - // Make this lunr index aware of all relevant fields. - Object.keys(documents[0]).forEach(field => { - lunrBuilder.field(field); + indexer.on('close', () => { + // Once the stream is closed, build the index and store the documents in + // memory for later retrieval. + this.lunrIndices[type] = indexer.buildIndex(); + this.docStore = { ...this.docStore, ...indexer.getDocumentStore() }; }); - // Set "location" field as reference field - lunrBuilder.ref('location'); - - documents.forEach((document: IndexableDocument) => { - // Add document to Lunar index - lunrBuilder.add(document); - // Store documents in memory to be able to look up document using the ref during query time - // This is not how you should implement your SearchEngine implementation! Do not copy! - this.docStore[document.location] = document; - }); - - // "Rotate" the index by simply overwriting any existing index of the same name. - this.lunrIndices[type] = lunrBuilder.build(); + return indexer; } async query(query: SearchQuery): Promise { diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..fbb6b153d0 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import lunr from 'lunr'; +import { range } from 'lodash'; +import { TestPipeline } from '../test-utils'; +import { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; + +const lunrBuilderAddSpy = jest.fn(); +const lunrBuilderRefSpy = jest.fn(); +const lunrBuilderFieldSpy = jest.fn(); +const lunrBuilderPipelineAddSpy = jest.fn(); +const lunrBuilderSearchPipelineAddSpy = jest.fn(); + +jest.mock('lunr', () => { + const actualLunr = jest.requireActual('lunr'); + return { + ...actualLunr, + Builder: jest.fn().mockImplementation(() => { + const actualBuilder = new actualLunr.Builder(); + actualBuilder.add = lunrBuilderAddSpy; + actualBuilder.ref = lunrBuilderRefSpy; + actualBuilder.field = lunrBuilderFieldSpy; + actualBuilder.pipeline.add = lunrBuilderPipelineAddSpy; + actualBuilder.searchPipeline.add = lunrBuilderSearchPipelineAddSpy; + return actualBuilder; + }), + }; +}); + +describe('LunrSearchEngineIndexer', () => { + let indexer: LunrSearchEngineIndexer; + + beforeEach(() => { + jest.clearAllMocks(); + indexer = new LunrSearchEngineIndexer(); + }); + + it('should index documents', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + expect(lunrBuilderAddSpy).toHaveBeenCalledWith(documents[0]); + }); + + it('should index documents in bulk', async () => { + const documents = range(350).map(i => ({ + title: `Hello World ${i}`, + text: 'Lorem Ipsum', + location: `location-${i}`, + })); + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + expect(lunrBuilderAddSpy).toHaveBeenCalledTimes(350); + }); + + it('should initialize schema', async () => { + const documents = [ + { + title: 'testTerm', + text: 'testText', + location: 'test/location', + extra: 'field', + }, + ]; + + await TestPipeline.withSubject(indexer).withDocuments(documents).execute(); + + // Builder ref should be set to location (and only once). + expect(lunrBuilderRefSpy).toHaveBeenCalledTimes(1); + expect(lunrBuilderRefSpy).toHaveBeenLastCalledWith('location'); + + // Builder fields should be based on document fields. + expect(lunrBuilderFieldSpy).toHaveBeenCalledTimes(4); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('title'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('text'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('location'); + expect(lunrBuilderFieldSpy).toHaveBeenCalledWith('extra'); + }); + + it('should configure lunr pipeline', async () => { + expect(lunrBuilderSearchPipelineAddSpy).toHaveBeenLastCalledWith( + lunr.stemmer, + ); + expect(lunrBuilderPipelineAddSpy).toHaveBeenCalledWith( + ...[lunr.trimmer, lunr.stopWordFilter, lunr.stemmer], + ); + }); +}); diff --git a/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts new file mode 100644 index 0000000000..01c95f07d4 --- /dev/null +++ b/plugins/search-backend-node/src/engines/LunrSearchEngineIndexer.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import lunr from 'lunr'; +import { BatchSearchEngineIndexer } from '../indexing'; + +/** + * @beta + */ +export class LunrSearchEngineIndexer extends BatchSearchEngineIndexer { + private schemaInitialized = false; + private builder: lunr.Builder; + private docStore: Record = {}; + + constructor() { + super({ batchSize: 100 }); + + this.builder = new lunr.Builder(); + this.builder.pipeline.add(lunr.trimmer, lunr.stopWordFilter, lunr.stemmer); + this.builder.searchPipeline.add(lunr.stemmer); + } + + // No async initialization required. + async initialize(): Promise {} + async finalize(): Promise {} + + async index(documents: IndexableDocument[]): Promise { + if (!this.schemaInitialized) { + // Make this lunr index aware of all relevant fields. + Object.keys(documents[0]).forEach(field => { + this.builder.field(field); + }); + + // Set "location" field as reference field + this.builder.ref('location'); + + this.schemaInitialized = true; + } + + documents.forEach(document => { + // Add document to Lunar index + this.builder.add(document); + + // Store documents in memory to be able to look up document using the ref during query time + // This is not how you should implement your SearchEngine implementation! Do not copy! + this.docStore[document.location] = document; + }); + } + + buildIndex() { + return this.builder.build(); + } + + getDocumentStore() { + return this.docStore; + } +} diff --git a/plugins/search-backend-node/src/engines/index.ts b/plugins/search-backend-node/src/engines/index.ts index 7e6fb86bd4..0d710eadc2 100644 --- a/plugins/search-backend-node/src/engines/index.ts +++ b/plugins/search-backend-node/src/engines/index.ts @@ -15,4 +15,8 @@ */ export { LunrSearchEngine } from './LunrSearchEngine'; -export type { ConcreteLunrQuery } from './LunrSearchEngine'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, +} from './LunrSearchEngine'; +export type { LunrSearchEngineIndexer } from './LunrSearchEngineIndexer'; diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index d49ef195f2..2edab1b01d 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -23,6 +23,18 @@ export { IndexBuilder } from './IndexBuilder'; export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; +export type { + ConcreteLunrQuery, + LunrQueryTranslator, + LunrSearchEngineIndexer, +} from './engines'; +export type { + IndexBuilderOptions, + RegisterCollatorParameters, + RegisterDecoratorParameters, +} from './types'; +export * from './indexing'; +export * from './test-utils'; /** * @deprecated Import from @backstage/search-common instead diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts new file mode 100644 index 0000000000..b692ce3aff --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +import { TestPipeline } from '../test-utils'; + +const indexSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteBatchIndexer extends BatchSearchEngineIndexer { + async index(documents: IndexableDocument[]): Promise { + return indexSpy(documents); + } + async initialize(): Promise { + return initializeSpy(); + } + async finalize(): Promise { + return finalizeSpy(); + } +} + +describe('BatchSearchEngineIndexer', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + await TestPipeline.withSubject(indexer) + .withDocuments([document, document, document]) + .execute(); + expect(indexSpy).toHaveBeenCalledTimes(3); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + indexer.write(document); + }); + }); + + it('should call index according to batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert the batches. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(2); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document, document]); + expect(indexSpy).toHaveBeenNthCalledWith(2, [document]); + done(); + }); + + // Write batchSize + 1 documents and end the stream. + indexer.write(document); + indexer.write(document); + indexer.write(document); + indexer.end(); + }); + }); + + it('should call index without exceeding batchSize', () => { + const indexer = new ConcreteBatchIndexer({ batchSize: 2 }); + + return new Promise(done => { + // Listen for it to finish and assert that it still wrote. + indexer.on('finish', () => { + expect(indexSpy).toHaveBeenCalledTimes(1); + expect(indexSpy).toHaveBeenNthCalledWith(1, [document]); + done(); + }); + + // Write batchSize - 1 documents and end the stream. + indexer.write(document); + indexer.end(); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('index error'); + indexSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const indexer = new ConcreteBatchIndexer({ batchSize: 1 }); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + indexer.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + indexer.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts new file mode 100644 index 0000000000..b368dd8184 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/BatchSearchEngineIndexer.ts @@ -0,0 +1,125 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Writable } from 'stream'; + +/** + * @beta + */ +export type BatchSearchEngineOptions = { + batchSize: number; +}; + +/** + * Base class encapsulating batch-based stream processing. Useful as a base + * class for search engine indexers. + * @beta + */ +export abstract class BatchSearchEngineIndexer extends Writable { + private batchSize: number; + private currentBatch: IndexableDocument[] = []; + private initialized: Promise; + + constructor(options: BatchSearchEngineOptions) { + super({ objectMode: true }); + this.batchSize = options.batchSize; + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Receives an array of indexable documents (of size this.batchSize) which + * should be written to the search engine. This method won't be called again + * at least until it resolves. + */ + public abstract index(documents: IndexableDocument[]): Promise; + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates batch stream write logic. + * @internal + */ + async _write( + doc: IndexableDocument, + _e: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + this.currentBatch.push(doc); + if (this.currentBatch.length < this.batchSize) { + done(); + return; + } + + try { + await this.index(this.currentBatch); + this.currentBatch = []; + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + // Index any remaining documents. + if (this.currentBatch.length) { + await this.index(this.currentBatch); + this.currentBatch = []; + } + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts new file mode 100644 index 0000000000..3b045dcddf --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.test.ts @@ -0,0 +1,157 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { DecoratorBase } from './DecoratorBase'; +import { TestPipeline } from '../test-utils'; + +const decorateSpy = jest.fn().mockResolvedValue(undefined); +const initializeSpy = jest.fn().mockResolvedValue(undefined); +const finalizeSpy = jest.fn().mockResolvedValue(undefined); + +class ConcreteDecorator extends DecoratorBase { + public initialize(): Promise { + return initializeSpy(); + } + public decorate( + document: IndexableDocument, + ): Promise { + return decorateSpy(document); + } + public finalize(): Promise { + return finalizeSpy(); + } +} + +describe('DecoratorBase', () => { + const document = { + title: 'Some Document', + text: 'Some document text.', + location: '/some/location', + }; + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should work end-to-end', async () => { + decorateSpy.mockImplementation(doc => ({ + ...doc, + transformed: true, + })); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(documents.length).toBe(3); + expect((documents[0] as unknown as any).transformed).toBe(true); + expect((documents[1] as unknown as any).transformed).toBe(true); + expect((documents[2] as unknown as any).transformed).toBe(true); + }); + + it('should allow filtering', async () => { + decorateSpy.mockResolvedValue(undefined); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(0); + }); + + it('should allow fanning', async () => { + decorateSpy.mockImplementation(doc => { + return [doc, doc]; + }); + + const decorator = new ConcreteDecorator(); + const { documents } = await TestPipeline.withSubject(decorator) + .withDocuments([document, document, document]) + .execute(); + + expect(decorateSpy).toHaveBeenCalledTimes(3); + expect(documents.length).toBe(6); + }); + + it('should call initialize at construction', () => { + // @ts-expect-error + const _indexer = new ConcreteDecorator(); + + return new Promise(done => { + // Allow initialization to complete. + setImmediate(() => { + expect(initializeSpy).toHaveBeenCalled(); + done(); + }); + }); + }); + + it('should emit error if initialization throws', () => { + // Cause the initializer to throw. + const expectedError = new Error('some error'); + initializeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + // Write a document to force the error state to become known. + decorator.write(document); + }); + }); + + it('should emit error if index throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('decorate error'); + decorateSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.write(document); + }); + }); + + it('should emit error if finalize throws', () => { + // Cause the indexer to throw. + const expectedError = new Error('finalize error'); + finalizeSpy.mockRejectedValue(expectedError); + const decorator = new ConcreteDecorator(); + + return new Promise(done => { + // Listen for the error and assert it's what was thrown. + decorator.on('error', error => { + expect(error).toStrictEqual(expectedError); + done(); + }); + + decorator.end(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/indexing/DecoratorBase.ts b/plugins/search-backend-node/src/indexing/DecoratorBase.ts new file mode 100644 index 0000000000..382ce443e3 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/DecoratorBase.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { assertError } from '@backstage/errors'; +import { IndexableDocument } from '@backstage/search-common'; +import { Transform } from 'stream'; + +/** + * Base class encapsulating simple async transformations. Useful as a base + * class for Backstage search decorators. + * @beta + */ +export abstract class DecoratorBase extends Transform { + private initialized: Promise; + + constructor() { + super({ objectMode: true }); + + // @todo Once node v15 is minimum, convert to _construct implementation. + this.initialized = new Promise(done => { + // Necessary to allow concrete implementation classes to construct + // themselves before calling their initialize() methods. + setImmediate(async () => { + try { + await this.initialize(); + done(undefined); + } catch (e) { + assertError(e); + done(e); + } + }); + }); + } + + /** + * Any asynchronous setup tasks can be performed here. + */ + public abstract initialize(): Promise; + + /** + * Receives a single indexable document. In your decorate method, you can: + * + * - Resolve `undefined` to indicate the record should be omitted. + * - Resolve a single modified document, which could contain new fields, + * edited fields, or removed fields. + * - Resolve an array of indexable documents, if the purpose if the decorator + * is to convert one document into multiple derivative documents. + */ + public abstract decorate( + document: IndexableDocument, + ): Promise; + + /** + * Any asynchronous teardown tasks can be performed here. + */ + public abstract finalize(): Promise; + + /** + * Encapsulates simple transform stream logic. + * @internal + */ + async _transform( + document: IndexableDocument, + _: any, + done: (error?: Error | null) => void, + ) { + // Wait for init before proceeding. Throw error if initialization failed. + const maybeError = await this.initialized; + if (maybeError) { + done(maybeError); + return; + } + + try { + const decorated = await this.decorate(document); + + // If undefined was returned, omit the record and move on. + if (decorated === undefined) { + done(); + return; + } + + // If an array of documents was given, push them all. + if (Array.isArray(decorated)) { + decorated.forEach(doc => { + this.push(doc); + }); + done(); + return; + } + + // Otherwise, just push the decorated document. + this.push(decorated); + done(); + } catch (e) { + assertError(e); + done(e); + } + } + + /** + * Encapsulates finalization and final error handling logic. + * @internal + */ + async _final(done: (error?: Error | null) => void) { + try { + await this.finalize(); + done(); + } catch (e) { + assertError(e); + done(e); + } + } +} diff --git a/plugins/search-backend-node/src/indexing/index.ts b/plugins/search-backend-node/src/indexing/index.ts new file mode 100644 index 0000000000..e86235df16 --- /dev/null +++ b/plugins/search-backend-node/src/indexing/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { BatchSearchEngineIndexer } from './BatchSearchEngineIndexer'; +export { DecoratorBase } from './DecoratorBase'; +export type { BatchSearchEngineOptions } from './BatchSearchEngineIndexer'; diff --git a/plugins/search-backend-node/src/test-utils/TestPipeline.ts b/plugins/search-backend-node/src/test-utils/TestPipeline.ts new file mode 100644 index 0000000000..c0c2bf5f89 --- /dev/null +++ b/plugins/search-backend-node/src/test-utils/TestPipeline.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { IndexableDocument } from '@backstage/search-common'; +import { pipeline, Readable, Transform, Writable } from 'stream'; + +/** + * Object resolved after a test pipeline is executed. + * @beta + */ +export type TestPipelineResult = { + /** + * If an error was emitted by the pipeline, it will be set here. + */ + error: unknown; + + /** + * A list of documents collected at the end of the pipeline. If the subject + * under test is an indexer, this will be an empty array (because your + * indexer should have received the documents instead). + */ + documents: IndexableDocument[]; +}; + +/** + * Test utility for Backstage Search collators, decorators, and indexers. + * @beta + */ +export class TestPipeline { + private collator?: Readable; + private decorator?: Transform; + private indexer?: Writable; + + private constructor({ + collator, + decorator, + indexer, + }: { + collator?: Readable; + decorator?: Transform; + indexer?: Writable; + }) { + this.collator = collator; + this.decorator = decorator; + this.indexer = indexer; + } + + /** + * Provide the collator, decorator, or indexer to be tested. + */ + static withSubject(subject: Readable | Transform | Writable) { + if (subject instanceof Transform) { + return new TestPipeline({ decorator: subject }); + } + + if (subject instanceof Readable) { + return new TestPipeline({ collator: subject }); + } + + if (subject instanceof Writable) { + return new TestPipeline({ indexer: subject }); + } + + throw new Error( + 'Unknown test subject: are you passing a readable, writable, or transform stream?', + ); + } + + /** + * Provide documents for testing decorators and indexers. + */ + withDocuments(documents: IndexableDocument[]): TestPipeline { + if (this.collator) { + throw new Error('Cannot provide documents when testing a collator.'); + } + + // Set a naive readable stream that just pushes all given documents. + this.collator = new Readable({ objectMode: true }); + this.collator._read = () => {}; + process.nextTick(() => { + documents.forEach(document => { + this.collator!.push(document); + }); + this.collator!.push(null); + }); + + return this; + } + + /** + * Execute the test pipeline so that you can make assertions about the result + * or behavior of the given test subject. + */ + async execute(): Promise { + const documents: IndexableDocument[] = []; + if (!this.collator) { + throw new Error( + 'Cannot execute pipeline without a collator or documents', + ); + } + + // If we are here and there is no indexer, we are testing a collator or a + // decorator. Set up a naive writable that captures documents in memory. + if (!this.indexer) { + this.indexer = new Writable({ objectMode: true }); + this.indexer._write = (document: IndexableDocument, _, done) => { + documents.push(document); + done(); + }; + } + + return new Promise(done => { + const pipes: (Readable | Transform | Writable)[] = [this.collator!]; + if (this.decorator) { + pipes.push(this.decorator); + } + pipes.push(this.indexer!); + + pipeline(pipes, error => { + done({ + error, + documents, + }); + }); + }); + } +} diff --git a/plugins/search-backend-node/src/test-utils/index.ts b/plugins/search-backend-node/src/test-utils/index.ts new file mode 100644 index 0000000000..185d25433b --- /dev/null +++ b/plugins/search-backend-node/src/test-utils/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { TestPipeline } from './TestPipeline'; +export type { TestPipelineResult } from './TestPipeline'; diff --git a/plugins/search-backend-node/src/types.ts b/plugins/search-backend-node/src/types.ts index df83357f6c..68ccfe4c7e 100644 --- a/plugins/search-backend-node/src/types.ts +++ b/plugins/search-backend-node/src/types.ts @@ -14,10 +14,24 @@ * limitations under the License. */ -import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; +import { + DocumentCollatorFactory, + DocumentDecoratorFactory, + SearchEngine, +} from '@backstage/search-common'; +import { Logger } from 'winston'; + +/** + * @beta + */ +export type IndexBuilderOptions = { + searchEngine: SearchEngine; + logger: Logger; +}; /** * Parameters required to register a collator. + * @beta */ export interface RegisterCollatorParameters { /** @@ -26,17 +40,18 @@ export interface RegisterCollatorParameters { defaultRefreshIntervalSeconds: number; /** - * The collator class responsible for returning all documents of the given type. + * The class responsible for returning the document collator of the given type. */ - collator: DocumentCollator; + factory: DocumentCollatorFactory; } /** * Parameters required to register a decorator + * @beta */ export interface RegisterDecoratorParameters { /** - * The decorator class responsible for appending or modifying documents of the given type(s). + * The class responsible for returning the decorator which appends, modifies, or filters documents. */ - decorator: DocumentDecorator; + factory: DocumentDecoratorFactory; } diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts index 0a7b45e164..137ffc24f1 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts @@ -62,7 +62,7 @@ describe('AuthorizedSearchEngine', () => { setTranslator: () => { throw new Error('Function not implemented. 1'); }, - index: () => { + getIndexer: () => { throw new Error('Function not implemented.2'); }, query: mockedQuery, diff --git a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts index d0488af00d..82ed061196 100644 --- a/plugins/search-backend/src/service/AuthorizedSearchEngine.ts +++ b/plugins/search-backend/src/service/AuthorizedSearchEngine.ts @@ -25,7 +25,6 @@ import { } from '@backstage/plugin-permission-common'; import { DocumentTypeInfo, - IndexableDocument, QueryRequestOptions, QueryTranslator, SearchEngine, @@ -35,6 +34,7 @@ import { } from '@backstage/search-common'; import { Config } from '@backstage/config'; import { InputError } from '@backstage/errors'; +import { Writable } from 'stream'; export function decodePageCursor(pageCursor?: string): { page: number } { if (!pageCursor) { @@ -78,8 +78,8 @@ export class AuthorizedSearchEngine implements SearchEngine { this.searchEngine.setTranslator(translator); } - async index(type: string, documents: IndexableDocument[]): Promise { - this.searchEngine.index(type, documents); + async getIndexer(type: string): Promise { + return this.searchEngine.getIndexer(type); } async query( diff --git a/plugins/search-backend/src/service/router.test.ts b/plugins/search-backend/src/service/router.test.ts index 0a94250aeb..bdf46a240b 100644 --- a/plugins/search-backend/src/service/router.test.ts +++ b/plugins/search-backend/src/service/router.test.ts @@ -105,7 +105,7 @@ describe('createRouter', () => { beforeAll(async () => { const logger = getVoidLogger(); mockSearchEngine = { - index: jest.fn(), + getIndexer: jest.fn(), setTranslator: jest.fn(), query: jest.fn(), }; diff --git a/plugins/tech-insights-backend/src/service/router.ts b/plugins/tech-insights-backend/src/service/router.ts index b13802358f..b86677ab90 100644 --- a/plugins/tech-insights-backend/src/service/router.ts +++ b/plugins/tech-insights-backend/src/service/router.ts @@ -27,7 +27,7 @@ import { Logger } from 'winston'; import { DateTime } from 'luxon'; import { PersistenceContext } from './persistence/persistenceContext'; import { - EntityName, + CompoundEntityRef, parseEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -99,8 +99,10 @@ export async function createRouter< }); router.post('/checks/run', async (req, res) => { - const { checks, entities }: { checks: string[]; entities: EntityName[] } = - req.body; + const { + checks, + entities, + }: { checks: string[]; entities: CompoundEntityRef[] } = req.body; const tasks = entities.map(async entity => { const entityTriplet = typeof entity === 'string' ? entity : stringifyEntityRef(entity); diff --git a/plugins/tech-insights/api-report.md b/plugins/tech-insights/api-report.md index 79854e505f..deadd06010 100644 --- a/plugins/tech-insights/api-report.md +++ b/plugins/tech-insights/api-report.md @@ -9,7 +9,7 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; import { BulkCheckResponse } from '@backstage/plugin-tech-insights-common'; import { CheckResult } from '@backstage/plugin-tech-insights-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { default as React_2 } from 'react'; import { RouteRef } from '@backstage/core-plugin-api'; @@ -52,11 +52,14 @@ export interface TechInsightsApi { ) => CheckResultRenderer | undefined; // (undocumented) runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; // (undocumented) - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; } // @public diff --git a/plugins/tech-insights/src/api/TechInsightsApi.ts b/plugins/tech-insights/src/api/TechInsightsApi.ts index b1287a62dc..0a6bb937a6 100644 --- a/plugins/tech-insights/src/api/TechInsightsApi.ts +++ b/plugins/tech-insights/src/api/TechInsightsApi.ts @@ -21,7 +21,7 @@ import { } from '@backstage/plugin-tech-insights-common'; import { Check } from './types'; import { CheckResultRenderer } from '../components/CheckResultRenderer'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; /** * {@link @backstage/core-plugin-api#ApiRef} for the {@link TechInsightsApi} @@ -45,9 +45,12 @@ export interface TechInsightsApi { description?: string, ) => CheckResultRenderer | undefined; getAllChecks(): Promise; - runChecks(entityParams: EntityName, checks?: Check[]): Promise; + runChecks( + entityParams: CompoundEntityRef, + checks?: Check[], + ): Promise; runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise; } diff --git a/plugins/tech-insights/src/api/TechInsightsClient.ts b/plugins/tech-insights/src/api/TechInsightsClient.ts index 5e29749854..5b9af9557f 100644 --- a/plugins/tech-insights/src/api/TechInsightsClient.ts +++ b/plugins/tech-insights/src/api/TechInsightsClient.ts @@ -22,7 +22,7 @@ import { import { Check } from './types'; import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api'; import { ResponseError } from '@backstage/errors'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { CheckResultRenderer, @@ -74,7 +74,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runChecks( - entityParams: EntityName, + entityParams: CompoundEntityRef, checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); @@ -102,7 +102,7 @@ export class TechInsightsClient implements TechInsightsApi { } async runBulkChecks( - entities: EntityName[], + entities: CompoundEntityRef[], checks?: Check[], ): Promise { const url = await this.discoveryApi.getBaseUrl('tech-insights'); diff --git a/plugins/techdocs-backend/api-report.md b/plugins/techdocs-backend/api-report.md index 5d106cfafb..155f096614 100644 --- a/plugins/techdocs-backend/api-report.md +++ b/plugins/techdocs-backend/api-report.md @@ -3,9 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; -import { DocumentCollator } from '@backstage/search-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { Entity } from '@backstage/catalog-model'; import express from 'express'; import { GeneratorBuilder } from '@backstage/techdocs-common'; import { Knex } from 'knex'; @@ -15,14 +18,15 @@ import { PluginCacheManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PreparerBuilder } from '@backstage/techdocs-common'; import { PublisherBase } from '@backstage/techdocs-common'; +import { Readable } from 'stream'; import { TechDocsDocument } from '@backstage/techdocs-common'; import { TokenManager } from '@backstage/backend-common'; // @public export function createRouter(options: RouterOptions): Promise; -// @public -export class DefaultTechDocsCollator implements DocumentCollator { +// @public @deprecated +export class DefaultTechDocsCollator { // (undocumented) protected applyArgsToFormat( format: string, @@ -41,6 +45,27 @@ export class DefaultTechDocsCollator implements DocumentCollator { readonly visibilityPermission: Permission; } +// @public +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + // (undocumented) + static fromConfig( + config: Config, + options: TechDocsCollatorFactoryOptions, + ): DefaultTechDocsCollatorFactory; + // (undocumented) + getCollator(): Promise; + // (undocumented) + readonly type: string; + // (undocumented) + readonly visibilityPermission: Permission; +} + +// @public +export interface DocsBuildStrategy { + // (undocumented) + shouldBuild(params: ShouldBuildParameters): Promise; +} + // @public export type OutOfTheBoxDeploymentOptions = { preparers: PreparerBuilder; @@ -51,6 +76,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -60,6 +86,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; // @public @@ -67,6 +94,22 @@ export type RouterOptions = | RecommendedDeploymentOptions | OutOfTheBoxDeploymentOptions; +// @public +export type ShouldBuildParameters = { + entity: Entity; +}; + +// @public +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger_2; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + // @public export type TechDocsCollatorOptions = { discovery: PluginEndpointDiscovery; diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 7e8241bfea..20100f4848 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@backstage/cli": "^0.14.1", + "@backstage/plugin-search-backend-node": "0.4.7", "@backstage/test-utils": "^0.2.6", "@types/dockerode": "^3.3.0", "msw": "^0.35.0", diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 12e7df1e9e..570da95092 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -25,10 +25,18 @@ export type { RouterOptions, RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, + DocsBuildStrategy, + ShouldBuildParameters, } from './service'; -export { DefaultTechDocsCollator } from './search'; -export type { TechDocsCollatorOptions } from './search'; +export { + DefaultTechDocsCollator, + DefaultTechDocsCollatorFactory, +} from './search'; +export type { + TechDocsCollatorFactoryOptions, + TechDocsCollatorOptions, +} from './search'; /** * @deprecated Use directly from @backstage/techdocs-common diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts index d9f75767a7..887dc60b1a 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts @@ -72,18 +72,6 @@ const expectedEntities: Entity[] = [ owner: 'someone', }, }, - { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'test-entity', - description: 'The expected description', - }, - spec: { - type: 'some-type', - lifecycle: 'experimental', - }, - }, ]; describe('DefaultTechDocsCollator with legacyPathCasing configuration', () => { diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts index 73c030cc51..d36e1c0ae5 100644 --- a/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts @@ -24,7 +24,6 @@ import { RELATION_OWNED_BY, stringifyEntityRef, } from '@backstage/catalog-model'; -import { DocumentCollator } from '@backstage/search-common'; import fetch from 'node-fetch'; import unescape from 'lodash/unescape'; import { Logger } from 'winston'; @@ -69,8 +68,10 @@ type EntityInfo = { * A search collator responsible for gathering and transforming TechDocs documents. * * @public + * @deprecated Upgrade to a more recent `@backstage/search-backend-node` and + * use `DefaultTechDocsCollatorFactory` instead. */ -export class DefaultTechDocsCollator implements DocumentCollator { +export class DefaultTechDocsCollator { public readonly type: string = 'techdocs'; public readonly visibilityPermission = catalogEntityReadPermission; diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts new file mode 100644 index 0000000000..fafaae71b9 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.test.ts @@ -0,0 +1,245 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + getVoidLogger, + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { TestPipeline } from '@backstage/plugin-search-backend-node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; +import { Readable } from 'stream'; +import { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; + +const logger = getVoidLogger(); + +const mockSearchDocIndex = { + config: { + lang: ['en'], + min_search_length: 3, + prebuild_index: false, + separator: '[\\s\\-]+', + }, + docs: [ + { + location: '', + text: 'docs docs docs', + title: 'Home', + }, + { + location: 'local-development/', + text: 'Docs for first subtitle', + title: 'Local development', + }, + { + location: 'local-development/#development', + text: 'Docs for sub-subtitle', + title: 'Development', + }, + ], +}; + +const expectedEntities: Entity[] = [ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + title: 'Test Entity with Docs!', + name: 'test-entity-with-docs', + description: 'Documented description', + annotations: { + 'backstage.io/techdocs-ref': './', + }, + }, + spec: { + type: 'dog', + lifecycle: 'experimental', + owner: 'someone', + }, + }, +]; + +describe('DefaultTechDocsCollatorFactory', () => { + const config = new ConfigReader({}); + const mockDiscoveryApi: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValue('http://test-backend'), + getExternalBaseUrl: jest.fn(), + }; + const mockTokenManager: jest.Mocked = { + getToken: jest.fn().mockResolvedValue({ token: '' }), + authenticate: jest.fn(), + }; + const options = { + discovery: mockDiscoveryApi, + logger: getVoidLogger(), + tokenManager: mockTokenManager, + }; + + it('has expected type', () => { + const factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + expect(factory.type).toBe('techdocs'); + }); + + describe('getCollator', () => { + let factory: DefaultTechDocsCollatorFactory; + let collator: Readable; + + const worker = setupServer(); + setupRequestMockHandlers(worker); + + beforeEach(async () => { + factory = DefaultTechDocsCollatorFactory.fromConfig(config, options); + collator = await factory.getCollator(); + + worker.use( + rest.get( + 'http://test-backend/static/docs/default/Component/test-entity-with-docs/search/search_index.json', + (_, res, ctx) => res(ctx.status(200), ctx.json(mockSearchDocIndex)), + ), + rest.get('http://test-backend/entities', (req, res, ctx) => { + // Imitate offset/limit pagination. + const offset = parseInt( + req.url.searchParams.get('offset') || '0', + 10, + ); + const limit = parseInt( + req.url.searchParams.get('limit') || '500', + 10, + ); + + // Limit 50 corresponds to a case testing pagination. + if (limit === 50) { + // Return 50 copies of invalid entities on the first request. + if (offset === 0) { + return res(ctx.status(200), ctx.json(Array(50).fill({}))); + } + // Then just the regular 2 on the second. + return res(ctx.status(200), ctx.json(expectedEntities)); + } + return res( + ctx.status(200), + ctx.json(expectedEntities.slice(offset, limit + offset)), + ); + }), + ); + }); + + it('returns a readable stream', async () => { + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured catalog and tech docs services', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('catalog'); + expect(mockDiscoveryApi.getBaseUrl).toHaveBeenCalledWith('techdocs'); + expect(documents).toHaveLength(mockSearchDocIndex.docs.length); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind.toLocaleLowerCase('en-US'), + name: entity.metadata.name, + }); + }); + }); + + it('maps a returned entity with a custom locationTemplate', async () => { + // Provide an alternate location template. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + discovery: mockDiscoveryApi, + tokenManager: mockTokenManager, + locationTemplate: '/software/:name', + logger, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + expect(documents[0]).toMatchObject({ + location: '/software/test-entity-with-docs', + }); + }); + + it('paginates through catalog entities using batchSize', async () => { + // A parallelismLimit of 1 is a catalog limit of 50 per request. Code + // above in the /entities handler ensures valid entities are only + // returned on the second page. + factory = DefaultTechDocsCollatorFactory.fromConfig(config, { + ...options, + parallelismLimit: 1, + }); + collator = await factory.getCollator(); + + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + + // Only 1 entity with TechDocs configured multipled by 3 pages. + expect(documents).toHaveLength(3); + }); + + describe('with legacyPathCasing configuration', () => { + beforeEach(async () => { + const legacyConfig = new ConfigReader({ + techdocs: { + legacyUseCaseSensitiveTripletPaths: true, + }, + }); + factory = DefaultTechDocsCollatorFactory.fromConfig( + legacyConfig, + options, + ); + collator = await factory.getCollator(); + }); + + it('should create documents for each tech docs search index', async () => { + const pipeline = TestPipeline.withSubject(collator); + const { documents } = await pipeline.execute(); + const entity = expectedEntities[0]; + documents.forEach((document, idx) => { + expect(document).toMatchObject({ + title: mockSearchDocIndex.docs[idx].title, + location: `/docs/default/Component/${entity.metadata.name}/${mockSearchDocIndex.docs[idx].location}`, + text: mockSearchDocIndex.docs[idx].text, + namespace: 'default', + entityTitle: entity!.metadata.title, + componentType: entity!.spec!.type, + lifecycle: entity!.spec!.lifecycle, + owner: '', + kind: entity.kind, + name: entity.metadata.name, + }); + }); + }); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts new file mode 100644 index 0000000000..181f032b54 --- /dev/null +++ b/plugins/techdocs-backend/src/search/DefaultTechDocsCollatorFactory.ts @@ -0,0 +1,254 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginEndpointDiscovery, + TokenManager, +} from '@backstage/backend-common'; +import { + CatalogApi, + CatalogClient, + CATALOG_FILTER_EXISTS, +} from '@backstage/catalog-client'; +import { + Entity, + parseEntityRef, + RELATION_OWNED_BY, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common'; +import { DocumentCollatorFactory } from '@backstage/search-common'; +import { TechDocsDocument } from '@backstage/techdocs-common'; +import unescape from 'lodash/unescape'; +import fetch from 'node-fetch'; +import pLimit from 'p-limit'; +import { Readable } from 'stream'; +import { Logger } from 'winston'; + +interface MkSearchIndexDoc { + title: string; + text: string; + location: string; +} + +/** + * Options to configure the TechDocs collator factory + * + * @public + */ +export type TechDocsCollatorFactoryOptions = { + discovery: PluginEndpointDiscovery; + logger: Logger; + tokenManager: TokenManager; + locationTemplate?: string; + catalogClient?: CatalogApi; + parallelismLimit?: number; + legacyPathCasing?: boolean; +}; + +type EntityInfo = { + name: string; + namespace: string; + kind: string; +}; + +/** + * A search collator factory responsible for gathering and transforming + * TechDocs documents. + * + * @public + */ +export class DefaultTechDocsCollatorFactory implements DocumentCollatorFactory { + public readonly type: string = 'techdocs'; + public readonly visibilityPermission = catalogEntityReadPermission; + + private discovery: PluginEndpointDiscovery; + private locationTemplate: string; + private readonly logger: Logger; + private readonly catalogClient: CatalogApi; + private readonly tokenManager: TokenManager; + private readonly parallelismLimit: number; + private readonly legacyPathCasing: boolean; + + private constructor(options: TechDocsCollatorFactoryOptions) { + this.discovery = options.discovery; + this.locationTemplate = + options.locationTemplate || '/docs/:namespace/:kind/:name/:path'; + this.logger = options.logger; + this.catalogClient = + options.catalogClient || + new CatalogClient({ discoveryApi: options.discovery }); + this.parallelismLimit = options.parallelismLimit ?? 10; + this.legacyPathCasing = options.legacyPathCasing ?? false; + this.tokenManager = options.tokenManager; + } + + static fromConfig(config: Config, options: TechDocsCollatorFactoryOptions) { + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + return new DefaultTechDocsCollatorFactory({ ...options, legacyPathCasing }); + } + + async getCollator(): Promise { + return Readable.from(this.execute()); + } + + private async *execute(): AsyncGenerator { + const limit = pLimit(this.parallelismLimit); + const techDocsBaseUrl = await this.discovery.getBaseUrl('techdocs'); + const { token } = await this.tokenManager.getToken(); + let entitiesRetrieved = 0; + let moreEntitiesToGet = true; + + // Offset/limit pagination is used on the Catalog Client in order to + // limit (and allow some control over) memory used by the search backend + // at index-time. The batchSize is calculated as a factor of the given + // parallelism limit to simplify configuration. + const batchSize = this.parallelismLimit * 50; + while (moreEntitiesToGet) { + const entities = ( + await this.catalogClient.getEntities( + { + filter: { + 'metadata.annotations.backstage.io/techdocs-ref': + CATALOG_FILTER_EXISTS, + }, + fields: [ + 'kind', + 'namespace', + 'metadata.annotations', + 'metadata.name', + 'metadata.title', + 'metadata.namespace', + 'spec.type', + 'spec.lifecycle', + 'relations', + ], + limit: batchSize, + offset: entitiesRetrieved, + }, + { token }, + ) + ).items; + + // Control looping through entity batches. + moreEntitiesToGet = entities.length === batchSize; + entitiesRetrieved += entities.length; + + const docPromises = entities + .filter(it => it.metadata?.annotations?.['backstage.io/techdocs-ref']) + .map((entity: Entity) => + limit(async (): Promise => { + const entityInfo = + DefaultTechDocsCollatorFactory.handleEntityInfoCasing( + this.legacyPathCasing, + { + kind: entity.kind, + namespace: entity.metadata.namespace || 'default', + name: entity.metadata.name, + }, + ); + + try { + const searchIndexResponse = await fetch( + DefaultTechDocsCollatorFactory.constructDocsIndexUrl( + techDocsBaseUrl, + entityInfo, + ), + { + headers: { + Authorization: `Bearer ${token}`, + }, + }, + ); + const searchIndex = await searchIndexResponse.json(); + + return searchIndex.docs.map((doc: MkSearchIndexDoc) => ({ + title: unescape(doc.title), + text: unescape(doc.text || ''), + location: this.applyArgsToFormat( + this.locationTemplate || '/docs/:namespace/:kind/:name/:path', + { + ...entityInfo, + path: doc.location, + }, + ), + path: doc.location, + ...entityInfo, + entityTitle: entity.metadata.title, + componentType: entity.spec?.type?.toString() || 'other', + lifecycle: (entity.spec?.lifecycle as string) || '', + owner: getSimpleEntityOwnerString(entity), + authorization: { + resourceRef: stringifyEntityRef(entity), + }, + })); + } catch (e) { + this.logger.debug( + `Failed to retrieve tech docs search index for entity ${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}`, + e, + ); + return []; + } + }), + ); + yield* (await Promise.all(docPromises)).flat(); + } + } + + private applyArgsToFormat( + format: string, + args: Record, + ): string { + let formatted = format; + for (const [key, value] of Object.entries(args)) { + formatted = formatted.replace(`:${key}`, value); + } + return formatted; + } + + private static constructDocsIndexUrl( + techDocsBaseUrl: string, + entityInfo: { kind: string; namespace: string; name: string }, + ) { + return `${techDocsBaseUrl}/static/docs/${entityInfo.namespace}/${entityInfo.kind}/${entityInfo.name}/search/search_index.json`; + } + + private static handleEntityInfoCasing( + legacyPaths: boolean, + entityInfo: EntityInfo, + ): EntityInfo { + return legacyPaths + ? entityInfo + : Object.entries(entityInfo).reduce((acc, [key, value]) => { + return { ...acc, [key]: value.toLocaleLowerCase('en-US') }; + }, {} as EntityInfo); + } +} + +function getSimpleEntityOwnerString(entity: Entity): string { + if (entity.relations) { + const owner = entity.relations.find(r => r.type === RELATION_OWNED_BY); + if (owner) { + const { name } = parseEntityRef(owner.targetRef); + return name; + } + } + return ''; +} diff --git a/plugins/techdocs-backend/src/search/index.ts b/plugins/techdocs-backend/src/search/index.ts index fbbd23b964..68e3b4edc7 100644 --- a/plugins/techdocs-backend/src/search/index.ts +++ b/plugins/techdocs-backend/src/search/index.ts @@ -13,6 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; +export { DefaultTechDocsCollatorFactory } from './DefaultTechDocsCollatorFactory'; +export type { TechDocsCollatorFactoryOptions } from './DefaultTechDocsCollatorFactory'; + +/** + * todo(backstage/techdocs-core): stop exporting these in a future release. + */ +export { DefaultTechDocsCollator } from './DefaultTechDocsCollator'; export type { TechDocsCollatorOptions } from './DefaultTechDocsCollator'; diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts index 678a4b0539..19684486dc 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.test.ts @@ -16,11 +16,11 @@ import { CachedEntityLoader } from './CachedEntityLoader'; import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; describe('CachedEntityLoader', () => { const catalog: jest.Mocked = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), } as any; const cache: jest.Mocked = { @@ -28,7 +28,7 @@ describe('CachedEntityLoader', () => { set: jest.fn(), } as any; - const entityName: EntityName = { + const entityName: CompoundEntityRef = { kind: 'component', namespace: 'default', name: 'test', @@ -53,7 +53,7 @@ describe('CachedEntityLoader', () => { it('writes entities to cache', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); @@ -71,12 +71,12 @@ describe('CachedEntityLoader', () => { const result = await loader.load(entityName, token); expect(result).toEqual(entity); - expect(catalog.getEntityByName).not.toBeCalled(); + expect(catalog.getEntityByRef).not.toBeCalled(); }); it('does not cache missing entites', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(undefined); + catalog.getEntityByRef.mockResolvedValue(undefined); const result = await loader.load(entityName, token); @@ -86,7 +86,7 @@ describe('CachedEntityLoader', () => { it('uses entity ref as cache key for anonymous users', async () => { cache.get.mockResolvedValue(undefined); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, undefined); @@ -103,7 +103,7 @@ describe('CachedEntityLoader', () => { setTimeout(() => resolve(undefined), 10000); }), ); - catalog.getEntityByName.mockResolvedValue(entity); + catalog.getEntityByRef.mockResolvedValue(entity); const result = await loader.load(entityName, token); diff --git a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts index 81c6ec7431..424d9b9541 100644 --- a/plugins/techdocs-backend/src/service/CachedEntityLoader.ts +++ b/plugins/techdocs-backend/src/service/CachedEntityLoader.ts @@ -17,7 +17,7 @@ import { CatalogClient } from '@backstage/catalog-client'; import { CacheClient } from '@backstage/backend-common'; import { Entity, - EntityName, + CompoundEntityRef, stringifyEntityRef, } from '@backstage/catalog-model'; @@ -37,17 +37,17 @@ export class CachedEntityLoader { } async load( - entityName: EntityName, + entityRef: CompoundEntityRef, token: string | undefined, ): Promise { - const cacheKey = this.getCacheKey(entityName, token); + const cacheKey = this.getCacheKey(entityRef, token); let result = await this.getFromCache(cacheKey); if (result) { return result; } - result = await this.catalog.getEntityByName(entityName, { token }); + result = await this.catalog.getEntityByRef(entityRef, { token }); if (result) { this.cache.set(cacheKey, result, { ttl: 5000 }); @@ -66,7 +66,7 @@ export class CachedEntityLoader { } private getCacheKey( - entityName: EntityName, + entityName: CompoundEntityRef, token: string | undefined, ): string { const key = ['catalog', stringifyEntityRef(entityName)]; diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts new file mode 100644 index 0000000000..84bd960f8f --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DefaultDocsBuildStrategy } from './DocsBuildStrategy'; +import { ConfigReader } from '@backstage/config'; + +const MockedConfigReader = ConfigReader as jest.MockedClass< + typeof ConfigReader +>; + +jest.mock('@backstage/config'); + +describe('DefaultDocsBuildStrategy', () => { + const entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + uid: '0', + name: 'test', + }, + }; + + const config = new ConfigReader({}); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('shouldBuild', () => { + it('should return true when techdocs.build is set to local', async () => { + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); + + MockedConfigReader.prototype.getString.mockReturnValue('local'); + + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); + + expect(result).toBe(true); + }); + + it('should return false when techdocs.build is set to external', async () => { + const defaultDocsBuildStrategy = + DefaultDocsBuildStrategy.fromConfig(config); + + MockedConfigReader.prototype.getString.mockReturnValue('external'); + + const result = await defaultDocsBuildStrategy.shouldBuild({ entity }); + + expect(result).toBe(false); + }); + }); +}); diff --git a/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts new file mode 100644 index 0000000000..42a16234f9 --- /dev/null +++ b/plugins/techdocs-backend/src/service/DocsBuildStrategy.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; + +/** + * Parameters passed to the shouldBuild method on the DocsBuildStrategy interface + * + * @public + */ +export type ShouldBuildParameters = { + entity: Entity; +}; + +/** + * A strategy for when to build TechDocs locally, and when to skip building TechDocs (allowing for an external build) + * + * @public + */ +export interface DocsBuildStrategy { + shouldBuild(params: ShouldBuildParameters): Promise; +} + +export class DefaultDocsBuildStrategy { + private readonly config: Config; + + private constructor(config: Config) { + this.config = config; + } + + static fromConfig(config: Config): DefaultDocsBuildStrategy { + return new DefaultDocsBuildStrategy(config); + } + + async shouldBuild(_: ShouldBuildParameters): Promise { + return this.config.getString('techdocs.builder') === 'local'; + } +} diff --git a/plugins/techdocs-backend/src/service/index.ts b/plugins/techdocs-backend/src/service/index.ts index 0065e33a2a..7355a34e32 100644 --- a/plugins/techdocs-backend/src/service/index.ts +++ b/plugins/techdocs-backend/src/service/index.ts @@ -20,3 +20,7 @@ export type { RecommendedDeploymentOptions, OutOfTheBoxDeploymentOptions, } from './router'; +export type { + DocsBuildStrategy, + ShouldBuildParameters, +} from './DocsBuildStrategy'; diff --git a/plugins/techdocs-backend/src/service/router.test.ts b/plugins/techdocs-backend/src/service/router.test.ts index 43e9641efb..c7cda203bb 100644 --- a/plugins/techdocs-backend/src/service/router.test.ts +++ b/plugins/techdocs-backend/src/service/router.test.ts @@ -38,6 +38,7 @@ import { RouterOptions, } from './router'; import { TechDocsCache } from '../cache'; +import { DocsBuildStrategy } from './DocsBuildStrategy'; jest.mock('@backstage/catalog-client'); jest.mock('@backstage/config'); @@ -120,6 +121,9 @@ describe('createRouter', () => { const cache: jest.Mocked = { getClient: jest.fn(), }; + const docsBuildStrategy: jest.Mocked = { + shouldBuild: jest.fn(), + }; const outOfTheBoxOptions = { preparers, generators, @@ -128,6 +132,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; const recommendedOptions = { publisher, @@ -135,6 +140,7 @@ describe('createRouter', () => { logger: getVoidLogger(), discovery, cache, + docsBuildStrategy, }; beforeEach(() => { @@ -181,10 +187,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -198,10 +204,10 @@ describe('createRouter', () => { expect(response.status).toBe(304); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -210,7 +216,7 @@ describe('createRouter', () => { expect(response.status).toBe(500); expect(response.text).toMatch( - /Invalid configuration\. 'techdocs\.builder' was set to 'local' but no 'preparer' was provided to the router initialization/, + /Invalid configuration\. docsBuildStrategy\.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization./, ); expect(MockDocsSynchronizer.prototype.doSync).toBeCalledTimes(0); @@ -219,7 +225,7 @@ describe('createRouter', () => { it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -244,7 +250,7 @@ describe('createRouter', () => { it('should return on updated', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { @@ -297,10 +303,10 @@ describe('createRouter', () => { expect(response.status).toBe(404); }); - it('should not check for an update without local builder', async () => { + it('should not check for an update when shouldBuild returns false', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('external'); + docsBuildStrategy.shouldBuild.mockResolvedValue(false); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doCacheSync.mockImplementation( async ({ responseHandler }) => @@ -322,10 +328,10 @@ data: {"updated":false} ); }); - it('should error if missing builder', async () => { + it('should error if build is required and is missing preparer', async () => { const app = await createApp(recommendedOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); const response = await request(app) @@ -337,7 +343,7 @@ data: {"updated":false} expect(response.get('content-type')).toBe('text/event-stream'); expect(response.text).toEqual( `event: error -data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization." +data: "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization." `, ); @@ -348,7 +354,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should execute synchronization', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => @@ -376,7 +382,7 @@ data: "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'prep it('should return an event-stream', async () => { const app = await createApp(outOfTheBoxOptions); - MockedConfigReader.prototype.getString.mockReturnValue('local'); + docsBuildStrategy.shouldBuild.mockResolvedValue(true); MockCachedEntityLoader.prototype.load.mockResolvedValue(entity); MockDocsSynchronizer.prototype.doSync.mockImplementation( async ({ responseHandler }) => { diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 5877dda099..73f212d049 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -35,6 +35,10 @@ import { ScmIntegrations } from '@backstage/integration'; import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer'; import { createCacheMiddleware, TechDocsCache } from '../cache'; import { CachedEntityLoader } from './CachedEntityLoader'; +import { + DefaultDocsBuildStrategy, + DocsBuildStrategy, +} from './DocsBuildStrategy'; /** * Required dependencies for running TechDocs in the "out-of-the-box" @@ -51,6 +55,7 @@ export type OutOfTheBoxDeploymentOptions = { database?: Knex; // TODO: Make database required when we're implementing database stuff. config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -65,6 +70,7 @@ export type RecommendedDeploymentOptions = { discovery: PluginEndpointDiscovery; config: Config; cache: PluginCacheManager; + docsBuildStrategy?: DocsBuildStrategy; }; /** @@ -99,6 +105,8 @@ export async function createRouter( const router = Router(); const { publisher, config, logger, discovery } = options; const catalogClient = new CatalogClient({ discoveryApi: discovery }); + const docsBuildStrategy = + options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config); // Entities are cached to optimize the /static/docs request path, which can be called many times // when loading a single techdocs page. @@ -210,10 +218,13 @@ export async function createRouter( responseHandler = createEventStream(res); } - // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' - // If set to 'external', it will assume that an external process (e.g. CI/CD pipeline - // of the repository) is responsible for building and publishing documentation to the storage provider - if (config.getString('techdocs.builder') !== 'local') { + // By default, techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to + // 'local'. If set to 'external', it will assume that an external process (e.g. CI/CD pipeline + // of the repository) is responsible for building and publishing documentation to the storage provider. + // Altering the implementation of the injected docsBuildStrategy allows for more complex behaviours, based on + // either config or the properties of the entity (e.g. annotations, labels, spec fields etc.). + const shouldBuild = await docsBuildStrategy.shouldBuild({ entity }); + if (!shouldBuild) { // However, if caching is enabled, take the opportunity to check and // invalidate stale cache entries. if (cache) { @@ -244,7 +255,7 @@ export async function createRouter( responseHandler.error( new Error( - "Invalid configuration. 'techdocs.builder' was set to 'local' but no 'preparer' was provided to the router initialization.", + "Invalid configuration. docsBuildStrategy.shouldBuild returned 'true', but no 'preparer' was provided to the router initialization.", ), ); }); diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 917c1d5d85..f7ef6e984a 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -7,11 +7,11 @@ import { ApiRef } from '@backstage/core-plugin-api'; import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { CSSProperties } from '@material-ui/styles'; import { DiscoveryApi } from '@backstage/core-plugin-api'; import { Entity } from '@backstage/catalog-model'; -import { EntityName } from '@backstage/catalog-model'; import { FetchApi } from '@backstage/core-plugin-api'; import { IdentityApi } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; @@ -89,7 +89,7 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; @@ -161,7 +161,7 @@ export const Reader: (props: ReaderProps) => JSX.Element; // @public export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -187,9 +187,11 @@ export type TabsConfig = TabConfig[]; export interface TechDocsApi { getApiOrigin(): Promise; // (undocumented) - getEntityMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; // (undocumented) - getTechDocsMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -208,8 +210,10 @@ export class TechDocsClient implements TechDocsApi { discoveryApi: DiscoveryApi; // (undocumented) getApiOrigin(): Promise; - getEntityMetadata(entityId: EntityName): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; } // @public @@ -297,7 +301,7 @@ export const TechDocsReaderPageHeader: ( // @public export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; @@ -315,7 +319,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; @@ -324,7 +328,7 @@ export const TechDocsSearch: (props: TechDocsSearchProps) => JSX.Element; // @public export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; @@ -348,18 +352,18 @@ export interface TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; // (undocumented) - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } @@ -384,18 +388,18 @@ export class TechDocsStorageClient implements TechDocsStorageApi { // (undocumented) getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; // (undocumented) getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; // (undocumented) getStorageUrl(): Promise; // (undocumented) identityApi: IdentityApi; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; } diff --git a/plugins/techdocs/dev/index.tsx b/plugins/techdocs/dev/index.tsx index 8dc0e5d30b..d08e914736 100644 --- a/plugins/techdocs/dev/index.tsx +++ b/plugins/techdocs/dev/index.tsx @@ -17,7 +17,7 @@ import { createDevApp } from '@backstage/dev-utils'; import { NotFoundError } from '@backstage/errors'; import React from 'react'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Reader, SyncResult, @@ -82,7 +82,10 @@ function createPage({ }); } - async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) { + async syncEntityDocs( + _: CompoundEntityRef, + logHandler?: (line: string) => void, + ) { if (syncDocsDelay) { for (let i = 0; i < 10; i++) { setTimeout( diff --git a/plugins/techdocs/src/api.ts b/plugins/techdocs/src/api.ts index 4566bbf028..b349431b0d 100644 --- a/plugins/techdocs/src/api.ts +++ b/plugins/techdocs/src/api.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsEntityMetadata, TechDocsMetadata } from './types'; import { createApiRef } from '@backstage/core-plugin-api'; @@ -55,14 +55,14 @@ export interface TechDocsStorageApi { getApiOrigin(): Promise; getStorageUrl(): Promise; getBuilder(): Promise; - getEntityDocs(entityId: EntityName, path: string): Promise; + getEntityDocs(entityId: CompoundEntityRef, path: string): Promise; syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler?: (line: string) => void, ): Promise; getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise; } @@ -77,6 +77,8 @@ export interface TechDocsApi { * Set to techdocs.requestUrl as the URL for techdocs-backend API. */ getApiOrigin(): Promise; - getTechDocsMetadata(entityId: EntityName): Promise; - getEntityMetadata(entityId: EntityName): Promise; + getTechDocsMetadata(entityId: CompoundEntityRef): Promise; + getEntityMetadata( + entityId: CompoundEntityRef, + ): Promise; } diff --git a/plugins/techdocs/src/client.ts b/plugins/techdocs/src/client.ts index eac5da0b6b..f260f00547 100644 --- a/plugins/techdocs/src/client.ts +++ b/plugins/techdocs/src/client.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { DiscoveryApi, @@ -62,7 +62,9 @@ export class TechDocsClient implements TechDocsApi { * * @param entityId - Object containing entity data like name, namespace, etc. */ - async getTechDocsMetadata(entityId: EntityName): Promise { + async getTechDocsMetadata( + entityId: CompoundEntityRef, + ): Promise { const { kind, namespace, name } = entityId; const apiOrigin = await this.getApiOrigin(); @@ -84,7 +86,7 @@ export class TechDocsClient implements TechDocsApi { * @param entityId - Object containing entity data like name, namespace, etc. */ async getEntityMetadata( - entityId: EntityName, + entityId: CompoundEntityRef, ): Promise { const { kind, namespace, name } = entityId; @@ -149,7 +151,10 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @returns HTML content of the docs page as string * @throws Throws error when the page is not found. */ - async getEntityDocs(entityId: EntityName, path: string): Promise { + async getEntityDocs( + entityId: CompoundEntityRef, + path: string, + ): Promise { const { kind, namespace, name } = entityId; const storageUrl = await this.getStorageUrl(); @@ -190,7 +195,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { * @throws Throws error on error from sync endpoint in Techdocs Backend */ async syncEntityDocs( - entityId: EntityName, + entityId: CompoundEntityRef, logHandler: (line: string) => void = () => {}, ): Promise { const { kind, namespace, name } = entityId; @@ -243,7 +248,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi { async getBaseUrl( oldBaseUrl: string, - entityId: EntityName, + entityId: CompoundEntityRef, path: string, ): Promise { const { kind, namespace, name } = entityId; diff --git a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx index 8618c4432f..14fdb88395 100644 --- a/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/DefaultTechDocsHome.test.tsx @@ -23,8 +23,8 @@ import { import { CatalogApi, catalogApiRef, - DefaultStarredEntitiesApi, starredEntitiesApiRef, + MockStarredEntitiesApi, } from '@backstage/plugin-catalog-react'; import { MockStorageApi, @@ -45,7 +45,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: () => Promise.resolve(), + getEntityByRef: () => Promise.resolve(), getEntities: async () => ({ items: [ { @@ -73,7 +73,7 @@ describe('TechDocs Home', () => { [catalogApiRef, mockCatalogApi], [configApiRef, configApi], [storageApiRef, storageApi], - [starredEntitiesApiRef, new DefaultStarredEntitiesApi({ storageApi })], + [starredEntitiesApiRef, new MockStarredEntitiesApi()], ); it('should render a TechDocs home page', async () => { diff --git a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx index a8fd472f7c..16d8d48bc3 100644 --- a/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx +++ b/plugins/techdocs/src/home/components/LegacyTechDocsHome.test.tsx @@ -33,7 +33,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx index b875ba4e98..883c496191 100644 --- a/plugins/techdocs/src/home/components/Tables/DocsTable.tsx +++ b/plugins/techdocs/src/home/components/Tables/DocsTable.tsx @@ -20,7 +20,7 @@ import useCopyToClipboard from 'react-use/lib/useCopyToClipboard'; import { useRouteRef, useApi, configApiRef } from '@backstage/core-plugin-api'; import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { - formatEntityRefTitle, + humanizeEntityRef, getEntityRelations, } from '@backstage/plugin-catalog-react'; import { rootDocsRouteRef } from '../../../routes'; @@ -76,7 +76,7 @@ export const DocsTable = (props: DocsTableProps) => { }), ownedByRelations, ownedByRelationsTitle: ownedByRelations - .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) + .map(r => humanizeEntityRef(r, { defaultKind: 'group' })) .join(', '), }, }; diff --git a/plugins/techdocs/src/home/components/Tables/actions.tsx b/plugins/techdocs/src/home/components/Tables/actions.tsx index d0dbb8bf47..3fd8881718 100644 --- a/plugins/techdocs/src/home/components/Tables/actions.tsx +++ b/plugins/techdocs/src/home/components/Tables/actions.tsx @@ -16,11 +16,16 @@ import React from 'react'; import ShareIcon from '@material-ui/icons/Share'; -import { - favoriteEntityIcon, - favoriteEntityTooltip, -} from '@backstage/plugin-catalog-react'; import { DocsTableRow } from './types'; +import { withStyles } from '@material-ui/styles'; +import Star from '@material-ui/icons/Star'; +import StarBorder from '@material-ui/icons/StarBorder'; + +const YellowStar = withStyles({ + root: { + color: '#f3ba37', + }, +})(Star); /** * Not directly exported, but through DocsTable.actions and EntityListDocsTable.actions @@ -46,8 +51,8 @@ export const actionFactories = { const isStarred = isStarredEntity(entity); return { cellStyle: { paddingLeft: '1em' }, - icon: () => favoriteEntityIcon(isStarred), - tooltip: favoriteEntityTooltip(isStarred), + icon: () => (isStarred ? : ), + tooltip: isStarred ? 'Remove from favorites' : 'Add to favorites', onClick: () => toggleStarredEntity(entity), }; }; diff --git a/plugins/techdocs/src/home/components/Tables/types.ts b/plugins/techdocs/src/home/components/Tables/types.ts index c076d4c12d..6c06526d07 100644 --- a/plugins/techdocs/src/home/components/Tables/types.ts +++ b/plugins/techdocs/src/home/components/Tables/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { Entity, CompoundEntityRef } from '@backstage/catalog-model'; /** * Generic representing the metadata structure for a docs table row. @@ -26,6 +26,6 @@ export type DocsTableRow = { resolved: { docsUrl: string; ownedByRelationsTitle: string; - ownedByRelations: EntityName[]; + ownedByRelations: CompoundEntityRef[]; }; }; diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx index e9c3a55b15..2088dbf4fc 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.test.tsx @@ -31,7 +31,7 @@ jest.mock('@backstage/plugin-catalog-react', () => { }); const mockCatalogApi = { - getEntityByName: jest.fn(), + getEntityByRef: jest.fn(), getEntities: async () => ({ items: [ { diff --git a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx index c8149f1ff2..d2f0266af6 100644 --- a/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx +++ b/plugins/techdocs/src/home/components/TechDocsCustomHome.tsx @@ -232,7 +232,12 @@ function useOwnUser(): AsyncState { return useAsync(async () => { const identity = await identityApi.getBackstageIdentity(); - return catalogApi.getEntityByName( + // TODO(freben): Defensively parse with defaults even though getEntityByRef + // supports the string form, since some auth resolvers have been known to + // return incomplete refs (just the name part) historically. This can be + // simplified in the future to just pass the ref immediately to + // getEntityByRef. + return catalogApi.getEntityByRef( parseEntityRef(identity.userEntityRef, { defaultKind: 'User', defaultNamespace: DEFAULT_NAMESPACE, diff --git a/plugins/techdocs/src/reader/components/Reader.tsx b/plugins/techdocs/src/reader/components/Reader.tsx index 4042585b6d..6aa5c99be4 100644 --- a/plugins/techdocs/src/reader/components/Reader.tsx +++ b/plugins/techdocs/src/reader/components/Reader.tsx @@ -34,7 +34,7 @@ import { alpha, } from '@material-ui/core'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, configApiRef } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; import { BackstageTheme } from '@backstage/theme'; @@ -68,7 +68,7 @@ import { useReaderState } from './useReaderState'; * @public */ export type ReaderProps = { - entityRef: EntityName; + entityRef: CompoundEntityRef; withSearch?: boolean; onReady?: () => void; }; @@ -95,7 +95,7 @@ const TechDocsReaderContext = createContext( const TechDocsReaderProvider = ({ children, entityRef, -}: PropsWithChildren<{ entityRef: EntityName }>) => { +}: PropsWithChildren<{ entityRef: CompoundEntityRef }>) => { const { '*': path } = useParams(); const { kind, namespace, name } = entityRef; const value = useReaderState(kind, namespace, name, path); @@ -116,7 +116,7 @@ const TechDocsReaderProvider = ({ * @internal */ export const withTechDocsReaderProvider = - (Component: ComponentType, entityRef: EntityName) => + (Component: ComponentType, entityRef: CompoundEntityRef) => (props: T) => ( @@ -157,7 +157,9 @@ const headings: TypographyHeadingsKeys[] = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']; * todo: Make public or stop exporting (see others: "altReaderExperiments") * @internal */ -export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { +export const useTechDocsReaderDom = ( + entityRef: CompoundEntityRef, +): Element | null => { const navigate = useNavigate(); const theme = useTheme(); const techdocsStorageApi = useApi(techdocsStorageApiRef); @@ -322,7 +324,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { --md-source-version-icon: url('data:image/svg+xml;charset=utf-8,'); --md-version-icon: url('data:image/svg+xml;charset=utf-8,'); } - + :host > * { /* CODE */ --md-code-fg-color: ${theme.palette.text.primary}; @@ -438,7 +440,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-main__inner { margin-top: 0; } - + .md-sidebar { height: calc(100% - 100px); position: fixed; @@ -450,13 +452,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-sidebar--secondary { right: ${theme.spacing(3)}px; } - + .md-content { max-width: calc(100% - 16rem * 2); margin-left: 16rem; margin-bottom: 50px; } - + .md-footer { position: fixed; bottom: 0px; @@ -471,7 +473,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-dialog { background-color: unset; } - + @media screen and (max-width: 76.1875em) { .md-nav { transition: none !important; @@ -567,7 +569,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { }), injectCss({ // Typeset - css: ` + css: ` .md-typeset { font-size: var(--md-typeset-font-size); } @@ -600,11 +602,11 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset .md-content__button { color: var(--md-default-fg-color); } - + .md-typeset hr { border-bottom: 0.05rem dotted ${theme.palette.divider}; } - + .md-typeset details { font-size: var(--md-typeset-font-size) !important; } @@ -621,7 +623,7 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .md-typeset details[open] > summary:after { transform: rotate(90deg) translateX(-50%) !important; } - + .md-typeset blockquote { color: var(--md-default-fg-color--light); border-left: 0.2rem solid var(--md-default-fg-color--light); @@ -667,13 +669,13 @@ export const useTechDocsReaderDom = (entityRef: EntityName): Element | null => { .highlight .md-clipboard:after { content: unset; } - + .highlight .nx { color: ${isDarkTheme ? '#ff53a3' : '#ec407a'}; } /* CODE HILITE */ - .codehilite .gd { + .codehilite .gd { background-color: ${ isDarkTheme ? 'rgba(248,81,73,0.65)' : '#fdd' }; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx index d03ea2ef61..184ab24b6d 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPage.tsx @@ -21,12 +21,12 @@ import useAsync from 'react-use/lib/useAsync'; import { techdocsApiRef } from '../../api'; import { LegacyTechDocsPage } from './LegacyTechDocsPage'; import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { useApi, useApp } from '@backstage/core-plugin-api'; import { Page } from '@backstage/core-components'; /** - * Helper function that gives the children of {@link TechDocsReaderPage} acccess to techdocs and entity metadata + * Helper function that gives the children of {@link TechDocsReaderPage} access to techdocs and entity metadata * * @public */ @@ -37,7 +37,7 @@ export type TechDocsReaderPageRenderFunction = ({ }: { techdocsMetadataValue?: TechDocsMetadata | undefined; entityMetadataValue?: TechDocsEntityMetadata | undefined; - entityRef: EntityName; + entityRef: CompoundEntityRef; onReady: () => void; }) => JSX.Element; diff --git a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx index 928833efed..5867641c48 100644 --- a/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsReaderPageHeader.tsx @@ -19,7 +19,7 @@ import CodeIcon from '@material-ui/icons/Code'; import { useRouteRef } from '@backstage/core-plugin-api'; import { Header, HeaderLabel } from '@backstage/core-components'; -import { EntityName, RELATION_OWNED_BY } from '@backstage/catalog-model'; +import { CompoundEntityRef, RELATION_OWNED_BY } from '@backstage/catalog-model'; import { EntityRefLink, EntityRefLinks, @@ -35,7 +35,7 @@ import { TechDocsEntityMetadata, TechDocsMetadata } from '../../types'; * @public */ export type TechDocsReaderPageHeaderProps = PropsWithChildren<{ - entityRef: EntityName; + entityRef: CompoundEntityRef; entityMetadata?: TechDocsEntityMetadata; techDocsMetadata?: TechDocsMetadata; }>; diff --git a/plugins/techdocs/src/reader/components/useRawPage.ts b/plugins/techdocs/src/reader/components/useRawPage.ts index 61b04a5a82..311f8554a9 100644 --- a/plugins/techdocs/src/reader/components/useRawPage.ts +++ b/plugins/techdocs/src/reader/components/useRawPage.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import useAsyncRetry from 'react-use/lib/useAsyncRetry'; import { AsyncState } from 'react-use/lib/useAsync'; import { techdocsStorageApiRef } from '../../api'; @@ -22,7 +22,7 @@ import { useApi } from '@backstage/core-plugin-api'; export type RawPage = { content: string; path: string; - entityId: EntityName; + entityId: CompoundEntityRef; }; export function useRawPage( diff --git a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts index f3906b7b91..2e3b853e42 100644 --- a/plugins/techdocs/src/reader/transformers/addBaseUrl.ts +++ b/plugins/techdocs/src/reader/transformers/addBaseUrl.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TechDocsStorageApi } from '../../api'; import type { Transformer } from './transformer'; type AddBaseUrlOptions = { techdocsStorageApi: TechDocsStorageApi; - entityId: EntityName; + entityId: CompoundEntityRef; path: string; }; diff --git a/plugins/techdocs/src/search/components/TechDocsSearch.tsx b/plugins/techdocs/src/search/components/TechDocsSearch.tsx index c817fb3217..30cc692b7c 100644 --- a/plugins/techdocs/src/search/components/TechDocsSearch.tsx +++ b/plugins/techdocs/src/search/components/TechDocsSearch.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { SearchContextProvider, useSearch } from '@backstage/plugin-search'; import { makeStyles, @@ -42,7 +42,7 @@ const useStyles = makeStyles({ * @public */ export type TechDocsSearchProps = { - entityId: EntityName; + entityId: CompoundEntityRef; debounceTime?: number; }; diff --git a/plugins/todo-backend/api-report.md b/plugins/todo-backend/api-report.md index 71094e03f8..f3f2af3fb9 100644 --- a/plugins/todo-backend/api-report.md +++ b/plugins/todo-backend/api-report.md @@ -4,8 +4,8 @@ ```ts import { CatalogApi } from '@backstage/catalog-client'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { EntityName } from '@backstage/catalog-model'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; import { ScmIntegrations } from '@backstage/integration'; @@ -19,7 +19,7 @@ export function createTodoParser(options?: TodoParserOptions): TodoParser; // @public (undocumented) export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { diff --git a/plugins/todo-backend/src/service/TodoReaderService.test.ts b/plugins/todo-backend/src/service/TodoReaderService.test.ts index 985687c6bb..943462bd63 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.test.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.test.ts @@ -44,6 +44,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { const mock = { addLocation: jest.fn(), getEntities: jest.fn(), + getEntityByRef: jest.fn(), getEntityByName: jest.fn(), getLocationByRef: jest.fn(), getLocationById: jest.fn(), @@ -54,7 +55,7 @@ function mockCatalogClient(entity?: Entity): jest.Mocked { getEntityFacets: jest.fn(), }; if (entity) { - mock.getEntityByName.mockReturnValue(entity); + mock.getEntityByRef.mockReturnValue(entity); } return mock; } @@ -93,7 +94,7 @@ describe('TodoReaderService', () => { offset: 0, limit: 10, }); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); @@ -304,7 +305,7 @@ describe('TodoReaderService', () => { message: 'Entity not found, component:default/my-component', }), ); - expect(catalogClient.getEntityByName).toHaveBeenCalledWith(entityName, { + expect(catalogClient.getEntityByRef).toHaveBeenCalledWith(entityName, { token: undefined, }); }); diff --git a/plugins/todo-backend/src/service/TodoReaderService.ts b/plugins/todo-backend/src/service/TodoReaderService.ts index 86f53057da..4397f34045 100644 --- a/plugins/todo-backend/src/service/TodoReaderService.ts +++ b/plugins/todo-backend/src/service/TodoReaderService.ts @@ -66,7 +66,7 @@ export class TodoReaderService implements TodoService { throw new InputError('Entity filter is required to list TODOs'); } const token = options?.token; - const entity = await this.catalogClient.getEntityByName(req.entity, { + const entity = await this.catalogClient.getEntityByRef(req.entity, { token, }); if (!entity) { diff --git a/plugins/todo-backend/src/service/router.ts b/plugins/todo-backend/src/service/router.ts index a0e6a6ec94..33356a83d6 100644 --- a/plugins/todo-backend/src/service/router.ts +++ b/plugins/todo-backend/src/service/router.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { EntityName, parseEntityRef } from '@backstage/catalog-model'; +import { CompoundEntityRef, parseEntityRef } from '@backstage/catalog-model'; import { InputError } from '@backstage/errors'; import express from 'express'; import Router from 'express-promise-router'; @@ -52,7 +52,7 @@ export async function createRouter( if (entityRef && typeof entityRef !== 'string') { throw new InputError(`entity query must be a string`); } - let entity: EntityName | undefined = undefined; + let entity: CompoundEntityRef | undefined = undefined; if (entityRef) { try { entity = parseEntityRef(entityRef); diff --git a/plugins/todo-backend/src/service/types.ts b/plugins/todo-backend/src/service/types.ts index 90ee6bc913..88d2f2b340 100644 --- a/plugins/todo-backend/src/service/types.ts +++ b/plugins/todo-backend/src/service/types.ts @@ -14,12 +14,12 @@ * limitations under the License. */ -import { EntityName } from '@backstage/catalog-model'; +import { CompoundEntityRef } from '@backstage/catalog-model'; import { TodoItem } from '../lib'; /** @public */ export type ListTodosRequest = { - entity?: EntityName; + entity?: CompoundEntityRef; offset?: number; limit?: number; orderBy?: { diff --git a/plugins/todo/src/plugin.test.tsx b/plugins/todo/src/plugin.test.tsx index 57b9be291d..af4704c16f 100644 --- a/plugins/todo/src/plugin.test.tsx +++ b/plugins/todo/src/plugin.test.tsx @@ -19,6 +19,7 @@ import { Route } from 'react-router'; import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; import { todoPlugin, EntityTodoContent } from './plugin'; import { todoApiRef } from './api'; +import { EntityProvider } from '@backstage/plugin-catalog-react'; describe('todo', () => { it('should export plugin', () => { @@ -47,7 +48,15 @@ describe('todo', () => { ], ]} > - } /> + + } /> + , ); diff --git a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx index 8cfb3a7257..7025167658 100644 --- a/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx +++ b/plugins/user-settings/src/components/AuthProviders/DefaultProviderSettings.tsx @@ -17,11 +17,9 @@ import Star from '@material-ui/icons/Star'; import React from 'react'; import { ProviderSettingsItem } from './ProviderSettingsItem'; import { - auth0AuthApiRef, githubAuthApiRef, gitlabAuthApiRef, googleAuthApiRef, - oauth2ApiRef, oktaAuthApiRef, microsoftAuthApiRef, bitbucketAuthApiRef, @@ -66,14 +64,6 @@ export const DefaultProviderSettings = ({ configuredProviders }: Props) => ( icon={Star} /> )} - {configuredProviders.includes('auth0') && ( - - )} {configuredProviders.includes('okta') && ( ( icon={Star} /> )} - {configuredProviders.includes('oauth2') && ( - - )} ); diff --git a/scripts/api-extractor.ts b/scripts/api-extractor.ts index 6635dd8b22..afe2c1a1bb 100644 --- a/scripts/api-extractor.ts +++ b/scripts/api-extractor.ts @@ -213,6 +213,7 @@ const NO_WARNING_PACKAGES = [ 'packages/errors', 'packages/integration', 'packages/integration-react', + 'packages/search-common', 'packages/techdocs-common', 'packages/test-utils', 'packages/theme', @@ -235,6 +236,7 @@ const NO_WARNING_PACKAGES = [ 'plugins/scaffolder-backend-module-rails', 'plugins/scaffolder-backend-module-yeoman', 'plugins/scaffolder-common', + 'plugins/search-backend-node', 'plugins/techdocs-backend', 'plugins/tech-insights', 'plugins/tech-insights-backend', diff --git a/scripts/list-deprecations.js b/scripts/list-deprecations.js index 9a159927fc..8047158138 100755 --- a/scripts/list-deprecations.js +++ b/scripts/list-deprecations.js @@ -57,9 +57,7 @@ class ReleaseProvider { ); // Filter out just the releases - const releases = tagOutput - .split('\n') - .filter(l => l.startsWith('release-')); + const releases = tagOutput.split('\n').filter(l => l.startsWith('v')); // Then find the earliest release that affected our package for (const release of releases) { diff --git a/storybook/.storybook/apis.js b/storybook/.storybook/apis.js index 4c0813a8e9..37cc8eb605 100644 --- a/storybook/.storybook/apis.js +++ b/storybook/.storybook/apis.js @@ -5,10 +5,8 @@ import { GithubAuth, GitlabAuth, GoogleAuth, - OAuth2, OAuthRequestManager, OktaAuth, - Auth0Auth, ConfigReader, LocalStorageFeatureFlags, } from '@backstage/core-app-api'; @@ -20,10 +18,8 @@ import { gitlabAuthApiRef, googleAuthApiRef, identityApiRef, - oauth2ApiRef, oauthRequestApiRef, oktaAuthApiRef, - auth0AuthApiRef, configApiRef, featureFlagsApiRef, } from '@backstage/core-plugin-api'; @@ -59,16 +55,6 @@ const oktaAuthApi = OktaAuth.create({ basePath: '/auth/', oauthRequestApi, }); -const auth0AuthApi = Auth0Auth.create({ - apiOrigin: 'http://localhost:7007', - basePath: '/auth/', - oauthRequestApi, -}); -const oauth2Api = OAuth2.create({ - apiOrigin: 'http://localhost:7007', - basePath: '/auth/', - oauthRequestApi, -}); export const apis = [ [configApiRef, configApi], @@ -81,6 +67,4 @@ export const apis = [ [githubAuthApiRef, githubAuthApi], [gitlabAuthApiRef, gitlabAuthApi], [oktaAuthApiRef, oktaAuthApi], - [auth0AuthApiRef, auth0AuthApi], - [oauth2ApiRef, oauth2Api], ]; diff --git a/yarn.lock b/yarn.lock index 561b736e7e..2b13c60201 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1359,7 +1359,9 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.10.0": - version "0.11.0" + version "0.10.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.10.1.tgz#dcc3415eb4d4ee3d437355c477e85c7479626b3b" + integrity sha512-c004aQeO9cxtSZZc2iBcE6eoqurQLdj7YUm8mHWs8hEaPTA2UPVHawt+wlt89VywkI89X0wF7BuXV2LKVUfXvw== dependencies: "@backstage/config" "^0.1.15" "@backstage/errors" "^0.2.2" @@ -1371,16 +1373,20 @@ uuid "^8.0.0" "@backstage/catalog-model@^0.9.7": - version "0.11.0" + version "0.9.10" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.9.10.tgz#bd5662e1ad7bd7c9604f3f45d055c99b5b2bb87f" + integrity sha512-KhCjbZKhS5zZhHiGHmBMq6hDGDshMSZOPGXehtdhr6/oW7Ee5fDcOnhMqreCi1Ebm4RIWJhZcRrxO6X1TTi4TQ== dependencies: - "@backstage/config" "^0.1.15" - "@backstage/errors" "^0.2.2" - "@backstage/types" "^0.1.3" + "@backstage/config" "^0.1.13" + "@backstage/errors" "^0.2.0" + "@backstage/types" "^0.1.1" "@types/json-schema" "^7.0.5" + "@types/yup" "^0.29.13" ajv "^7.0.3" json-schema "^0.4.0" lodash "^4.17.21" uuid "^8.0.0" + yup "^0.32.9" "@backstage/core-plugin-api@^0.6.0", "@backstage/core-plugin-api@^0.6.1": version "0.6.1" @@ -1808,14 +1814,14 @@ ts-node "^9" tslib "^2" -"@eslint/eslintrc@^1.0.5": - version "1.0.5" - resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.0.5.tgz#33f1b838dbf1f923bfa517e008362b78ddbbf318" - integrity sha512-BLxsnmK3KyPunz5wmCCpqy0YelEoxxGmH73Is+Z74oOTMtExcjkr3dDR6quwrjh1YspA8DH9gnX1o069KiS9AQ== +"@eslint/eslintrc@^1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.0.tgz#7ce1547a5c46dfe56e1e45c3c9ed18038c721c6a" + integrity sha512-igm9SjJHNEJRiUnecP/1R5T3wKLEJ7pL6e2P+GUSfCd0dGjPYYZve08uzw8L2J8foVHFz+NGu12JxRcU2gGo6w== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.2.0" + espree "^9.3.1" globals "^13.9.0" ignore "^4.0.6" import-fresh "^3.2.1" @@ -2557,7 +2563,17 @@ dependencies: "@hapi/hoek" "^9.0.0" -"@hot-loader/react-dom@^17.0.2": +"@hot-loader/react-dom-v16@npm:@hot-loader/react-dom@^16.0.2": + version "16.14.0" + resolved "https://registry.npmjs.org/@hot-loader/react-dom/-/react-dom-16.14.0.tgz#3cfc64e40bb78fa623e59b582b8f09dcdaad648a" + integrity sha512-EN9czvcLsMYmSDo5yRKZOAq3ZGRlDpad1gPtX0NdMMomJXcPE3yFSeFzE94X/NjOaiSVimB7LuqPYpkWVaIi4Q== + dependencies: + loose-envify "^1.1.0" + object-assign "^4.1.1" + prop-types "^15.6.2" + scheduler "^0.19.1" + +"@hot-loader/react-dom-v17@npm:@hot-loader/react-dom@^17.0.2": version "17.0.2" resolved "https://registry.npmjs.org/@hot-loader/react-dom/-/react-dom-17.0.2.tgz#0b24e484093e8f97eb5c72bebdda44fc20bc8400" integrity sha512-G2RZrFhsQClS+bdDh/Ojpk3SgocLPUGnvnJDTQYnmKSSwXtU+Yh+8QMs+Ia3zaAvBiOSpIIDSUxuN69cvKqrWg== @@ -4050,9 +4066,9 @@ integrity sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA== "@microsoft/microsoft-graph-types@^2.6.0": - version "2.13.0" - resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-2.13.0.tgz#aa584e4897665df5a9c8869a226264cd6ec5882b" - integrity sha512-63FfWBLcyNo8tMP4oPcdqHQvk4ehuWpiUMjVLD7zJXPENIowpdwudP969AALkKzlwsjWImamdivGKd2Zc8Z1Uw== + version "2.15.0" + resolved "https://registry.npmjs.org/@microsoft/microsoft-graph-types/-/microsoft-graph-types-2.15.0.tgz#1705ea1ce84c3de4705957392d7f0e3ae465c9f8" + integrity sha512-EyuOpZs55HUoC37Ujrp6IRgE5ghf/wtDrlWuJm7J/DKoB7B/Iek7eXdavTygx2uBeDZ5b4jXXvwl4PiDLlEcsw== "@microsoft/tsdoc-config@~0.15.2": version "0.15.2" @@ -4848,9 +4864,9 @@ react-use "^17.2.4" "@rollup/plugin-commonjs@^21.0.1": - version "21.0.1" - resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-21.0.1.tgz#1e57c81ae1518e4df0954d681c642e7d94588fee" - integrity sha512-EA+g22lbNJ8p5kuZJUYyhhDK7WgJckW5g4pNN7n4mAFUM96VuwUnNT3xr2Db2iCZPI1pJPbGyfT5mS9T1dHfMg== + version "21.0.2" + resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-21.0.2.tgz#0b9c539aa1837c94abfaf87945838b0fc8564891" + integrity sha512-d/OmjaLVO4j/aQX69bwpWPpbvI3TJkQuxoAk7BH8ew1PyoMBLTOuvJTjzG8oEoW7drIIqB0KCJtfFLu/2GClWg== dependencies: "@rollup/pluginutils" "^3.1.0" commondir "^1.0.1" @@ -6236,9 +6252,9 @@ redux "^4.0.0" "@types/react-sparklines@^1.7.0": - version "1.7.0" - resolved "https://registry.npmjs.org/@types/react-sparklines/-/react-sparklines-1.7.0.tgz#f956d0f7b0e746ad445ce1cd250fe81f8a384684" - integrity sha512-Vd+cME7+Yy3kFNhnid9EBIKiyCQ/at8nqDczIs0UYfIB8AtaRJPqekigv02biOsIbQCvxyvIAIjiTKOC+hHNbA== + version "1.7.2" + resolved "https://registry.npmjs.org/@types/react-sparklines/-/react-sparklines-1.7.2.tgz#c14e80623abd3669a10f18d13f6fb9fbdc322f70" + integrity sha512-N1GwO7Ri5C5fE8+CxhiDntuSw1qYdGytBuedKrCxWpaojXm4WnfygbdBdc5sXGX7feMxDXBy9MNhxoUTwrMl4A== dependencies: "@types/react" "*" @@ -6710,15 +6726,23 @@ eslint-utils "^3.0.0" "@typescript-eslint/parser@^5.9.0": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.9.1.tgz#b114011010a87e17b3265ca715e16c76a9834cef" - integrity sha512-PLYO0AmwD6s6n0ZQB5kqPgfvh73p0+VqopQQLuNfi7Lm0EpfKyDalchpVwkE+81k5HeiRrTV/9w1aNHzjD7C4g== + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.13.0.tgz#0394ed8f2f849273c0bf4b811994d177112ced5c" + integrity sha512-GdrU4GvBE29tm2RqWOM0P5QfCtgCyN4hXICj/X9ibKED16136l9ZpoJvCL5pSKtmJzA+NRDzQ312wWMejCVVfg== dependencies: - "@typescript-eslint/scope-manager" "5.9.1" - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/typescript-estree" "5.9.1" + "@typescript-eslint/scope-manager" "5.13.0" + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/typescript-estree" "5.13.0" debug "^4.3.2" +"@typescript-eslint/scope-manager@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.13.0.tgz#cf6aff61ca497cb19f0397eea8444a58f46156b6" + integrity sha512-T4N8UvKYDSfVYdmJq7g2IPJYCRzwtp74KyDZytkR4OL3NRupvswvmJQJ4CX5tDSurW2cvCc1Ia1qM7d0jpa7IA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + "@typescript-eslint/scope-manager@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.0.tgz#02dfef920290c1dcd7b1999455a3eaae7a1a3117" @@ -6727,14 +6751,6 @@ "@typescript-eslint/types" "5.9.0" "@typescript-eslint/visitor-keys" "5.9.0" -"@typescript-eslint/scope-manager@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.9.1.tgz#6c27be89f1a9409f284d95dfa08ee3400166fe69" - integrity sha512-8BwvWkho3B/UOtzRyW07ffJXPaLSUKFBjpq8aqsRvu6HdEuzCY57+ffT7QoV4QXJXWSU1+7g3wE4AlgImmQ9pQ== - dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - "@typescript-eslint/type-utils@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.9.0.tgz#fd5963ead04bc9b7af9c3a8e534d8d39f1ce5f93" @@ -6744,15 +6760,28 @@ debug "^4.3.2" tsutils "^3.21.0" +"@typescript-eslint/types@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.13.0.tgz#da1de4ae905b1b9ff682cab0bed6b2e3be9c04e5" + integrity sha512-LmE/KO6DUy0nFY/OoQU0XelnmDt+V8lPQhh8MOVa7Y5k2gGRd6U9Kp3wAjhB4OHg57tUO0nOnwYQhRRyEAyOyg== + "@typescript-eslint/types@5.9.0": version "5.9.0" resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.0.tgz#e5619803e39d24a03b3369506df196355736e1a3" integrity sha512-mWp6/b56Umo1rwyGCk8fPIzb9Migo8YOniBGPAQDNC6C52SeyNGN4gsVwQTAR+RS2L5xyajON4hOLwAGwPtUwg== -"@typescript-eslint/types@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.9.1.tgz#1bef8f238a2fb32ebc6ff6d75020d9f47a1593c6" - integrity sha512-SsWegWudWpkZCwwYcKoDwuAjoZXnM1y2EbEerTHho19Hmm+bQ56QG4L4jrtCu0bI5STaRTvRTZmjprWlTw/5NQ== +"@typescript-eslint/typescript-estree@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.13.0.tgz#b37c07b748ff030a3e93d87c842714e020b78141" + integrity sha512-Q9cQow0DeLjnp5DuEDjLZ6JIkwGx3oYZe+BfcNuw/POhtpcxMTy18Icl6BJqTSd+3ftsrfuVb7mNHRZf7xiaNA== + dependencies: + "@typescript-eslint/types" "5.13.0" + "@typescript-eslint/visitor-keys" "5.13.0" + debug "^4.3.2" + globby "^11.0.4" + is-glob "^4.0.3" + semver "^7.3.5" + tsutils "^3.21.0" "@typescript-eslint/typescript-estree@5.9.0": version "5.9.0" @@ -6767,18 +6796,13 @@ semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/typescript-estree@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.9.1.tgz#d5b996f49476495070d2b8dd354861cf33c005d6" - integrity sha512-gL1sP6A/KG0HwrahVXI9fZyeVTxEYV//6PmcOn1tD0rw8VhUWYeZeuWHwwhnewnvEMcHjhnJLOBhA9rK4vmb8A== +"@typescript-eslint/visitor-keys@5.13.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.13.0.tgz#f45ff55bcce16403b221ac9240fbeeae4764f0fd" + integrity sha512-HLKEAS/qA1V7d9EzcpLFykTePmOQqOFim8oCvhY3pZgQ8Hi38hYpHd9e5GN6nQBFQNecNhws5wkS9Y5XIO0s/g== dependencies: - "@typescript-eslint/types" "5.9.1" - "@typescript-eslint/visitor-keys" "5.9.1" - debug "^4.3.2" - globby "^11.0.4" - is-glob "^4.0.3" - semver "^7.3.5" - tsutils "^3.21.0" + "@typescript-eslint/types" "5.13.0" + eslint-visitor-keys "^3.0.0" "@typescript-eslint/visitor-keys@5.9.0": version "5.9.0" @@ -6788,14 +6812,6 @@ "@typescript-eslint/types" "5.9.0" eslint-visitor-keys "^3.0.0" -"@typescript-eslint/visitor-keys@5.9.1": - version "5.9.1" - resolved "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.9.1.tgz#f52206f38128dd4f675cf28070a41596eee985b7" - integrity sha512-Xh37pNz9e9ryW4TVdwiFzmr4hloty8cFj8GTWMXh3Z8swGwyQWeCcNgF0hm6t09iZd6eiZmIf4zHedQVP6TVtg== - dependencies: - "@typescript-eslint/types" "5.9.1" - eslint-visitor-keys "^3.0.0" - "@vscode/sqlite3@^5.0.7": version "5.0.7" resolved "https://registry.npmjs.org/@vscode/sqlite3/-/sqlite3-5.0.7.tgz#358df36bb0e9e735c54785e3e4b9b2dce1d32895" @@ -7284,7 +7300,7 @@ apollo-server-caching@^3.3.0: dependencies: lru-cache "^6.0.0" -apollo-server-core@^3.6.1, apollo-server-core@^3.6.3: +apollo-server-core@^3.6.3: version "3.6.3" resolved "https://registry.npmjs.org/apollo-server-core/-/apollo-server-core-3.6.3.tgz#6b12ffa1af8bc8799930f72360090834915033d1" integrity sha512-TFJmAlI6vPp1MHOSXqYkE6leAyMekWv/D/3ma11uETkcd3EPjERGmxtTXPJElMVEkOK9BEElYKthCrH7bjYLuw== @@ -7322,7 +7338,7 @@ apollo-server-errors@^3.3.1: resolved "https://registry.npmjs.org/apollo-server-errors/-/apollo-server-errors-3.3.1.tgz#ba5c00cdaa33d4cbd09779f8cb6f47475d1cd655" integrity sha512-xnZJ5QWs6FixHICXHxUfm+ZWqqxrNuPlQ+kj5m6RtEgIpekOPssH/SD9gf2B4HuWV0QozorrygwZnux8POvyPA== -apollo-server-express@^3.0.0, apollo-server-express@^3.6.1: +apollo-server-express@^3.0.0, apollo-server-express@^3.6.3: version "3.6.3" resolved "https://registry.npmjs.org/apollo-server-express/-/apollo-server-express-3.6.3.tgz#5daf58bf0bdf0107ded7cd52c7e6ce6cd32c8b44" integrity sha512-3CjahZ+n+1T7pHH1qW1B6Ns0BzwOMeupAp2u0+M8ruOmE/e7VKn0OSOQQckZ8Z2AcWxWeno9K89fIv3PoSYgYA== @@ -7356,12 +7372,12 @@ apollo-server-types@^3.5.1: apollo-server-env "^4.2.1" apollo-server@^3.0.0: - version "3.6.1" - resolved "https://registry.npmjs.org/apollo-server/-/apollo-server-3.6.1.tgz#29420b1c0cddbf2e18147a3ca7299485f17137a2" - integrity sha512-Y2MY2/WvaTiofVoIR5ZIYt6c6wX8klZRaXI9x+7JBiFV9HMcOuLLpU3+P4r2EVXuN1LLe82m1PgiAYr+a1OmQg== + version "3.6.3" + resolved "https://registry.npmjs.org/apollo-server/-/apollo-server-3.6.3.tgz#0ba0ddb2835ccf27056d20b6f5b83b0ce9545a79" + integrity sha512-kNvOiDNkIaO+MsfR9v40Vz4ArlDdc9VwVKGJy5dniLW9AoDa/tSF99m8ItfGoMypqlRPMgrNGxkMuToBnvYXNQ== dependencies: - apollo-server-core "^3.6.1" - apollo-server-express "^3.6.1" + apollo-server-core "^3.6.3" + apollo-server-express "^3.6.3" express "^4.17.1" aproba@^1.0.3: @@ -11532,9 +11548,9 @@ escodegen@^2.0.0: source-map "~0.6.1" eslint-config-prettier@^8.3.0: - version "8.3.0" - resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.3.0.tgz#f7471b20b6fe8a9a9254cc684454202886a2dd7a" - integrity sha512-BgZuLUSeKzvlL/VUjx/Yb787VQ26RU3gGjA3iiFvdsp/2bMfVIWUVP7tjxtjS0e+HP409cPlPvNkQloz8C91ew== + version "8.5.0" + resolved "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz#5a81680ec934beca02c7b1a61cf8ca34b66feab1" + integrity sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q== eslint-formatter-friendly@^7.0.0: version "7.0.0" @@ -11688,10 +11704,10 @@ eslint-scope@5.1.1, eslint-scope@^5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-scope@^7.1.0: - version "7.1.0" - resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.0.tgz#c1f6ea30ac583031f203d65c73e723b01298f153" - integrity sha512-aWwkhnS0qAXqNOgKOK0dJ2nvzEbhEvpy8OlJ9kZ0FeZnA6zpjv1/Vei+puGFFX7zkPCkHHXb7IDX3A+7yPrRWg== +eslint-scope@^7.1.1: + version "7.1.1" + resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz#fff34894c2f65e5226d3041ac480b4513a163642" + integrity sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" @@ -11708,10 +11724,10 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== -eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.1.0, eslint-visitor-keys@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.2.0.tgz#6fbb166a6798ee5991358bc2daa1ba76cc1254a1" - integrity sha512-IOzT0X126zn7ALX0dwFiUQEdsfzrm4+ISsQS8nukaJXwEyYKRSnEIIDULYg1mCtGp7UUXgfGl7BIolXREQK+XQ== +eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.3.0: + version "3.3.0" + resolved "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" + integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== eslint-webpack-plugin@^2.6.0: version "2.6.0" @@ -11726,11 +11742,11 @@ eslint-webpack-plugin@^2.6.0: schema-utils "^3.1.1" eslint@^8.6.0: - version "8.7.0" - resolved "https://registry.npmjs.org/eslint/-/eslint-8.7.0.tgz#22e036842ee5b7cf87b03fe237731675b4d3633c" - integrity sha512-ifHYzkBGrzS2iDU7KjhCAVMGCvF6M3Xfs8X8b37cgrUlDt6bWRTpRh6T/gtSXv1HJ/BUGgmjvNvOEGu85Iif7w== + version "8.10.0" + resolved "https://registry.npmjs.org/eslint/-/eslint-8.10.0.tgz#931be395eb60f900c01658b278e05b6dae47199d" + integrity sha512-tcI1D9lfVec+R4LE1mNDnzoJ/f71Kl/9Cv4nG47jOueCMBrCCKYXr4AUVS7go6mWYGFD4+EoN6+eXSrEbRzXVw== dependencies: - "@eslint/eslintrc" "^1.0.5" + "@eslint/eslintrc" "^1.2.0" "@humanwhocodes/config-array" "^0.9.2" ajv "^6.10.0" chalk "^4.0.0" @@ -11738,10 +11754,10 @@ eslint@^8.6.0: debug "^4.3.2" doctrine "^3.0.0" escape-string-regexp "^4.0.0" - eslint-scope "^7.1.0" + eslint-scope "^7.1.1" eslint-utils "^3.0.0" - eslint-visitor-keys "^3.2.0" - espree "^9.3.0" + eslint-visitor-keys "^3.3.0" + espree "^9.3.1" esquery "^1.4.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -11771,14 +11787,14 @@ esm@^3.2.25: resolved "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10" integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA== -espree@^9.2.0, espree@^9.3.0: - version "9.3.0" - resolved "https://registry.npmjs.org/espree/-/espree-9.3.0.tgz#c1240d79183b72aaee6ccfa5a90bc9111df085a8" - integrity sha512-d/5nCsb0JcqsSEeQzFZ8DH1RmxPcglRWh24EFTlUEmCKoehXGdpsx0RkHDubqUI8LSAIKMQp4r9SzQ3n+sm4HQ== +espree@^9.3.1: + version "9.3.1" + resolved "https://registry.npmjs.org/espree/-/espree-9.3.1.tgz#8793b4bc27ea4c778c19908e0719e7b8f4115bcd" + integrity sha512-bvdyLmJMfwkV3NCRl5ZhJf22zBFo1y8bYh3VYb+bfzqNB4Je68P2sSuXyuFquzWLebHpNd2/d5uv7yoP9ISnGQ== dependencies: acorn "^8.7.0" acorn-jsx "^5.3.1" - eslint-visitor-keys "^3.1.0" + eslint-visitor-keys "^3.3.0" esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0: version "4.0.1" @@ -16235,9 +16251,9 @@ ldap-filter@^0.3.3: assert-plus "^1.0.0" ldapjs@^2.2.0: - version "2.3.1" - resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.1.tgz#04136815fb1f21d692ac87fab5961a04d86e8b04" - integrity sha512-kf0tHHLrpwKaBAQOhYHXgdeh2PkFuCCxWgLb1MRn67ZQVo787D2pij3mmHVZx193GIdM8xcfi8HF6AIYYnj0fQ== + version "2.3.2" + resolved "https://registry.npmjs.org/ldapjs/-/ldapjs-2.3.2.tgz#a599d081519f70462941cc33a50e9354c32f35b7" + integrity sha512-FU+GR/qbQ96WUZ2DUb7FzaEybYvv3240wTVPcbsdELB3o4cK92zGVjntsh68siVkLeCmlCcsd/cIQzyGXSS7LA== dependencies: abstract-logging "^2.0.0" asn1 "^0.2.4" @@ -18139,15 +18155,10 @@ nanoclone@^0.2.1: resolved "https://registry.npmjs.org/nanoclone/-/nanoclone-0.2.1.tgz#dd4090f8f1a110d26bb32c49ed2f5b9235209ed4" integrity sha512-wynEP02LmIbLpcYw8uBKpcfF6dmg2vcpKqxeH5UcoKEYdExslsdUA4ugFauuaeYdTB76ez6gJW8XAZ6CgkXYxA== -nanoid@^3.1.23: - version "3.2.0" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" - integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== - -nanoid@^3.2.0: - version "3.3.0" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.0.tgz#5906f776fd886c66c24f3653e0c46fcb1d4ad6b0" - integrity sha512-JzxqqT5u/x+/KOFSd7JP15DOo9nOoHpx6DYatqIHUW2+flybkm+mdcraotSQR5WcnZr+qhGVh8Ted0KdfSMxlg== +nanoid@^3.1.23, nanoid@^3.3.1: + version "3.3.1" + resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz#6347a18cac88af88f58af0b3594b723d5e99bb35" + integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw== nanomatch@^1.2.9: version "1.2.13" @@ -20142,11 +20153,11 @@ postcss-value-parser@^4.0.2, postcss-value-parser@^4.1.0, postcss-value-parser@^ integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== postcss@^8.1.0, postcss@^8.4.5: - version "8.4.6" - resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.6.tgz#c5ff3c3c457a23864f32cb45ac9b741498a09ae1" - integrity sha512-OovjwIzs9Te46vlEx7+uXB0PLijpwjXGKXjVGGPIGubGpq7uh5Xgf6D6FiJ/SzJMBosHDp6a2hiXOS97iBXcaA== + version "8.4.7" + resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.7.tgz#f99862069ec4541de386bf57f5660a6c7a0875a8" + integrity sha512-L9Ye3r6hkkCeOETQX6iOaWZgjp3LL6Lpqm6EtgbKrgqGGteRMNb9vzBfRL96YOSu8o7x3MfIH9Mo5cPJFGrW6A== dependencies: - nanoid "^3.2.0" + nanoid "^3.3.1" picocolors "^1.0.0" source-map-js "^1.0.2" @@ -20650,9 +20661,9 @@ raw-body@2.4.3: unpipe "1.0.0" raw-body@^2.4.1: - version "2.5.0" - resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.0.tgz#865890d9435243e9fe6141feb4decf929a6e1525" - integrity sha512-XpyZ6O7PVu3ItMQl0LslfsRoKxMOxi3SzDkrOtxMES5AqLFpYjQCryxI4LGygUN2jL+RgFsPkMPPlG7cg/47+A== + version "2.5.1" + resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: bytes "3.1.2" http-errors "2.0.0"